Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: gate GGML_CUDA_P2P on a confirmed NVLink fabric, not a product name (#10646)

Daniel Han10a95da91652
diff --git a/scripts/p2p_integrity_probe.py b/scripts/p2p_integrity_probe.py
new file mode 100644
index 00000000000..3dabf759aa3
--- /dev/null
+++ b/scripts/p2p_integrity_probe.py
@@ -0,0 +1,261 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Probe: do GPU-to-GPU peer copies on this host actually transfer the data?
+
+On bare-metal Linux with a translating IOMMU, a PCIe peer-to-peer copy between two
+NVIDIA GPUs can be discarded by the chipset while CUDA reports ``cudaSuccess`` and
+``torch.cuda.can_device_access_peer`` returns True, so nothing notices. In
+llama.cpp with ``GGML_CUDA_P2P`` set, the model emits ``!!!!!``, a repeated token
+or word salad, looking exactly like a broken quant or chat template (#10613).
+
+The destination is sentinel-filled before each copy, so a transfer that moves
+nothing is distinguishable from one that legitimately writes zeros, and every
+ordered pair is tested at several sizes, since small copies can take a different
+path than large ones. Run with no arguments on the host in question::
+
+ python scripts/p2p_integrity_probe.py
+
+Exit 0: every peer copy transferred intact. Exit 1: at least one dropped data, so
+do NOT set ``GGML_CUDA_P2P`` here. Exit 2: could not run (no torch, <2 GPUs).
+
+``GGML_CUDA_P2P=0`` does not disable the flag upstream: llama.cpp tests it for
+presence, not value, so it must be unset entirely. Unsloth Studio unsets it for
+you and gates the flag on verified NVLink.
+"""
+
+from __future__ import annotations
+
+import argparse
+import shutil
+import subprocess
+import sys
+
+SENTINEL = -7.0
+SIZES_MIB = (1, 4, 16, 64)
+REPEATS = 3
+
+
+def _print_topology() -> None:
+ """Show `nvidia-smi topo -m`, the cheap read that answers this in advance. NV#
+ is NVLink, which never traverses the PCIe root complex; NODE / PHB / PXB / PIX
+ / SYS all do."""
+ if shutil.which("nvidia-smi") is None:
+ print("nvidia-smi not found; skipping topology.\n")
+ return
+ try:
+ out = subprocess.run(
+ ["nvidia-smi", "topo", "-m"],
+ capture_output = True,
+ text = True,
+ timeout = 30,
+ )
+ except Exception as e: # noqa: BLE001 -- diagnostic, never fatal
+ print(f"topology read failed: {e}\n")
+ return
+ if out.returncode != 0:
+ print("topology read returned non-zero; skipping.\n")
+ return
+ print("=== nvidia-smi topo -m ===")
+ # The legend after the matrix is not what the reader needs.
+ for line in out.stdout.splitlines():
+ if line.strip().startswith("Legend"):
+ break
+ print(line)
+ print()
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description = __doc__.splitlines()[0])
+ parser.add_argument(
+ "--sizes-mib",
+ type = int,
+ nargs = "+",
+ default = list(SIZES_MIB),
+ help = "copy sizes to test, in MiB",
+ )
+ parser.add_argument(
+ "--repeats",
+ type = int,
+ default = REPEATS,
+ help = "copies per pair per size",
+ )
+ args = parser.parse_args()
+
+ # A probe that tests nothing must not print PASS: its whole job is to license
+ # setting GGML_CUDA_P2P, so an empty run is the worst answer it could give.
+ if args.repeats <= 0:
+ parser.error("--repeats must be positive")
+ if any(size <= 0 for size in args.sizes_mib):
+ parser.error("--sizes-mib values must all be positive")
+
+ _print_topology()
+
+ try:
+ import torch
+ except ImportError:
+ print("torch is not installed; cannot probe peer copies.")
+ return 2
+
+ # AMD SDK wheels leave version.hip unset and only encode "rocm" in
+ # __version__, which is why the backend's _torch_is_rocm checks both.
+ if (
+ getattr(torch.version, "hip", None) is not None
+ or "rocm" in getattr(torch, "__version__", "").lower()
+ ):
+ # ROCm reuses the torch.cuda namespace, so every test below would run and
+ # PASS on AMD, then recommend GGML_CUDA_P2P, which only the CUDA backend
+ # reads. A pass ending in a no-op instruction is worse than no answer.
+ print(
+ "This is a ROCm/HIP build of torch. GGML_CUDA_P2P is read only by\n"
+ "llama.cpp's CUDA backend, so this probe has no advice to give for AMD\n"
+ "GPUs and does not test them."
+ )
+ return 2
+
+ if not (torch.cuda.is_available() and torch.cuda.device_count() >= 2):
+ print(
+ f"need at least 2 visible CUDA GPUs, found "
+ f"{torch.cuda.device_count() if torch.cuda.is_available() else 0}. "
+ "Peer copies are not used on a single GPU, so this host is unaffected."
+ )
+ return 2
+
+ count = torch.cuda.device_count()
+ print(f"=== {count} visible CUDA GPUs ===")
+ for i in range(count):
+ print(f" cuda:{i} {torch.cuda.get_device_name(i)}")
+ print()
+
+ failures = 0
+ inconclusive = 0
+ checked = 0
+ print(f"{'pair':>10} {'size':>8} {'peer access':>11} verdict")
+ for src in range(count):
+ for dst in range(count):
+ if src == dst:
+ continue
+ can = torch.cuda.can_device_access_peer(src, dst)
+ for mib in args.sizes_mib:
+ elements = mib * 1024 * 1024 // 4
+ worst_dropped = 0
+ worst_wrong = 0
+ error = None
+ setup_error = None
+ for _ in range(args.repeats):
+ s = d = None
+ try:
+ try:
+ s = torch.arange(elements, dtype = torch.float32, device = f"cuda:{src}")
+ d = torch.full(
+ (elements,),
+ SENTINEL,
+ dtype = torch.float32,
+ device = f"cuda:{dst}",
+ )
+ except Exception as alloc_exc: # noqa: BLE001
+ # No transfer was attempted (usually a resident model
+ # holds the memory), so this is inconclusive, NOT
+ # evidence that peer copies drop data.
+ setup_error = alloc_exc
+ break
+ d.copy_(s)
+ torch.cuda.synchronize(src)
+ torch.cuda.synchronize(dst)
+ # Compare against the source, not just the sentinel: a
+ # scrambled transfer overwrites it and would score as a
+ # pass. The sentinel count stays because "never written"
+ # is a different diagnosis, a dropped DMA.
+ host_dst = d.cpu()
+ wrong = int((host_dst != s.cpu()).sum())
+ dropped = int((host_dst == SENTINEL).sum())
+ worst_wrong = max(worst_wrong, wrong)
+ worst_dropped = max(worst_dropped, dropped)
+ except Exception as e: # noqa: BLE001 -- a raising copy is also a failure
+ error = e
+ break
+ finally:
+ del s, d
+ torch.cuda.empty_cache()
+
+ checked += 1
+ pair = f"{src}->{dst}"
+ access = "yes" if can else "no"
+ # Corruption seen on an earlier repeat outranks a later allocation
+ # failure: repeats exist to catch INTERMITTENT drops, so downgrading
+ # a real mismatch to SKIPPED would hide what they are for.
+ if setup_error is not None and not worst_wrong:
+ inconclusive += 1
+ print(
+ f"{pair:>10} {str(mib) + ' MiB':>8} {access:>11} "
+ f"SKIPPED (could not allocate): {setup_error}"
+ )
+ elif error is not None:
+ failures += 1
+ print(f"{pair:>10} {str(mib) + ' MiB':>8} {access:>11} ERROR: {error}")
+ elif worst_wrong:
+ failures += 1
+ pct = 100.0 * worst_wrong / elements
+ detail = (
+ f"{worst_dropped} never written"
+ if worst_dropped
+ else "written with the wrong data"
+ )
+ print(
+ f"{pair:>10} {str(mib) + ' MiB':>8} {access:>11} "
+ f"CORRUPT: {worst_wrong}/{elements} elements wrong "
+ f"({pct:.0f}%, {detail})"
+ )
+ else:
+ print(f"{pair:>10} {str(mib) + ' MiB':>8} {access:>11} ok")
+
+ print()
+ if failures:
+ print(
+ f"FAIL: {failures} of {checked} peer-copy tests lost data.\n"
+ "\n"
+ "Peer-to-peer copies are NOT safe on this host. Do not set GGML_CUDA_P2P,\n"
+ "and note that GGML_CUDA_P2P=0 does not disable it (llama.cpp tests the\n"
+ "variable for presence, not value) -- it must be unset entirely.\n"
+ "\n"
+ "If CUDA reported success above while dropping data, check the kernel log\n"
+ "for IOMMU faults: dmesg | grep -i -E 'DMAR|AMD-Vi'\n"
+ "NVIDIA does not support bare-metal PCIe peer-to-peer with the IOMMU in\n"
+ "translating mode; booting with intel_iommu=off / amd_iommu=off (or\n"
+ "iommu=pt) is the usual remedy where that is acceptable."
+ )
+ return 1
+
+ if inconclusive:
+ # Exit 1 is for copies that were tested and lost data. Nothing was tested
+ # here, so "unsafe" would push someone off a working optimisation.
+ print(
+ f"INCONCLUSIVE: {inconclusive} of {checked} tests could not allocate "
+ "their buffers,\n"
+ f"and {checked - inconclusive} completed intact. No peer copy was "
+ "shown to lose data,\n"
+ "but the host was not fully tested. Free the GPUs (stop any resident\n"
+ "model) and re-run, or lower --sizes-mib."
+ )
+ return 2
+
+ print(
+ f"PASS: all {checked} peer-copy tests transferred intact.\n"
+ "P2P is safe on this host. Unsloth Studio still gates GGML_CUDA_P2P on\n"
+ "verified NVLink, so to enable it here anyway:\n"
+ "\n"
+ " data-center cards (A100, H100, L40S, RTX 6000 Ada, ...):\n"
+ " UNSLOTH_FORCE_DC_P2P=1\n"
+ " anything else, including GeForce:\n"
+ " GGML_CUDA_P2P=1\n"
+ "\n"
+ "UNSLOTH_FORCE_DC_P2P only reaches the fabric check, which sits behind the\n"
+ "data-center gate, so it does nothing on a consumer card. Setting\n"
+ "GGML_CUDA_P2P yourself is passed through on every card (any value enables\n"
+ "it upstream, so unset it to turn it back off; 0 will NOT)."
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/studio/NVLINK_P2P.md b/studio/NVLINK_P2P.md
new file mode 100644
index 00000000000..4043b0607ec
--- /dev/null
+++ b/studio/NVLINK_P2P.md
@@ -0,0 +1,121 @@
+# GPU peer-to-peer (`GGML_CUDA_P2P`) and why it is gated on NVLink
+
+Recorded so the gate in `_apply_datacenter_env` is not "fixed" back into a product-name
+allowlist by someone who reasonably assumes a data-center GPU implies a working peer
+fabric. It does not. Background: issue #10613.
+
+## Summary
+
+| | verdict |
+|---|---|
+| NVLink-connected multi-GPU (`NV#` in the topology matrix) | **P2P enabled** — the configuration PR #6098 benchmarked, +33-51% tensor-split |
+| PCIe multi-GPU (`NODE` / `PHB` / `PXB` / `PIX` / `SYS`) | **P2P not enabled** — copies can be silently discarded |
+| topology unreadable | **P2P not enabled** — unknown means no |
+| single GPU | not applicable, no peer traffic |
+
+`GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F` is unaffected by all of this and still applies to
+every data-center part. It is a property of the silicon, not of the interconnect.
+
+## The failure it prevents
+
+On bare-metal Linux with the IOMMU in translating mode, a PCIe peer-to-peer copy between
+two NVIDIA GPUs can be dropped by the chipset while CUDA reports `cudaSuccess`. The write
+faults (`DMAR: [DMA Write NO_PASID] ... [fault reason 0x71]` in `dmesg`), the IOMMU
+handles it as an Unsupported Request and discards it, and `cudaMemcpyPeerAsync` has no
+return code for "the platform ate your DMA".
+
+Inside llama.cpp with `GGML_CUDA_P2P` set, that means tensors that never arrive. The model
+emits `!!!!!`, `/////`, a repeated Cyrillic token in reply to English, or fluent word
+salad. Nothing is logged and no error is raised, so it looks exactly like a broken quant
+or a wrong chat template. The reporter of #10613 spent a day on it.
+
+It is also load-dependent: whether a given layer placement pushes enough traffic across
+the bus varies, so the same configuration can look healthy on a short prompt and collapse
+on a long one. A clean short test proves nothing here.
+
+NVIDIA documents the configuration as unsupported ([CUDA C++ Programming Guide, IOMMU on
+Linux](https://docs.nvidia.com/cuda/cuda-programming-guide/index.html#iommu-on-linux)):
+bare-metal PCIe peer-to-peer copies require the IOMMU disabled. Most distributions enable
+it by default, so "bare-metal Linux, no NVLink" is the common case, not a corner case.
+
+## Why a name allowlist could not work
+
+Four families on the data-center allowlist have no NVLink connector at all: RTX 6000 Ada,
+RTX PRO 6000, L40 / L40S and L4. They ship in 2-, 4- and 8-way workstations and servers.
+A product name cannot tell you whether the box has a bridge.
+
+## Why the driver is not asked either
+
+`torch.cuda.can_device_access_peer()` returns `True` and `nvidia-smi topo -p2p w` reports
+`OK` on hosts where every peer copy drops. The driver's answer is the thing that is wrong;
+asking it again is not verification. vLLM reached the same conclusion and performs a real
+data-integrity check (`can_actually_p2p`, from vllm#2728).
+
+Studio reads `nvidia-smi topo -m` instead and requires `NV#` between every selected pair.
+NVLink traffic does not traverse the PCIe root complex, so the IOMMU fault class above
+cannot apply to it. The probe fails closed: a missing `nvidia-smi`, a non-zero exit, a
+timeout, an unparsable matrix, or a device mask that cannot be mapped to PCI indices all
+mean no P2P.
+
+### Partially bridged boxes, and which pairs get checked
+
+Only the pairs actually selected are checked, so on a 4-way or 8-way box with NVLink
+bridges over pairs (0-1 and 2-3, say) and PCIe between the islands, running on a bridged
+pair keeps P2P while a selection spanning the islands does not.
+
+Both index spaces here come from nvidia-smi: the GPU selection is sourced from
+`nvidia-smi --query-gpu=index` and the matrix from `nvidia-smi topo -m`, one enumeration,
+so the selection indexes the matrix directly. Do not "translate" it into CUDA ordinals.
+CUDA enumerates in `FASTEST_FIRST` order by default, so under a permutation that remapping
+would turn a PCIe-crossing selection into an NVLinked-looking one and enable the flag this
+gate exists to withhold.
+
+When no explicit selection is given, every pair on the visible box must be `NV#`.
+
+The log line names the pair that vetoed it.
+
+## Checking a host
+
+```
+nvidia-smi topo -m
+```
+
+`NV#` between the GPU pair means NVLink and you are unaffected. `NODE`, `PHB`, `PXB`,
+`PIX` or `SYS` means the copy goes over PCIe.
+
+To test whether peer copies on this host actually move data:
+
+```
+python scripts/p2p_integrity_probe.py
+```
+
+It fills the destination with a sentinel first, so a copy that transfers nothing is
+distinguishable from one that legitimately writes zeros, and it sweeps every ordered pair
+at several sizes. Exit 0 means intact, 1 means data was lost, 2 means it could not run.
+
+## Environment variables
+
+| variable | effect |
+|---|---|
+| `UNSLOTH_DISABLE_DC_TUNING=1` | disables all data-center tuning, including FP32 accumulate |
+| `UNSLOTH_DISABLE_DC_P2P=1` | disables `GGML_CUDA_P2P` only, keeping FP32 accumulate and `CUDA_SCALE_LAUNCH_QUEUES`. Also removes an inherited `GGML_CUDA_P2P`, so it holds even if the variable is already set elsewhere in your environment |
+| `UNSLOTH_FORCE_DC_P2P=1` | enables P2P on an unverified fabric (use after the probe passes). It cannot override `UNSLOTH_DISABLE_DC_P2P=1`, an off-meaning `GGML_CUDA_P2P` in your environment, or the data-center gate itself |
+
+`CUDA_SCALE_LAUNCH_QUEUES` is deliberately not gated on the fabric: it sizes a CUDA command
+buffer, moves no data between devices, and the reporter of #10613 measured it clean in
+isolation on the affected host. The set of hosts that receive it is unchanged by this gate.
+
+### `GGML_CUDA_P2P=0` does not disable P2P
+
+llama.cpp tests the variable for presence, not value:
+
+```c
+// ggml/src/ggml-cuda/ggml-cuda.cu
+if (getenv("GGML_CUDA_P2P") != nullptr) { ...
+bool use_peer_access = getenv("GGML_CUDA_P2P") != nullptr;
+```
+
+So `GGML_CUDA_P2P=0` turns peer access **on**. The variable must be unset entirely.
+Studio deletes it from the llama-server environment when its inherited value reads as off
+(`0`, `false`, `no`, `off`, empty), on every backend, so the intuitive spelling of the
+opt-out does what the user meant. Set it to `1` and it is honoured as a deliberate request.
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 8b65d4a94d2..97fe84e0cf6 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -9034,25 +9034,31 @@ class LlamaCppBackend:
return True
return LlamaCppBackend._unified_memory_would_help(gpu_indices, need_bytes = need_bytes)
- # Datacenter / professional NVIDIA parts that benefit from the llama.cpp
- # FP32-accum / P2P tunings. Whole-word (\b) so short markers don't match
- # workstation parts as substrings: "a100" must not fire on "RTX A1000".
+ # Datacenter / professional NVIDIA parts that get the FP32-accum tuning.
+ # Whole-word (\b) so "a100" does not fire on "RTX A1000".
_DATACENTER_GPU_RE = re.compile(
r"\b(?:a100|a30|h100|h200|h800|gh200|b200|b100|b300|gb200|gb300|"
r"l40s?|l4|rtx pro 6000|rtx 6000 ada)\b"
)
+ # The NVLink-capable subset. The rest (RTX 6000 Ada/PRO, L40/L40S, L4) have no
+ # NVLink connector, so peer copies cross PCIe, where a translating IOMMU on
+ # bare metal discards them silently while CUDA reports success (#10613).
+ # NECESSARY, never SUFFICIENT: _p2p_veto_reason also demands a positive NV#
+ # link, since a name cannot see an absent bridge.
+ _NVLINK_FABRIC_GPU_RE = re.compile(
+ r"\b(?:a100|a30|h100|h200|h800|gh200|b200|b100|b300|gb200|gb300)\b"
+ )
+
@staticmethod
- def _is_datacenter_gpu(gpu_indices = None) -> bool:
- """True iff every selected NVIDIA GPU is a datacenter/professional part.
- NVIDIA-only, fails open to False (consumer GeForce, ROCm, CPU and errors
- are left untouched); a mixed DC+consumer selection counts as non-DC.
-
- gpu_indices are PHYSICAL ids (see _get_gpu_free_memory), but
- get_device_properties wants mask-relative ordinals, so we rebuild the
- ordinal->physical map from CUDA_VISIBLE_DEVICES and key names by physical
- id. Otherwise a masked host (CUDA_VISIBLE_DEVICES=4,5,6,7, selection [4,5])
- would drop the tuning or probe the wrong GPU."""
+ def _all_selected_gpus_match(pattern, gpu_indices = None) -> bool:
+ """True iff every selected NVIDIA GPU's name matches ``pattern``. False for
+ consumer GeForce / ROCm / CPU / errors and for a mixed selection.
+
+ gpu_indices are PHYSICAL ids (see _get_gpu_free_memory) but
+ get_device_properties wants mask-relative ordinals, so names are keyed by
+ physical id via CUDA_VISIBLE_DEVICES; otherwise a masked host would drop
+ the tuning or probe the wrong GPU."""
try:
import torch
@@ -9066,7 +9072,6 @@ class LlamaCppBackend:
# CUDA_VISIBLE_DEVICES; unset/unparsable leaves physical id == ordinal.
physical_ids = LlamaCppBackend._resolve_visible_physical_ids()
- pattern = LlamaCppBackend._DATACENTER_GPU_RE
names_by_id: dict[int, str] = {}
for ordinal in range(count):
try:
@@ -9093,6 +9098,301 @@ class LlamaCppBackend:
except Exception:
return False
+ @staticmethod
+ def _is_datacenter_gpu(gpu_indices = None) -> bool:
+ """True iff every selected GPU qualifies for the FP32-accum tuning. Mask
+ handling: _all_selected_gpus_match. Peer-copy safety: _p2p_veto_reason."""
+ return LlamaCppBackend._all_selected_gpus_match(
+ LlamaCppBackend._DATACENTER_GPU_RE, gpu_indices
+ )
+
+ # `nvidia-smi topo -m` vocabulary: NV# is a bonded set of # NVLinks; SYS /
+ # NODE / PHB / PXB / PIX all traverse PCIe and are unsafe. The header is
+ # ANSI-underlined, so escapes are stripped before parsing.
+ _TOPO_GPU_LABEL_RE = re.compile(r"^GPU(\d+)$")
+ _TOPO_NVLINK_RE = re.compile(r"^NV\d+$")
+ _ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
+
+ # One shell-out per process, not per launch. None = not yet probed; otherwise a
+ # 1-tuple whose value may itself be None ("probed and unavailable").
+ _NVLINK_TOPO_CACHE = None
+
+ @staticmethod
+ def _probe_nvlink_topology() -> Optional[dict]:
+ """Parse ``nvidia-smi topo -m`` into {(i, j): "NV18" | "NODE" | ...}, keyed
+ by the *nvidia-smi* index (PCI order, NOT the CUDA ordinal), both orders
+ present. None when the topology cannot be established. Never raises.
+
+ The table also carries NIC rows/columns, affinity columns and a Legend, so
+ the GPU columns are the leading run of GPU<n> header labels and only GPU<n>
+ rows are read."""
+ try:
+ result = subprocess.run(
+ ["nvidia-smi", "topo", "-m"],
+ capture_output = True,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 10,
+ env = child_env_without_native_path_secret(),
+ **_windows_hidden_subprocess_kwargs(),
+ )
+ if result.returncode != 0:
+ return None
+ columns: list[int] = []
+ matrix: dict = {}
+ for raw in result.stdout.splitlines():
+ line = LlamaCppBackend._ANSI_ESCAPE_RE.sub("", raw)
+ if line.strip().lower().startswith("legend"):
+ break
+ tokens = line.split()
+ if not tokens:
+ continue
+ if not columns:
+ # Header: the leading run of GPU<n> labels names the columns.
+ for token in tokens:
+ m = LlamaCppBackend._TOPO_GPU_LABEL_RE.match(token)
+ if m is None:
+ break
+ columns.append(int(m.group(1)))
+ # Data rows lead with GPU<n> too, so one column is more likely
+ # a missing header than a real table: keep looking.
+ if len(columns) < 2:
+ columns = []
+ continue
+ row = LlamaCppBackend._TOPO_GPU_LABEL_RE.match(tokens[0])
+ if row is None:
+ continue # NIC row, or the blank line before the legend
+ source = int(row.group(1))
+ labels = tokens[1 : 1 + len(columns)]
+ if len(labels) != len(columns):
+ return None # truncated row: refuse to guess
+ for col, label in zip(columns, labels):
+ if col != source:
+ matrix[(source, col)] = label.upper()
+ # Every directed pair the header promised, or nothing. Row-truncated
+ # output still exits 0 and parses cleanly, and the rows that arrived can
+ # be uniformly NV#, which reads as "the whole box is NVLinked" and would
+ # enable P2P on cards whose links were never seen (#10613).
+ if len(matrix) != len(columns) * (len(columns) - 1):
+ logger.debug(
+ f"nvidia-smi topo -m: {len(matrix)} pairs for {len(columns)} "
+ "GPUs, expected "
+ f"{len(columns) * (len(columns) - 1)}; treating as unreadable"
+ )
+ return None
+ return matrix or None
+ except Exception as e:
+ logger.debug(f"nvidia-smi topo probe failed: {e}")
+ return None
+
+ @classmethod
+ def _nvlink_topology(cls, refresh = False) -> Optional[dict]:
+ """_probe_nvlink_topology, cached for the life of the process."""
+ if refresh or cls._NVLINK_TOPO_CACHE is None:
+ cls._NVLINK_TOPO_CACHE = (cls._probe_nvlink_topology(),)
+ return cls._NVLINK_TOPO_CACHE[0]
+
+ @staticmethod
+ def _running_virtualized() -> bool:
+ """True when this kernel runs under a hypervisor. CUDA supports peer copies
+ via VM pass-through but NOT on bare metal behind a translating IOMMU, so the
+ IOMMU veto applies only off this path. False when unknown (no flags line on
+ non-x86), which keeps the veto in play."""
+ try:
+ with open("/proc/cpuinfo", "r", encoding = "utf-8", errors = "replace") as f:
+ for line in f:
+ if line.startswith("flags") and ":" in line:
+ return "hypervisor" in line.split(":", 1)[1].split()
+ except Exception:
+ pass
+ return False
+
+ # One directory per IOMMU group, each with a `type`. Overridable for tests.
+ _IOMMU_GROUPS_ROOT = "/sys/kernel/iommu_groups"
+
+ # Whether _get_gpu_memory ids are nvidia-smi PCI indices (True) or torch CUDA
+ # ordinals (False, the fallback); None until a probe has run. They coincide
+ # only under PCI_BUS_ID, so a caller pinning the child's device order has to
+ # know which it holds (#10613).
+ _GPU_IDS_ARE_PCI_INDICES = None
+
+ # Boot-time property: read once, not per launch (the #10613 host has 175
+ # groups). Only the default root is cached; an explicit root (tests) re-reads.
+ _IOMMU_CACHE = None
+
+ @classmethod
+ def _iommu_is_translating_cached(cls) -> Optional[bool]:
+ if cls._IOMMU_CACHE is None:
+ try:
+ cls._IOMMU_CACHE = (cls._iommu_is_translating(),)
+ except Exception as e:
+ logger.debug(f"IOMMU probe failed: {e}")
+ cls._IOMMU_CACHE = (None,)
+ return cls._IOMMU_CACHE[0]
+
+ @staticmethod
+ def _iommu_is_translating(root = None) -> Optional[bool]:
+ """Whether the IOMMU translates DMA: True for DMA / DMA-FQ groups, False
+ when every group is identity/passthrough or there are none, None when the
+ groups exist but none is readable. Never raises.
+
+ CUDA C++ Programming Guide, "IOMMU on Linux": bare-metal PCIe peer-to-peer
+ is unsupported while the IOMMU translates, and the dropped write looks like
+ a successful copy rather than an error (#10613). Read per group, not per
+ GPU: a host with ANY translating group is not one to infer P2P safety on,
+ and resolving each GPU's PCI address needs a third index-space join."""
+ root = root or LlamaCppBackend._IOMMU_GROUPS_ROOT
+ try:
+ groups = os.listdir(root)
+ except Exception:
+ # Absent = nothing translates; present but unreadable = unknown.
+ return None if os.path.isdir(root) else False
+ if not groups:
+ return False # IOMMU compiled in but off
+ kinds = []
+ for group in groups:
+ try:
+ with open(
+ os.path.join(root, group, "type"),
+ "r",
+ encoding = "utf-8",
+ errors = "replace",
+ ) as f:
+ kinds.append(f.read().strip().upper())
+ except OSError:
+ continue # pre-5.x kernels have no `type`; other groups may
+ if not kinds:
+ return None
+ return any(not kind.startswith("IDENTITY") for kind in kinds)
+
+ @classmethod
+ def _p2p_veto_reason(
+ cls,
+ gpu_indices = None,
+ launch_order_pinned = False,
+ ) -> Optional[str]:
+ """Fail-closed wrapper around _p2p_veto_reason_inner: anything unexpected
+ reads as "do not set P2P", since losing the tuning beats an exception
+ escaping into load_model and failing the load."""
+ try:
+ return cls._p2p_veto_reason_inner(gpu_indices, launch_order_pinned)
+ except Exception as e:
+ logger.debug(f"peer-fabric check failed: {e}")
+ return f"the peer-fabric check could not complete ({type(e).__name__})"
+
+ @classmethod
+ def _p2p_veto_reason_inner(
+ cls,
+ gpu_indices = None,
+ launch_order_pinned = False,
+ ) -> Optional[str]:
+ """Why GGML_CUDA_P2P must NOT be set for this selection, or None once a
+ working NVLink fabric is confirmed for every selected pair. Fails CLOSED on
+ every unknown.
+
+ Before #10613 the flag came from the device NAME alone, which cannot see an
+ absent NVLink bridge: 2x RTX 6000 Ada matched the allowlist and every model
+ returned garbage. The driver is no better, since cudaDeviceCanAccessPeer
+ and `nvidia-smi topo -p2p w` both answered OK on that host, so the only
+ evidence accepted here is a positive NV# link.
+
+ Index spaces: gpu_indices are physical ids, the matrix is keyed by
+ nvidia-smi index, and the two coincide only under
+ CUDA_DEVICE_ORDER=PCI_BUS_ID (CUDA otherwise enumerates FASTEST_FIRST).
+ Rather than lose the tuning on every host that never set it, an inexact
+ mapping demands a uniformly NV# matrix, which holds under any permutation.
+
+ Overrides come first: UNSLOTH_DISABLE_DC_P2P=1 drops P2P while KEEPING
+ FP32-accum (all-or-nothing UNSLOTH_DISABLE_DC_TUNING would discard a tuning
+ that is not implicated), UNSLOTH_FORCE_DC_P2P=1 opts back in for a real but
+ unparsable fabric -- check it with scripts/p2p_integrity_probe.py."""
+ if os.environ.get("UNSLOTH_DISABLE_DC_P2P") == "1":
+ return "disabled by UNSLOTH_DISABLE_DC_P2P=1"
+ if os.environ.get("UNSLOTH_FORCE_DC_P2P") == "1":
+ return None
+
+ def _pcie(reason: str) -> str:
+ """Append the IOMMU diagnosis to a veto that implies a PCIe peer path.
+ On bare-metal Linux with a translating IOMMU such a copy is not merely
+ slow but unsupported, the write silently discarded. Hangs off EVERY such
+ veto, not just the topology one: the #10613 reporter's 2x RTX 6000 Ada
+ fails the NAME gate first and would otherwise never learn why its output
+ was garbled."""
+ if cls._iommu_is_translating_cached() and not cls._running_virtualized():
+ reason += (
+ "; bare-metal Linux with a translating IOMMU, where CUDA "
+ "does not support PCIe peer copies at all"
+ )
+ return reason
+
+ if not cls._all_selected_gpus_match(cls._NVLINK_FABRIC_GPU_RE, gpu_indices):
+ return _pcie("no NVLink-capable part in the selection, so peer copies would cross PCIe")
+
+ matrix = cls._nvlink_topology()
+ if not matrix:
+ return "nvidia-smi topo -m gave no usable interconnect matrix"
+
+ gpu_ids = sorted({i for pair in matrix for i in pair})
+
+ # gpu_indices are ALREADY nvidia-smi indices (_get_gpu_memory reads
+ # `--query-gpu=index`, the same enumeration as `topo -m`), so they index the
+ # matrix VERBATIM and remapping them as CUDA ordinals is wrong: under
+ # FASTEST_FIRST that would turn a PCIe-crossing selection into an
+ # NVLinked-looking one and enable the flag this gate exists to withhold. A
+ # usable matrix means nvidia-smi answered, so the selection came from its
+ # branch too.
+ if gpu_indices is not None:
+ # Verifying these ids means something only if the CHILD resolves the
+ # same cards. Without a PCI_BUS_ID pin it reads them in FASTEST_FIRST
+ # order, so [0,1] here can be [0,2] there: refuse rather than confirm
+ # NVLink for a pair that is not the one about to run (#10613).
+ # Unless the whole box is NVLinked, where the mapping cannot matter and
+ # vetoing would lose P2P on every auto-fit NVSwitch launch (#6098).
+ if not (
+ launch_order_pinned
+ or os.environ.get("CUDA_DEVICE_ORDER") == "PCI_BUS_ID"
+ or all(cls._TOPO_NVLINK_RE.match(v) for v in matrix.values())
+ ):
+ return _pcie(
+ "the child's device order is not pinned to PCI_BUS_ID and the "
+ "box is not uniformly NVLinked, so the verified GPUs may not be "
+ "the ones it runs on; set CUDA_DEVICE_ORDER=PCI_BUS_ID to "
+ "enable P2P here"
+ )
+ selected = sorted(set(gpu_indices))
+ else:
+ # No explicit selection: the child uses every VISIBLE GPU. nvidia-smi
+ # ignores CUDA_VISIBLE_DEVICES, so the matrix covers devices the child
+ # never touches, and filtering it to the mask keeps a clean NVLinked
+ # pair from being vetoed by a PCIe edge to a hidden device.
+ # Sound only when mask and matrix share an index space: numeric mask
+ # entries are CUDA ordinals, the matrix is PCI indices, and under
+ # FASTEST_FIRST mask "0,1" can mean physical 0,2, which could confirm
+ # NV# for a pair that is not the one in use (_cuda_compute_caps declines
+ # the same mapping). Unmappable means check the whole box.
+ visible = cls._resolve_visible_physical_ids()
+ mappable = os.environ.get("CUDA_DEVICE_ORDER") == "PCI_BUS_ID"
+ selected = (
+ [i for i in gpu_ids if i in set(visible)]
+ if visible is not None and mappable
+ else list(gpu_ids)
+ )
+
+ if len(selected) < 2:
+ return "fewer than two GPUs resolved in the interconnect matrix"
+
+ for a in selected:
+ for b in selected:
+ if a == b:
+ continue
+ label = matrix.get((a, b))
+ if label is None:
+ return f"GPU {a} and GPU {b} are absent from the interconnect matrix"
+ if not cls._TOPO_NVLINK_RE.match(label):
+ return _pcie(f"GPU {a} to GPU {b} is {label}, not NVLink")
+ return None
+
@staticmethod
def _effective_gpu_count(gpu_indices = None) -> int:
"""GPUs llama-server will use: len(selection), else the visible CUDA
@@ -9108,23 +9408,136 @@ class LlamaCppBackend:
return 0
return 0
+ # Values a user writes to mean "off". ggml tests GGML_CUDA_P2P for PRESENCE, not
+ # value, so passing any of these through ENABLES what they turned off; only
+ # absence is off. Same trap as GGML_CUDA_ENABLE_UNIFIED_MEMORY (#8651).
+ _FALSY_ENV_VALUES = frozenset({"", "0", "false", "off", "no"})
+
+ # Property of the host, not the model, so warn once rather than per launch.
+ _warned_no_nvlink = False
+
@staticmethod
- def _apply_datacenter_env(env: dict, gpu_indices = None) -> bool:
- """Inject DC llama.cpp tuning into env in place via setdefault (user
- values win); return whether the box qualified. Opt out with
- UNSLOTH_DISABLE_DC_TUNING=1; only datacenter NVIDIA parts qualify
- (consumer/ROCm/CPU/error are a no-op). Sets GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F
- for any qualifying GPU (FP32 accum: ~0% cost on B200, real cost on GeForce),
- plus GGML_CUDA_P2P + CUDA_SCALE_LAUNCH_QUEUES=4x for multi-GPU (+33-51% pp
- tensor-split, +8-16% pipeline split on B200)."""
+ def _sanitize_p2p_env(env: dict) -> Optional[str]:
+ """Drop an inherited GGML_CUDA_P2P that must not reach the child; return the
+ removed value for logging, else None. ggml tests the variable for PRESENCE
+ and not value, so both cases mean OFF and neither is honoured upstream: a
+ falsy value ("0", "false", "off", ""), or UNSLOTH_DISABLE_DC_P2P=1, where
+ suppressing our own default is not enough because an inherited truthy value
+ would sail past it (#10613).
+
+ Belongs at the call site on EVERY launch path: env is inherited from
+ os.environ, and a consumer-GPU user who set the flag off by hand never
+ reaches the datacenter gate at all."""
+ value = env.get("GGML_CUDA_P2P")
+ if value is None:
+ return None
+ if (
+ str(value).strip().lower() in LlamaCppBackend._FALSY_ENV_VALUES
+ or os.environ.get("UNSLOTH_DISABLE_DC_P2P") == "1"
+ ):
+ del env["GGML_CUDA_P2P"]
+ logger.info(
+ "Dropped inherited GGML_CUDA_P2P=%r from the llama-server "
+ "environment: llama.cpp tests this variable for presence, not "
+ "value, so passing it through would have ENABLED peer copies "
+ "(#10613)",
+ value,
+ )
+ return value
+ return None
+
+ @staticmethod
+ def _p2p_user_opted_out() -> bool:
+ """Whether the USER asked for peer copies off. Read from os.environ, not a
+ child env _sanitize_p2p_env may already have cleaned, so the intent survives
+ the variable and the default cannot put it back."""
+ return (
+ os.environ.get("UNSLOTH_DISABLE_DC_P2P") == "1"
+ or str(os.environ.get("GGML_CUDA_P2P", "1")).strip().lower()
+ in LlamaCppBackend._FALSY_ENV_VALUES
+ )
+
+ @staticmethod
+ def _apply_datacenter_env(
+ env: dict,
+ gpu_indices = None,
+ p2p_opted_out = False,
+ launch_order_pinned = False,
+ ) -> bool:
+ """Inject DC llama.cpp tuning into env in place via setdefault (user values
+ win); return whether the box qualified. Only datacenter NVIDIA parts qualify
+ (consumer/ROCm/CPU/error are a no-op); opt out with
+ UNSLOTH_DISABLE_DC_TUNING=1. Any qualifying GPU gets
+ GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F (FP32 accum: ~0% cost on B200).
+
+ GGML_CUDA_P2P (+33-51% pp tensor-split on B200) additionally needs a
+ CONFIRMED NVLink fabric across the selection (_p2p_veto_reason): without one
+ the peer copy is silently discarded and every model emits garbage (#10613).
+ CUDA_SCALE_LAUNCH_QUEUES=4x (+8-16% pipeline split) is deliberately NOT
+ gated with it: it sizes a command buffer, moves no data across the bus, and
+ measured clean in isolation on the affected host.
+
+ p2p_opted_out records that the call site already removed a falsy
+ user-supplied GGML_CUDA_P2P (_sanitize_p2p_env), so the default must not put
+ it straight back."""
+ # Defence in depth, idempotent: called directly with a falsy value,
+ # setdefault would keep "0" and log "GGML_CUDA_P2P=0 (user)", which reads as
+ # off to a human and ON to ggml (#10613).
+ if LlamaCppBackend._sanitize_p2p_env(env) is not None:
+ p2p_opted_out = True
if os.environ.get("UNSLOTH_DISABLE_DC_TUNING") == "1":
return False
if not LlamaCppBackend._is_datacenter_gpu(gpu_indices):
return False
- env.setdefault("GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F", "1")
+
+ def _apply(name: str, value: str) -> str:
+ supplied = name in env
+ env.setdefault(name, value)
+ return f"{name}={env[name]}" + (" (user)" if supplied else "")
+
+ applied = [_apply("GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F", "1")]
if LlamaCppBackend._effective_gpu_count(gpu_indices) > 1:
- env.setdefault("GGML_CUDA_P2P", "1")
- env.setdefault("CUDA_SCALE_LAUNCH_QUEUES", "4x")
+ # Moves no data between GPUs, so it is not gated on the fabric.
+ applied.append(_apply("CUDA_SCALE_LAUNCH_QUEUES", "4x"))
+ # Asked for off: skip the probe rather than pay up to its 10s timeout to
+ # decide something the user already decided.
+ veto = (
+ "GGML_CUDA_P2P was turned off in the environment"
+ if p2p_opted_out
+ else LlamaCppBackend._p2p_veto_reason(gpu_indices, launch_order_pinned)
+ )
+ if veto is None:
+ applied.append(_apply("GGML_CUDA_P2P", "1"))
+ elif "GGML_CUDA_P2P" in env:
+ # Truthy and user-supplied: their value stands, but it steers
+ # straight into the #10613 failure.
+ logger.warning(
+ "GGML_CUDA_P2P is set in the environment (%s), so peer copies stay "
+ "ON despite: %s. Unset it entirely -- not =0, which reads as ON "
+ "upstream -- if model output is garbled (#10613).",
+ env["GGML_CUDA_P2P"],
+ veto,
+ )
+ elif p2p_opted_out or os.environ.get("UNSLOTH_DISABLE_DC_P2P") == "1":
+ # Asked for off and got off. The fabric was never inspected, so
+ # calling it unconfirmed would warn about corruption on a box that
+ # may be perfectly healthy.
+ logger.info("GGML_CUDA_P2P left unset: %s", veto)
+ elif not LlamaCppBackend._warned_no_nvlink:
+ LlamaCppBackend._warned_no_nvlink = True
+ logger.warning(
+ "Multi-GPU data-center box without a confirmed NVLink fabric: %s. "
+ "Leaving GGML_CUDA_P2P unset. A PCIe peer copy can be discarded "
+ "silently while CUDA still reports success, which surfaces as "
+ "garbled output rather than an error (#10613). Verify with "
+ "scripts/p2p_integrity_probe.py; if it passes, set "
+ "UNSLOTH_FORCE_DC_P2P=1 to opt back in.",
+ veto,
+ )
+ logger.info(
+ "Data-center GPU detected: applied DC llama.cpp env tuning (%s)",
+ ", ".join(applied),
+ )
return True
@staticmethod
@@ -10026,6 +10439,7 @@ class LlamaCppBackend:
# Match the docstring's sort-by-id guarantee (driver order isn't).
gpus.sort(key = lambda g: g[0])
if gpus:
+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = True
return gpus
except Exception as e:
logger.debug(f"nvidia-smi probe failed: {e}")
@@ -10045,6 +10459,8 @@ class LlamaCppBackend:
return []
if not hasattr(torch.cuda, "mem_get_info"):
return []
+ # CUDA ordinals from here on, not nvidia-smi PCI indices.
+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = False
# torch.cuda enumerates GPUs RELATIVE to the visibility mask. We
# feed these IDs back into the subprocess as CVD, so visible ordinals
# must be translated to physical indices first; otherwise CVD=2,3
@@ -12021,6 +12437,11 @@ class LlamaCppBackend:
WSL's librocdxg prepend is independent and is not gated by this flag.
"""
env = child_env_without_native_path_secret()
+ # Every llama-server child inherits os.environ, so an off-meaning
+ # GGML_CUDA_P2P is stripped HERE, not at one call site: chat, the STT
+ # sidecar and the embedding probe all build from this, and any path that
+ # skips it keeps the #10613 bug (presence, not value).
+ LlamaCppBackend._sanitize_p2p_env(env)
# _llama_lib_dir resolves the llama-server symlink to the real build/bin.
binary_dir = str(_llama_lib_dir(binary))
@@ -23783,12 +24204,77 @@ class LlamaCppBackend:
forced_cpu = _arch_gate_forced_cpu,
)
- # DC NVIDIA GPUs: FP32 accum (+ P2P / launch queues for multi-GPU).
- # See _apply_datacenter_env; opt out with UNSLOTH_DISABLE_DC_TUNING=1.
- if not is_vulkan_backend and self._apply_datacenter_env(env, gpu_indices):
- multi_gpu = self._effective_gpu_count(gpu_indices) > 1
- logger.info(
- f"Data-center GPU detected: applied DC llama.cpp env tuning (multi_gpu={multi_gpu})"
+ # An inherited GGML_CUDA_P2P=0 means OFF to the user and ON to
+ # llama.cpp, which tests presence and not value (#10613).
+ # _llama_server_env_for_binary already stripped it for every backend;
+ # this covers an env assembled another way.
+ self._sanitize_p2p_env(env)
+
+ # A multi-GPU box that is NOT a datacenter part never reaches the
+ # block below, so warn here or not at all: the 2x RTX 3090 case,
+ # whose own truthy GGML_CUDA_P2P rides through to the child.
+ # Computed here because a deliberate zero-offload load is masked to
+ # CUDA_VISIBLE_DEVICES=-1 and moves no tensors between GPUs, so
+ # probing or warning it is noise about traffic it never generates.
+ _cpu_only_zero_offload = (
+ gpu_memory_mode == "manual"
+ and gpu_layers == 0
+ and not is_vulkan_backend
+ and not self._zero_offload_keeps_gpu_visible(cmd, env)
+ )
+
+ # Whether the child will resolve gpu_indices as the nvidia-smi ids
+ # the P2P gate verifies. The launch below pins PCI_BUS_ID for an
+ # explicit pick, and for an auto-fit selection only when NO mask was
+ # inherited (nothing to reinterpret); anything else leaves CUDA on
+ # FASTEST_FIRST, where those ids may name other cards.
+ # That auto-fit arm also requires the ids to BE PCI indices: when the
+ # nvidia-smi query fails _get_gpu_memory falls back to torch CUDA
+ # ordinals, and pinning PCI order re-emits those numbers as a
+ # different set of cards. An explicit pick is unaffected, its ids
+ # come from the UI's own PCI-ordered picker.
+ _p2p_launch_order_pinned = bool(gpu_ids) or (
+ os.environ.get("CUDA_VISIBLE_DEVICES") is None
+ and LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is True
+ )
+
+ # Only when the fabric is NOT confirmed: on a verified NV# pair the
+ # flag is the benchmarked configuration. Datacenter boxes are
+ # excluded because _apply_datacenter_env warns about the same
+ # variable from its own veto branch.
+ if (
+ env.get("GGML_CUDA_P2P")
+ and not _cpu_only_zero_offload
+ and not is_vulkan_backend
+ and not self._is_datacenter_gpu(gpu_indices)
+ and self._effective_gpu_count(gpu_indices) > 1
+ and not LlamaCppBackend._warned_no_nvlink
+ ):
+ _p2p_veto = self._p2p_veto_reason(gpu_indices, _p2p_launch_order_pinned)
+ if _p2p_veto is not None:
+ LlamaCppBackend._warned_no_nvlink = True
+ logger.warning(
+ "GGML_CUDA_P2P=%s is set in the environment but this "
+ "selection has no confirmed NVLink fabric: %s. Peer copies "
+ "over PCIe can be discarded silently while CUDA still "
+ "reports success, which surfaces as garbled output rather "
+ "than an error. Verify with scripts/p2p_integrity_probe.py "
+ "and unset the variable entirely (not =0, which reads as ON "
+ "upstream) if it fails (#10613).",
+ env["GGML_CUDA_P2P"],
+ _p2p_veto,
+ )
+
+ # DC NVIDIA GPUs: FP32 accum and launch queues, plus P2P once a
+ # multi-GPU selection has a CONFIRMED NVLink fabric (#10613). Opt
+ # out with UNSLOTH_DISABLE_DC_TUNING=1, or UNSLOTH_DISABLE_DC_P2P=1
+ # for the peer flag alone.
+ if not (is_vulkan_backend or _cpu_only_zero_offload):
+ self._apply_datacenter_env(
+ env,
+ gpu_indices,
+ p2p_opted_out = self._p2p_user_opted_out(),
+ launch_order_pinned = _p2p_launch_order_pinned,
)
# Pin to selected GPU(s) (issue #7164; resolved above into gpu_indices).
@@ -23803,12 +24289,6 @@ class LlamaCppBackend:
# pin (extras or an inherited LLAMA_ARG_DEVICE) keeps control of its
# own devices, the child aborting on a pin it cannot see. The
# draft-device forms count too: parsed with no drafter loaded.
- _cpu_only_zero_offload = (
- gpu_memory_mode == "manual"
- and gpu_layers == 0
- and not is_vulkan_backend
- and not self._zero_offload_keeps_gpu_visible(cmd, env)
- )
_child_gpu_physical_ids: Optional[tuple[int, ...]] = None
if not is_vulkan_backend and _gpu_mem:
_child_gpu_physical_ids = self._unmasked_child_gpu_physical_ids()
@@ -23867,7 +24347,14 @@ class LlamaCppBackend:
# When the user picked GPUs by index, align CUDA's ordering with
# the PCI-bus order the picker enumerated, so "GPU 1" in the UI is
# GPU 1 to llama.cpp, not CUDA's FASTEST_FIRST default (#5025).
- if gpu_ids:
+ # Extended to an auto-fit selection ONLY when the inherited mask
+ # is ABSENT: a scheduler's CUDA_VISIBLE_DEVICES=0,1 meaning
+ # physical 2,0 would be re-read as physical 0,1 once the order
+ # changes, running on a device hidden on purpose. With no mask
+ # there is nothing to preserve, and the one emitted below is our
+ # own nvidia-smi indices, which pinning is what makes true. Where
+ # we cannot pin, _p2p_veto_reason withholds the flag.
+ if _p2p_launch_order_pinned:
env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
# Mask on AMD at the ROCr/HSA layer: HIP-only masking still
# enumerates every agent first, which segfaults on a deselected
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py
index 0900a7c61a1..73195e03d47 100644
--- a/studio/backend/core/rag/embed_llama_server.py
+++ b/studio/backend/core/rag/embed_llama_server.py
@@ -742,6 +742,12 @@ class LlamaServerBackend:
def _build_env(self, binary: str, *, use_gpu: bool) -> dict[str, str]:
env = child_env_without_native_path_secret()
+ # Not routed through _llama_server_env_for_binary, so it needs its own strip:
+ # an inherited GGML_CUDA_P2P=0 means OFF to the user and ON to llama.cpp, and
+ # a corrupt embedding degrades retrieval quietly (#10613).
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ LlamaCppBackend._sanitize_p2p_env(env)
env["LLAMA_SET_ROWS"] = "1"
if sys.platform == "darwin":
# _llama_lib_dir, not Path(binary).parent: the managed install puts an entrypoint in front of the
diff --git a/studio/backend/tests/test_datacenter_gpu_tuning.py b/studio/backend/tests/test_datacenter_gpu_tuning.py
index fd9b291e8a8..cd0e59d1d67 100644
--- a/studio/backend/tests/test_datacenter_gpu_tuning.py
+++ b/studio/backend/tests/test_datacenter_gpu_tuning.py
@@ -1,13 +1,18 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-"""Data-center llama.cpp env tuning: FP32 accum (+ P2P / launch queues for
-multi-GPU) must apply only to datacenter NVIDIA parts, never consumer GeForce,
-AMD/ROCm, CPU or macOS. User values win; UNSLOTH_DISABLE_DC_TUNING=1 disables.
+"""Data-center llama.cpp env tuning: datacenter NVIDIA parts only, never consumer
+GeForce, ROCm, CPU or macOS. User values win; UNSLOTH_DISABLE_DC_TUNING=1 disables.
+
+P2P has a second, stricter gate (#10613): a datacenter NAME is not evidence of an
+NVLink fabric, and on a non-NVLink box the peer copy is silently discarded while
+still reporting success, so every model emits garbage. It needs a confirmed NV#
+link across the selection and fails CLOSED on unknowns.
"""
from __future__ import annotations
+import subprocess
import sys
import types
@@ -22,7 +27,7 @@ def _fake_torch(
hip = None,
cuda_ok = True,
):
- """torch stub: version.hip, cuda.is_available/device_count, get_device_properties(i).name."""
+ """torch stub: version.hip, cuda.*, get_device_properties(i).name."""
t = types.ModuleType("torch")
t.version = types.SimpleNamespace(hip = hip)
t.cuda = types.SimpleNamespace(
@@ -33,11 +38,94 @@ def _fake_torch(
return t
+# Real `nvidia-smi topo -m` from an 8x B200 NVLink host: ANSI-underlined header, NIC
+# rows/columns, affinity columns and a Legend, none of which a toy string exercises.
+TOPO_NVLINK_8X = (
+ "\t\x1b[4mGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tNIC0\tNIC1\t"
+ "CPU Affinity\tNUMA Affinity\tGPU NUMA ID\x1b[0m\n"
+ "GPU0\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"
+ "GPU1\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"
+ "GPU2\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"
+ "GPU3\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"
+ "GPU4\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tPIX\tPIX\t48-95,144-191\t1\t\tN/A\n"
+ "GPU5\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tPIX\tPIX\t48-95,144-191\t1\t\tN/A\n"
+ "GPU6\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNODE\tNODE\t48-95,144-191\t1\t\tN/A\n"
+ "GPU7\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNODE\tNODE\t48-95,144-191\t1\t\tN/A\n"
+ "NIC0\tSYS\tSYS\tSYS\tSYS\tPIX\tPIX\tNODE\tNODE\t X \tPIX\t\t\t\n"
+ "NIC1\tSYS\tSYS\tSYS\tSYS\tPIX\tPIX\tNODE\tNODE\tPIX\t X \t\t\t\n"
+ "\n"
+ "Legend:\n"
+ "\n"
+ " X = Self\n"
+ " SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes\n"
+ " NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges\n"
+ " NV# = Connection traversing a bonded set of # NVLinks\n"
+)
+
+# The reporter's host (#10613): 2x RTX 6000 Ada, NODE (PCIe via a host bridge), no
+# NVLink. The driver still advertises P2P as available.
+TOPO_PCIE_2X = (
+ "\t\x1b[4mGPU0\tGPU1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\x1b[0m\n"
+ "GPU0\t X \tNODE\t0-23\t0\t\tN/A\n"
+ "GPU1\tNODE\t X \t0-23\t0\t\tN/A\n"
+ "\n"
+ "Legend:\n"
+ "\n"
+ " X = Self\n"
+ " NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges\n"
+)
+
+
@pytest.fixture(autouse = True)
def _clear_cuda_visible_devices(monkeypatch):
- """Detection reads CUDA_VISIBLE_DEVICES, so clear it by default (run unmasked,
- physical id == ordinal) regardless of host; masked tests set it explicitly."""
+ """Run unmasked by default (physical id == ordinal); masked tests set it."""
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ monkeypatch.delenv("CUDA_DEVICE_ORDER", raising = False)
+
+
+# Captured before the autouse fixture stubs it, so the probe can run for real.
+_REAL_IOMMU_IS_TRANSLATING = LlamaCppBackend.__dict__["_iommu_is_translating"].__func__
+
+
+def _no_nvidia_smi(*a, **k):
+ raise FileNotFoundError("nvidia-smi")
+
+
[email protected](autouse = True)
+def _isolate_host_topology(monkeypatch):
+ """Keep the P2P gate off the real host: only nvidia-smi's output is stubbed (the
+ parser always runs), caches are dropped either side, and the platform probes are
+ pinned so a CI box with GPUs cannot colour the results."""
+ LlamaCppBackend._NVLINK_TOPO_CACHE = None
+ LlamaCppBackend._IOMMU_CACHE = None
+ monkeypatch.setattr(subprocess, "run", _no_nvidia_smi)
+ monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(lambda *a: False))
+ monkeypatch.setattr(LlamaCppBackend, "_running_virtualized", staticmethod(lambda: False))
+ # Overrides off and the warn-once latch reset: no host env, no ordering leak.
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_P2P", raising = False)
+ monkeypatch.delenv("UNSLOTH_FORCE_DC_P2P", raising = False)
+ # Most tests are about the topology verdict, not device ordering, so default to
+ # a pinned order; the unpinned-path tests delete this themselves.
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ LlamaCppBackend._warned_no_nvlink = False
+ yield
+ LlamaCppBackend._NVLINK_TOPO_CACHE = None
+ LlamaCppBackend._IOMMU_CACHE = None
+ LlamaCppBackend._warned_no_nvlink = False
+
+
+def _use_topo(
+ monkeypatch,
+ text,
+ returncode = 0,
+):
+ """Feed the real parser canned `nvidia-smi topo -m` output."""
+ monkeypatch.setattr(
+ subprocess,
+ "run",
+ lambda *a, **k: types.SimpleNamespace(returncode = returncode, stdout = text, stderr = ""),
+ )
+ LlamaCppBackend._NVLINK_TOPO_CACHE = None
# ---------------------------------------------------------------------------
@@ -67,15 +155,14 @@ def _clear_cuda_visible_devices(monkeypatch):
(["NVIDIA GeForce RTX 3090"], False),
(["NVIDIA GeForce RTX 2080 Ti"], False),
(["NVIDIA GeForce GTX 1080"], False),
- # Workstation/laptop: short markers must not match as substrings
- # ("a100" in "A1000", "a30" in "A3000").
+ # Short markers must not match as substrings ("a100" in "A1000").
(["NVIDIA RTX A1000 Laptop GPU"], False),
(["NVIDIA RTX A1000 6GB Laptop GPU"], False),
(["NVIDIA RTX A3000 Laptop GPU"], False),
# Homogeneous multi-DC: all must match.
(["NVIDIA B200", "NVIDIA B200"], True),
(["NVIDIA H100 80GB HBM3", "NVIDIA H100 80GB HBM3"], True),
- # Mixed DC + consumer: non-DC, so tuning never lands on the GeForce.
+ # Mixed: non-DC, so tuning never lands on the GeForce.
(["NVIDIA B200", "NVIDIA GeForce RTX 4090"], False),
(["NVIDIA GeForce RTX 4090", "NVIDIA B200"], False),
],
@@ -86,7 +173,6 @@ def test_is_datacenter_gpu(monkeypatch, names, expected):
def test_is_datacenter_gpu_respects_selection(monkeypatch):
- # A mixed box where only the DC GPU is selected -> True; only consumer -> False.
monkeypatch.setitem(
sys.modules,
"torch",
@@ -99,15 +185,13 @@ def test_is_datacenter_gpu_respects_selection(monkeypatch):
def test_is_datacenter_gpu_out_of_range_indices_skipped(monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"]))
- # Out-of-range / negative indices are skipped; the one valid DC GPU still wins.
+ # Invalid indices are skipped; all invalid -> nothing seen -> False.
assert LlamaCppBackend._is_datacenter_gpu([0, 5, -1]) is True
- # Only invalid indices -> nothing seen -> False (fail closed for the flag).
assert LlamaCppBackend._is_datacenter_gpu([5, 9]) is False
def test_is_datacenter_gpu_masked_host_physical_ids(monkeypatch):
- # Mask 4,5,6,7 -> ordinals 0..3 == physical 4..7. PHYSICAL selection [4,5]
- # must resolve, not index out of range (the pre-fix bug: 4 >= device_count).
+ # PHYSICAL selection [4,5] must resolve (the pre-fix bug: 4 >= device_count).
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "4,5,6,7")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
assert LlamaCppBackend._is_datacenter_gpu([4, 5]) is True
@@ -117,15 +201,14 @@ def test_is_datacenter_gpu_masked_host_physical_ids(monkeypatch):
def test_is_datacenter_gpu_masked_host_reordered(monkeypatch):
- # Reordered mask preserves order: ordinal 0 -> physical 7, 1 -> 4, ...
+ # Reordered mask preserves order: ordinal 0 -> physical 7.
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "7,4,5,6")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100 80GB HBM3"] * 4))
assert LlamaCppBackend._is_datacenter_gpu([7, 4]) is True
def test_is_datacenter_gpu_masked_host_mixed_class(monkeypatch):
- # Mask 4,5: physical 4 = GeForce, physical 5 = B200. Detection must follow the
- # selected physical GPU, not a same-numbered ordinal.
+ # Detection must follow the selected physical GPU, not a same-numbered ordinal.
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "4,5")
monkeypatch.setitem(
sys.modules,
@@ -138,15 +221,14 @@ def test_is_datacenter_gpu_masked_host_mixed_class(monkeypatch):
def test_is_datacenter_gpu_unparsable_mask_falls_back(monkeypatch):
- # Unparsable (UUID) mask falls back to physical id == ordinal (mirrors
- # _get_gpu_free_memory), so ordinal lookup still classifies the device.
+ # Unparsable (UUID) mask falls back to physical id == ordinal.
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "GPU-abcdef12")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"]))
assert LlamaCppBackend._is_datacenter_gpu([0]) is True
def test_is_datacenter_gpu_rocm_is_false(monkeypatch):
- # ROCm reuses torch.cuda.*; an MI300X must not qualify.
+ # ROCm reuses torch.cuda.*.
monkeypatch.setitem(
sys.modules,
"torch",
@@ -177,7 +259,6 @@ def test_effective_gpu_count_explicit_selection(monkeypatch):
def test_effective_gpu_count_none_uses_visible(monkeypatch):
- # None -> visible device count.
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
assert LlamaCppBackend._effective_gpu_count(None) == 4
@@ -210,6 +291,7 @@ def test_apply_env_single_dc_gpu_sets_only_fp32(monkeypatch):
def test_apply_env_multi_dc_gpu_sets_all(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
@@ -218,9 +300,9 @@ def test_apply_env_multi_dc_gpu_sets_all(monkeypatch):
def test_apply_env_none_indices_uses_visible_count(monkeypatch):
- # None on a 2x DC box -> multi-GPU flags applied.
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100", "NVIDIA H100"]))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, None) is True
assert env["GGML_CUDA_P2P"] == "1"
@@ -238,12 +320,13 @@ def test_apply_env_consumer_gpu_is_noop(monkeypatch):
def test_apply_env_user_value_wins(monkeypatch):
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
env = {
- "GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "0", # user explicitly disabled
+ "GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "0", # user disabled
"CUDA_SCALE_LAUNCH_QUEUES": "8x", # user override
}
assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
- # setdefault must not clobber user values; the unset one still defaults.
+ # setdefault must not clobber user values; the unset one defaults.
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "0"
assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "8x"
assert env["GGML_CUDA_P2P"] == "1"
@@ -266,13 +349,714 @@ def test_apply_env_fail_open_on_detection_error(monkeypatch):
def test_apply_env_masked_host_multi_dc(monkeypatch):
- # End-to-end masked host (mask 4,5,6,7, physical selection [4,5]): pre-fix
- # applied no tuning; now all three multi-GPU flags must be set.
+ # Masked host end-to-end: pre-fix this applied no tuning at all.
monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "4,5,6,7")
monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
env: dict = {}
assert LlamaCppBackend._apply_datacenter_env(env, [4, 5]) is True
assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
assert env["GGML_CUDA_P2P"] == "1"
assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"
+
+
+# ---------------------------------------------------------------------------
+# nvidia-smi topo -m parsing
+# ---------------------------------------------------------------------------
+
+
+def test_topo_parses_real_nvlink_table(monkeypatch):
+ # 8 GPUs -> 8*7 ordered pairs; NIC rows, affinity columns and legend ignored.
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ matrix = LlamaCppBackend._nvlink_topology()
+ assert len(matrix) == 56
+ assert set(matrix.values()) == {"NV18"}
+ assert (0, 0) not in matrix # self ("X") is not a pair
+ assert matrix[(6, 7)] == "NV18"
+
+
+def test_topo_parses_pcie_table(monkeypatch):
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ assert LlamaCppBackend._nvlink_topology() == {(0, 1): "NODE", (1, 0): "NODE"}
+
+
+def test_topo_is_cached_then_refreshable(monkeypatch):
+ calls = []
+
+ def _once(*a, **k):
+ calls.append(1)
+ return types.SimpleNamespace(returncode = 0, stdout = TOPO_PCIE_2X, stderr = "")
+
+ monkeypatch.setattr(subprocess, "run", _once)
+ LlamaCppBackend._NVLINK_TOPO_CACHE = None
+ LlamaCppBackend._nvlink_topology()
+ LlamaCppBackend._nvlink_topology()
+ assert len(calls) == 1 # one shell-out per process
+ LlamaCppBackend._nvlink_topology(refresh = True)
+ assert len(calls) == 2
+
+
+def test_topo_unavailable_paths_are_none(monkeypatch):
+ # Missing binary, non-zero exit and a GPU-less table mean "unknown", never NVLink.
+ assert LlamaCppBackend._nvlink_topology() is None
+ _use_topo(monkeypatch, TOPO_NVLINK_8X, returncode = 9)
+ assert LlamaCppBackend._nvlink_topology() is None
+ _use_topo(monkeypatch, "Legend:\n X = Self\n")
+ assert LlamaCppBackend._nvlink_topology() is None
+
+
+def test_topo_truncated_row_refuses_to_guess(monkeypatch):
+ _use_topo(
+ monkeypatch,
+ "\tGPU0\tGPU1\tCPU Affinity\n"
+ "GPU0\t X \tNV18\t0-23\n"
+ "GPU1\tNV18\n", # row cut short: fewer labels than columns
+ )
+ assert LlamaCppBackend._nvlink_topology() is None
+
+
+# ---------------------------------------------------------------------------
+# _p2p_veto_reason (the #10613 gate)
+# ---------------------------------------------------------------------------
+
+
+def test_p2p_vetoed_on_rtx_6000_ada(monkeypatch):
+ # A datacenter NAME with no NVLink.
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ reason = LlamaCppBackend._p2p_veto_reason([0, 1])
+ assert reason is not None and "NVLink-capable" in reason
+
+
+ "name",
+ ["NVIDIA L40S", "NVIDIA L40", "NVIDIA L4", "NVIDIA RTX PRO 6000 Blackwell Server Edition"],
+)
+def test_p2p_vetoed_on_other_connectorless_parts(monkeypatch, name):
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch([name] * 2))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X) # even a lying matrix must not rescue them
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
+
+
+def test_p2p_allowed_on_confirmed_nvlink(monkeypatch):
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
+
+
+def test_p2p_vetoed_when_topology_unknown(monkeypatch):
+ # NVLink-capable parts but no nvidia-smi: fail CLOSED (the old code trusted the
+ # name alone).
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ reason = LlamaCppBackend._p2p_veto_reason([0, 1])
+ assert reason is not None and "interconnect matrix" in reason
+
+
+def test_p2p_vetoed_on_pcie_pair_between_nvlink_parts(monkeypatch):
+ # A100s with no bridge fitted is real, and the name gate would allow it.
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 2))
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ reason = LlamaCppBackend._p2p_veto_reason([0, 1])
+ assert reason is not None and "NODE" in reason
+
+
+def test_p2p_veto_names_the_iommu_on_bare_metal(monkeypatch):
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 2))
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(lambda *a: True))
+ reason = LlamaCppBackend._p2p_veto_reason([0, 1])
+ assert "translating IOMMU" in reason
+ # Under a hypervisor CUDA supports pass-through P2P, so the IOMMU is not it.
+ monkeypatch.setattr(LlamaCppBackend, "_running_virtualized", staticmethod(lambda: True))
+ assert "IOMMU" not in LlamaCppBackend._p2p_veto_reason([0, 1])
+
+
+def test_p2p_exact_mapping_consults_only_the_selected_pair(monkeypatch):
+ # PCI_BUS_ID makes the index spaces the same, so a partially linked box can still
+ # use its NVLinked pair.
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100"] * 3))
+ _use_topo(
+ monkeypatch,
+ "\tGPU0\tGPU1\tGPU2\tCPU Affinity\n"
+ "GPU0\t X \tNV18\tNODE\t0-23\n"
+ "GPU1\tNV18\t X \tNODE\t0-23\n"
+ "GPU2\tNODE\tNODE\t X \t0-23\n",
+ )
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
+ assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None
+ assert LlamaCppBackend._p2p_veto_reason([0, 1, 2]) is not None
+
+
+def test_p2p_checks_the_selected_pair_on_a_partially_linked_box(monkeypatch):
+ # Selection and matrix are both nvidia-smi indices, so a linked pair is allowed
+ # and an unlinked one refused on the same box, whatever CUDA_DEVICE_ORDER says.
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100"] * 3))
+ _use_topo(
+ monkeypatch,
+ "\tGPU0\tGPU1\tGPU2\tCPU Affinity\n"
+ "GPU0\t X \tNV18\tNODE\t0-23\n"
+ "GPU1\tNV18\t X \tNODE\t0-23\n"
+ "GPU2\tNODE\tNODE\t X \t0-23\n",
+ )
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
+ assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None
+ # No selection: the whole visible box has to qualify.
+ assert LlamaCppBackend._p2p_veto_reason(None) is not None
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
+
+
+# ---------------------------------------------------------------------------
+# End-to-end: the reported bug, and the opt-out that did not work
+# ---------------------------------------------------------------------------
+
+
+def test_apply_env_rtx_6000_ada_gets_fp32_but_not_p2p(monkeypatch):
+ """#10613 exactly: 2x RTX 6000 Ada keeps FP32 accum, harmless per the reporter's
+ own isolation, and loses the P2P that garbled every model."""
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ env: dict = {}
+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
+ assert "GGML_CUDA_P2P" not in env
+ # Launch-queue depth moves no data across the bus and #10613 measured it clean on
+ # that host, so it is deliberately NOT gated with P2P.
+ assert env == {"GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "1", "CUDA_SCALE_LAUNCH_QUEUES": "4x"}
+
+
+def test_apply_env_l40s_multi_gpu_gets_fp32_but_not_p2p(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA L40S"] * 4))
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ env: dict = {}
+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1, 2, 3]) is True
+ assert "GGML_CUDA_P2P" not in env
+ assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
+
+
+def test_sanitize_falsy_user_p2p_is_removed():
+ """ggml tests GGML_CUDA_P2P for presence, so passing a user's "0" through ENABLES
+ peer copies; the opt-out has to unset it."""
+ for value in ("0", "false", "OFF", "no", "", " 0 "):
+ env = {"GGML_CUDA_P2P": value, "OTHER": "kept"}
+ assert LlamaCppBackend._sanitize_p2p_env(env) == value
+ assert env == {"OTHER": "kept"}, value
+
+
+def test_sanitize_leaves_a_truthy_user_p2p_alone():
+ env = {"GGML_CUDA_P2P": "1"}
+ assert LlamaCppBackend._sanitize_p2p_env(env) is None
+ assert env == {"GGML_CUDA_P2P": "1"}
+
+
+def test_opted_out_p2p_is_not_reintroduced_by_the_default(monkeypatch):
+ # The call site strips it; the DC block must not put it back. The rest stands.
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ env: dict = {}
+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1], p2p_opted_out = True) is True
+ assert "GGML_CUDA_P2P" not in env
+ assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"
+ assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
+
+
+def test_disable_dc_p2p_drops_peer_flag_but_keeps_fp32(monkeypatch):
+ # The surgical opt-out: UNSLOTH_DISABLE_DC_TUNING is all-or-nothing and discards
+ # a tuning that is not implicated.
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ env: dict = {}
+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
+ assert "GGML_CUDA_P2P" not in env
+ assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
+
+
+def test_force_dc_p2p_opts_back_in_over_an_unreadable_topology(monkeypatch):
+ # For a real but unparsable fabric, once p2p_integrity_probe.py passes.
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_P2P", raising = False)
+ monkeypatch.setenv("UNSLOTH_FORCE_DC_P2P", "1")
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ env: dict = {}
+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
+ assert env["GGML_CUDA_P2P"] == "1"
+
+
+def test_disable_dc_p2p_beats_force_dc_p2p(monkeypatch):
+ # Both set: the safe direction wins.
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")
+ monkeypatch.setenv("UNSLOTH_FORCE_DC_P2P", "1")
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ env: dict = {}
+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
+ assert "GGML_CUDA_P2P" not in env
+
+
+def test_sanitize_applies_to_a_consumer_box_that_never_reaches_the_dc_gate():
+ # A 2x RTX 3090 never matches the allowlist, so only the call-site strip helps.
+ env = {"GGML_CUDA_P2P": "0"}
+ assert LlamaCppBackend._sanitize_p2p_env(env) == "0"
+ assert env == {}
+
+
+def test_apply_env_truthy_user_p2p_still_wins(monkeypatch):
+ # An explicit opt-in survives the veto.
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ env = {"GGML_CUDA_P2P": "1"}
+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
+ assert env["GGML_CUDA_P2P"] == "1"
+
+
+def test_apply_env_single_nvlink_gpu_still_skips_p2p(monkeypatch):
+ # One GPU has no peer.
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ env: dict = {}
+ assert LlamaCppBackend._apply_datacenter_env(env, [0]) is True
+ assert env == {"GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "1"}
+
+
+def test_apply_env_multi_dc_without_nvidia_smi_withholds_p2p(monkeypatch):
+ # Unknown topology must cost the optimisation, not the correctness.
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ env: dict = {}
+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
+ assert "GGML_CUDA_P2P" not in env
+ assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
+
+
+# ---------------------------------------------------------------------------
+# Platform probes
+# ---------------------------------------------------------------------------
+
+
+def _iommu_tree(tmp_path, types_by_group):
+ root = tmp_path / "iommu_groups"
+ for group, kind in types_by_group.items():
+ (root / str(group)).mkdir(parents = True)
+ if kind is not None:
+ (root / str(group) / "type").write_text(f"{kind}\n")
+ return str(root)
+
+
+def test_iommu_identity_groups_are_not_translating(tmp_path):
+ root = _iommu_tree(tmp_path, {0: "identity", 1: "identity"})
+ assert _REAL_IOMMU_IS_TRANSLATING(root) is False
+
+
+def test_iommu_dma_group_is_translating(tmp_path):
+ # The reporter's host: 175 groups in DMA-FQ (translating) mode.
+ root = _iommu_tree(tmp_path, {0: "identity", 1: "DMA-FQ"})
+ assert _REAL_IOMMU_IS_TRANSLATING(root) is True
+
+
+def test_iommu_absent_or_empty_is_not_translating(tmp_path):
+ assert _REAL_IOMMU_IS_TRANSLATING(str(tmp_path / "nope")) is False
+ (tmp_path / "empty").mkdir()
+ assert _REAL_IOMMU_IS_TRANSLATING(str(tmp_path / "empty")) is False
+
+
+def test_iommu_unreadable_types_are_unknown(tmp_path):
+ # Pre-5.x kernels expose groups with no `type`: unknown, not "safe".
+ root = _iommu_tree(tmp_path, {0: None, 1: None})
+ assert _REAL_IOMMU_IS_TRANSLATING(root) is None
+
+
+# ---------------------------------------------------------------------------
+# Gaps found reviewing the #10613 fix
+# ---------------------------------------------------------------------------
+
+
+def test_disable_dc_p2p_also_drops_an_inherited_truthy_value(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ env = {"GGML_CUDA_P2P": "1"}
+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])
+ # Presence is truth upstream, so "disabled" has to mean absent.
+ assert "GGML_CUDA_P2P" not in env
+
+
+def test_apply_env_does_not_leave_a_falsy_value_present_when_called_directly(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ env = {"GGML_CUDA_P2P": "0"}
+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])
+ assert "GGML_CUDA_P2P" not in env
+
+
+def test_shared_llama_server_env_builder_sanitizes_p2p(monkeypatch, tmp_path):
+ """The STT sidecar and the embedding probe build from this, so stripping here
+ covers every llama-server child, not just chat."""
+ monkeypatch.setenv("GGML_CUDA_P2P", "0")
+ binary = tmp_path / "llama-server"
+ binary.write_text("", encoding = "utf-8")
+ env = LlamaCppBackend._llama_server_env_for_binary(str(binary))
+ assert "GGML_CUDA_P2P" not in env
+
+
+def test_stt_sidecar_env_sanitizes_p2p(monkeypatch, tmp_path):
+ # Spawns llama-server with -ngl 99, so it is exposed like chat.
+ from core.inference import stt_mtmd_sidecar
+
+ monkeypatch.setenv("GGML_CUDA_P2P", "0")
+ binary = tmp_path / "llama-server"
+ binary.write_text("", encoding = "utf-8")
+ assert "GGML_CUDA_P2P" not in stt_mtmd_sidecar._llama_server_child_env(str(binary))
+
+
+def test_embedding_server_env_sanitizes_p2p(monkeypatch, tmp_path):
+ """Bypasses the shared builder, and a corrupt embedding degrades retrieval with
+ no garbled text to notice."""
+ from core.rag.embed_llama_server import LlamaServerBackend
+
+ monkeypatch.setenv("GGML_CUDA_P2P", "0")
+ binary = tmp_path / "llama-server"
+ binary.write_text("", encoding = "utf-8")
+ server = LlamaServerBackend.__new__(LlamaServerBackend)
+ env = server._build_env(str(binary), use_gpu = False)
+ assert "GGML_CUDA_P2P" not in env
+
+
+# ---------------------------------------------------------------------------
+# Found by the platform/hardware simulation matrix (temp/sim_10613)
+# ---------------------------------------------------------------------------
+
+
+# 4x A100 bridged over (0,1) and (2,3), PCIe between the islands: the standard
+# bridged build, which the whole-matrix fallback used to veto outright.
+TOPO_BRIDGED_4X = (
+ "\t\x1b[4mGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\tNUMA Affinity\x1b[0m\n"
+ "GPU0\t X \tNV12\tSYS\tSYS\t0-23\t0\n"
+ "GPU1\tNV12\t X \tSYS\tSYS\t0-23\t0\n"
+ "GPU2\tSYS\tSYS\t X \tNV12\t24-47\t1\n"
+ "GPU3\tSYS\tSYS\tNV12\t X \t24-47\t1\n"
+ "\nLegend:\n\n X = Self\n"
+)
+
+
+def test_bridged_pair_keeps_p2p_on_a_partially_bridged_box(monkeypatch):
+ """A genuinely NVLinked pair on a partially bridged box keeps P2P: gpu_indices
+ are already nvidia-smi indices, so the pair checked is the pair selected."""
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))
+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
+ assert LlamaCppBackend._p2p_veto_reason([2, 3]) is None
+ # Across the islands the copy really would cross PCIe.
+ assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None
+ assert LlamaCppBackend._p2p_veto_reason([1, 3]) is not None
+
+
+def test_selection_is_not_remapped_out_of_the_nvidia_smi_index_space(monkeypatch):
+ """The selection indexes the topology matrix VERBATIM: both come from nvidia-smi,
+ one enumeration. Remapping them as CUDA ordinals would, under FASTEST_FIRST on a
+ bridged box, turn a PCIe-crossing selection into an NVLinked-looking one and
+ enable the flag this gate exists to withhold."""
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))
+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)
+ for pair in ([0, 1], [2, 3]):
+ assert LlamaCppBackend._p2p_veto_reason(pair) is None, pair
+ for pair in ([0, 2], [0, 3], [1, 2], [1, 3]):
+ assert LlamaCppBackend._p2p_veto_reason(pair) is not None, pair
+ # An unset CUDA_DEVICE_ORDER must not change any of the above.
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")
+ LlamaCppBackend._NVLINK_TOPO_CACHE = None
+ assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None
+
+
+def test_name_gate_veto_still_names_the_iommu(monkeypatch):
+ """The reporter's host fails the NAME gate first, so without this the one
+ actionable diagnosis never reaches them (#10613)."""
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
+ monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(lambda *a: True))
+ LlamaCppBackend._IOMMU_CACHE = None
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ reason = LlamaCppBackend._p2p_veto_reason([0, 1])
+ assert "translating IOMMU" in reason
+
+
+def test_a_raising_probe_never_fails_the_model_load(monkeypatch):
+ # Losing the tuning is acceptable; taking the model load down is not.
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+
+ def boom(*a, **k):
+ raise OSError("sysfs exploded")
+
+ monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(boom))
+ monkeypatch.setattr(LlamaCppBackend, "_nvlink_topology", classmethod(boom))
+ LlamaCppBackend._IOMMU_CACHE = None
+ env: dict = {}
+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True
+ assert "GGML_CUDA_P2P" not in env
+ assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"
+
+
+def test_iommu_scan_is_cached_across_loads(monkeypatch):
+ # A boot-time property, and 175 groups on the reporter's host.
+ calls = []
+
+ def counted(*a, **k):
+ calls.append(1)
+ return True
+
+ monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(counted))
+ LlamaCppBackend._IOMMU_CACHE = None
+ for _ in range(5):
+ LlamaCppBackend._iommu_is_translating_cached()
+ assert len(calls) == 1
+
+
+def _capture_warnings(monkeypatch):
+ """Collect logger.warning calls: caplog does not see structlog here, so asserting
+ on it would pass vacuously."""
+ from core.inference import llama_cpp as _mod
+
+ seen: list[str] = []
+ real = _mod.logger.warning
+ monkeypatch.setattr(
+ _mod.logger,
+ "warning",
+ lambda msg, *a, **k: (seen.append(str(msg)), real(msg, *a, **k))[0],
+ )
+ return seen
+
+
+def test_no_pcie_warning_when_the_user_opts_in_on_a_verified_fabric(monkeypatch):
+ """A deliberate GGML_CUDA_P2P=1 on a confirmed NV# pair is the benchmarked
+ configuration, so warning there would push users off a working optimisation."""
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
+ seen = _capture_warnings(monkeypatch)
+ env = {"GGML_CUDA_P2P": "1"}
+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])
+ assert env["GGML_CUDA_P2P"] == "1"
+ assert not [m for m in seen if "peer copies" in m], seen
+
+
+def test_pcie_warning_still_fires_on_an_unverified_fabric(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ seen = _capture_warnings(monkeypatch)
+ env = {"GGML_CUDA_P2P": "1"}
+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])
+ assert [m for m in seen if "peer copies stay" in m], seen
+
+
+def test_masked_visible_devices_filter_needs_a_shared_index_space(monkeypatch):
+ """nvidia-smi ignores CUDA_VISIBLE_DEVICES, so the matrix covers cards the child
+ never touches and filtering it to the mask recovers a clean NVLinked pair. Sound
+ only under PCI_BUS_ID: numeric mask entries are CUDA ordinals, and under
+ FASTEST_FIRST "0,1" can mean physical 0,2."""
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 2))
+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")
+ assert LlamaCppBackend._p2p_veto_reason(None) is None
+ # A mask spanning the islands is still correctly refused.
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,2")
+ LlamaCppBackend._NVLINK_TOPO_CACHE = None
+ assert LlamaCppBackend._p2p_veto_reason(None) is not None
+ # Without a shared index space the mask is not trusted: the whole box has to
+ # qualify, which the bridged fixture does not.
+ monkeypatch.delenv("CUDA_DEVICE_ORDER")
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")
+ LlamaCppBackend._NVLINK_TOPO_CACHE = None
+ assert LlamaCppBackend._p2p_veto_reason(None) is not None
+
+
+def test_explicit_p2p_opt_out_does_not_warn_about_corruption(monkeypatch):
+ """UNSLOTH_DISABLE_DC_P2P=1 returns before the topology is inspected, so calling
+ the fabric unconfirmed would warn about corruption on a healthy box."""
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ seen = _capture_warnings(monkeypatch)
+ env: dict = {}
+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])
+ assert "GGML_CUDA_P2P" not in env
+ assert not [m for m in seen if "without a confirmed NVLink" in m], seen
+
+
+def test_auto_fit_selection_without_a_pinned_order_refuses_p2p(monkeypatch):
+ """The gate verifies nvidia-smi physical ids, and the child resolves the same
+ cards only under a pinned CUDA_DEVICE_ORDER. On a box that is NOT uniformly
+ NVLinked, [0,1] here can be [0,2] there, so refuse rather than confirm NVLink for
+ a pair that is not the one about to run."""
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))
+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)
+ monkeypatch.delenv("CUDA_DEVICE_ORDER")
+ # Auto-fit (the launch will not pin the order): withheld.
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
+ # Explicit user pick, so the launch pins PCI_BUS_ID: allowed.
+ assert LlamaCppBackend._p2p_veto_reason([0, 1], launch_order_pinned = True) is None
+ # Already pinned in this process: allowed either way.
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ LlamaCppBackend._NVLINK_TOPO_CACHE = None
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None
+
+
+def test_auto_fit_launch_does_not_rewrite_an_inherited_device_order(monkeypatch):
+ """Forcing PCI_BUS_ID for an auto-fit launch would re-read an inherited numeric
+ mask: a scheduler's CUDA_VISIBLE_DEVICES=0,1 meaning physical 2,0 would become
+ physical 0,1, running on a GPU that was hidden on purpose."""
+ import inspect
+
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ branch = src[src.index("elif gpu_indices is not None and not is_vulkan_backend") :]
+ branch = branch[: branch.index("_launch_pinned_ids")]
+ pin = 'env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"'
+ assert pin in branch
+ # The pin must stay behind a guard, never unconditional.
+ assert "if _p2p_launch_order_pinned:" in branch[: branch.index(pin)]
+ # The guard must require an ABSENT inherited mask: an inherited numeric mask is
+ # the thing that must not be re-read.
+ start = src.index("_p2p_launch_order_pinned = ")
+ guard = src[start : src.index("\n\n", start)]
+ assert 'os.environ.get("CUDA_VISIBLE_DEVICES") is None' in guard, guard
+
+
+def test_datacenter_box_warns_once_not_twice(monkeypatch):
+ """The call-site warning is for hosts that never reach _apply_datacenter_env; a
+ datacenter box does, and its veto branch warns about the same variable."""
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ assert LlamaCppBackend._is_datacenter_gpu([0, 1]) is True
+ seen = _capture_warnings(monkeypatch)
+ env = {"GGML_CUDA_P2P": "1"}
+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])
+ assert len([m for m in seen if "peer copies stay" in m]) == 1, seen
+
+
+def test_uniform_nvlink_box_keeps_p2p_without_a_pinned_order(monkeypatch):
+ """The NVSwitch case PR #6098 benchmarked. When every pair on the box is NV# the
+ index mapping is irrelevant, since whichever cards the child resolves are linked,
+ and requiring a pinned order would strip P2P from every DGX/HGX auto-fit load."""
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ monkeypatch.delenv("CUDA_DEVICE_ORDER")
+ for sel in ([0, 1], [0, 7], list(range(8)), None):
+ assert LlamaCppBackend._p2p_veto_reason(sel) is None, sel
+
+
+def test_partially_bridged_box_still_needs_a_pinned_order(monkeypatch):
+ # Not uniform, so a permutation really can change which pair runs.
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))
+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)
+ monkeypatch.delenv("CUDA_DEVICE_ORDER")
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
+
+
+def test_unmasked_auto_fit_keeps_p2p(monkeypatch):
+ """An Auto load that fits on a subset sets gpu_indices with gpu_ids empty. With no
+ inherited CUDA_VISIBLE_DEVICES there is nothing to reinterpret, so the launch pins
+ PCI_BUS_ID and the flag is kept, sparing ordinary Auto loads on NVLink boxes."""
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))
+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)
+ monkeypatch.delenv("CUDA_DEVICE_ORDER")
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ # Not uniform, so this only passes because the launch can pin the order.
+ assert LlamaCppBackend._p2p_veto_reason([0, 1], launch_order_pinned = True) is None
+
+
+def test_p2p_opt_out_skips_the_topology_probe(monkeypatch):
+ """A user who turned P2P off should not pay the nvidia-smi probe, up to a 10s
+ timeout, to decide something they already decided."""
+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ probed = []
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_nvlink_topology",
+ classmethod(lambda cls, *a, **k: (probed.append(1), None)[1]),
+ )
+ env: dict = {}
+ LlamaCppBackend._apply_datacenter_env(env, [0, 1], p2p_opted_out = True)
+ assert "GGML_CUDA_P2P" not in env
+ assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"
+ assert probed == [], "topology probed despite an explicit opt-out"
+
+
+def test_row_truncated_matrix_is_rejected(monkeypatch):
+ """Row-truncated output still exits 0 and parses cleanly, and the rows that
+ arrived can be uniformly NV#, which would read as "the whole box is NVLinked" and
+ enable P2P on cards whose links were never seen. Partial must mean none."""
+ header, *rows = TOPO_NVLINK_8X.splitlines()
+ truncated = "\n".join([header] + rows[:4]) + "\n"
+ _use_topo(monkeypatch, truncated)
+ assert LlamaCppBackend._nvlink_topology() is None
+
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))
+ monkeypatch.delenv("CUDA_DEVICE_ORDER")
+ LlamaCppBackend._NVLINK_TOPO_CACHE = None
+ # The pairs that DID parse are all NV18, so without the completeness check the
+ # uniform escape would allow this.
+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None
+
+
+def test_complete_matrix_is_still_accepted(monkeypatch):
+ # The completeness check must not reject the real thing.
+ _use_topo(monkeypatch, TOPO_NVLINK_8X)
+ assert len(LlamaCppBackend._nvlink_topology()) == 8 * 7
+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)
+ assert len(LlamaCppBackend._nvlink_topology()) == 4 * 3
+ _use_topo(monkeypatch, TOPO_PCIE_2X)
+ assert len(LlamaCppBackend._nvlink_topology()) == 2 * 1
+
+
+def test_pin_requires_ids_to_be_pci_indices(monkeypatch):
+ """When the nvidia-smi query fails, _get_gpu_memory falls back to torch CUDA
+ ordinals, and pinning PCI order re-emits those numbers as a different set of
+ cards, so Auto could run on GPUs other than the ones it measured."""
+ import inspect
+
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ start = src.index("_p2p_launch_order_pinned = ")
+ guard = src[start : src.index("\n\n", start)]
+ assert "_GPU_IDS_ARE_PCI_INDICES is True" in guard, guard
+ assert 'os.environ.get("CUDA_VISIBLE_DEVICES") is None' in guard, guard
+
+
+def test_gpu_id_provenance_is_recorded(monkeypatch):
+ # The nvidia-smi branch yields PCI indices; the torch fallback yields ordinals.
+ monkeypatch.setattr(
+ subprocess,
+ "run",
+ lambda *a, **k: types.SimpleNamespace(
+ returncode = 0,
+ stdout = "0, 1000, 2000\n1, 1000, 2000\n",
+ stderr = "",
+ ),
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b: False))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "llama-server")
+ )
+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = None
+ assert LlamaCppBackend._get_gpu_memory("llama-server")
+ assert LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is True
+
+ # nvidia-smi absent -> torch fallback -> ordinals.
+ monkeypatch.setattr(subprocess, "run", _no_nvidia_smi)
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))
+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = None
+ LlamaCppBackend._get_gpu_memory("llama-server")
+ assert LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not True