Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: gate GGML_CUDA_P2P on a confirmed NVLink fabric, not a product name (#10646)
Daniel Han10a95da91652
diff --git a/scripts/p2p_integrity_probe.py b/scripts/p2p_integrity_probe.pynew file mode 100644index 00000000000..3dabf759aa3--- /dev/null+++ b/scripts/p2p_integrity_probe.py@@ -0,0 +1,261 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Probe: do GPU-to-GPU peer copies on this host actually transfer the data?++On bare-metal Linux with a translating IOMMU, a PCIe peer-to-peer copy between two+NVIDIA GPUs can be discarded by the chipset while CUDA reports ``cudaSuccess`` and+``torch.cuda.can_device_access_peer`` returns True, so nothing notices. In+llama.cpp with ``GGML_CUDA_P2P`` set, the model emits ``!!!!!``, a repeated token+or word salad, looking exactly like a broken quant or chat template (#10613).++The destination is sentinel-filled before each copy, so a transfer that moves+nothing is distinguishable from one that legitimately writes zeros, and every+ordered pair is tested at several sizes, since small copies can take a different+path than large ones. Run with no arguments on the host in question::++ python scripts/p2p_integrity_probe.py++Exit 0: every peer copy transferred intact. Exit 1: at least one dropped data, so+do NOT set ``GGML_CUDA_P2P`` here. Exit 2: could not run (no torch, <2 GPUs).++``GGML_CUDA_P2P=0`` does not disable the flag upstream: llama.cpp tests it for+presence, not value, so it must be unset entirely. Unsloth Studio unsets it for+you and gates the flag on verified NVLink.+"""++from __future__ import annotations++import argparse+import shutil+import subprocess+import sys++SENTINEL = -7.0+SIZES_MIB = (1, 4, 16, 64)+REPEATS = 3+++def _print_topology() -> None:+ """Show `nvidia-smi topo -m`, the cheap read that answers this in advance. NV#+ is NVLink, which never traverses the PCIe root complex; NODE / PHB / PXB / PIX+ / SYS all do."""+ if shutil.which("nvidia-smi") is None:+ print("nvidia-smi not found; skipping topology.\n")+ return+ try:+ out = subprocess.run(+ ["nvidia-smi", "topo", "-m"],+ capture_output = True,+ text = True,+ timeout = 30,+ )+ except Exception as e: # noqa: BLE001 -- diagnostic, never fatal+ print(f"topology read failed: {e}\n")+ return+ if out.returncode != 0:+ print("topology read returned non-zero; skipping.\n")+ return+ print("=== nvidia-smi topo -m ===")+ # The legend after the matrix is not what the reader needs.+ for line in out.stdout.splitlines():+ if line.strip().startswith("Legend"):+ break+ print(line)+ print()+++def main() -> int:+ parser = argparse.ArgumentParser(description = __doc__.splitlines()[0])+ parser.add_argument(+ "--sizes-mib",+ type = int,+ nargs = "+",+ default = list(SIZES_MIB),+ help = "copy sizes to test, in MiB",+ )+ parser.add_argument(+ "--repeats",+ type = int,+ default = REPEATS,+ help = "copies per pair per size",+ )+ args = parser.parse_args()++ # A probe that tests nothing must not print PASS: its whole job is to license+ # setting GGML_CUDA_P2P, so an empty run is the worst answer it could give.+ if args.repeats <= 0:+ parser.error("--repeats must be positive")+ if any(size <= 0 for size in args.sizes_mib):+ parser.error("--sizes-mib values must all be positive")++ _print_topology()++ try:+ import torch+ except ImportError:+ print("torch is not installed; cannot probe peer copies.")+ return 2++ # AMD SDK wheels leave version.hip unset and only encode "rocm" in+ # __version__, which is why the backend's _torch_is_rocm checks both.+ if (+ getattr(torch.version, "hip", None) is not None+ or "rocm" in getattr(torch, "__version__", "").lower()+ ):+ # ROCm reuses the torch.cuda namespace, so every test below would run and+ # PASS on AMD, then recommend GGML_CUDA_P2P, which only the CUDA backend+ # reads. A pass ending in a no-op instruction is worse than no answer.+ print(+ "This is a ROCm/HIP build of torch. GGML_CUDA_P2P is read only by\n"+ "llama.cpp's CUDA backend, so this probe has no advice to give for AMD\n"+ "GPUs and does not test them."+ )+ return 2++ if not (torch.cuda.is_available() and torch.cuda.device_count() >= 2):+ print(+ f"need at least 2 visible CUDA GPUs, found "+ f"{torch.cuda.device_count() if torch.cuda.is_available() else 0}. "+ "Peer copies are not used on a single GPU, so this host is unaffected."+ )+ return 2++ count = torch.cuda.device_count()+ print(f"=== {count} visible CUDA GPUs ===")+ for i in range(count):+ print(f" cuda:{i} {torch.cuda.get_device_name(i)}")+ print()++ failures = 0+ inconclusive = 0+ checked = 0+ print(f"{'pair':>10} {'size':>8} {'peer access':>11} verdict")+ for src in range(count):+ for dst in range(count):+ if src == dst:+ continue+ can = torch.cuda.can_device_access_peer(src, dst)+ for mib in args.sizes_mib:+ elements = mib * 1024 * 1024 // 4+ worst_dropped = 0+ worst_wrong = 0+ error = None+ setup_error = None+ for _ in range(args.repeats):+ s = d = None+ try:+ try:+ s = torch.arange(elements, dtype = torch.float32, device = f"cuda:{src}")+ d = torch.full(+ (elements,),+ SENTINEL,+ dtype = torch.float32,+ device = f"cuda:{dst}",+ )+ except Exception as alloc_exc: # noqa: BLE001+ # No transfer was attempted (usually a resident model+ # holds the memory), so this is inconclusive, NOT+ # evidence that peer copies drop data.+ setup_error = alloc_exc+ break+ d.copy_(s)+ torch.cuda.synchronize(src)+ torch.cuda.synchronize(dst)+ # Compare against the source, not just the sentinel: a+ # scrambled transfer overwrites it and would score as a+ # pass. The sentinel count stays because "never written"+ # is a different diagnosis, a dropped DMA.+ host_dst = d.cpu()+ wrong = int((host_dst != s.cpu()).sum())+ dropped = int((host_dst == SENTINEL).sum())+ worst_wrong = max(worst_wrong, wrong)+ worst_dropped = max(worst_dropped, dropped)+ except Exception as e: # noqa: BLE001 -- a raising copy is also a failure+ error = e+ break+ finally:+ del s, d+ torch.cuda.empty_cache()++ checked += 1+ pair = f"{src}->{dst}"+ access = "yes" if can else "no"+ # Corruption seen on an earlier repeat outranks a later allocation+ # failure: repeats exist to catch INTERMITTENT drops, so downgrading+ # a real mismatch to SKIPPED would hide what they are for.+ if setup_error is not None and not worst_wrong:+ inconclusive += 1+ print(+ f"{pair:>10} {str(mib) + ' MiB':>8} {access:>11} "+ f"SKIPPED (could not allocate): {setup_error}"+ )+ elif error is not None:+ failures += 1+ print(f"{pair:>10} {str(mib) + ' MiB':>8} {access:>11} ERROR: {error}")+ elif worst_wrong:+ failures += 1+ pct = 100.0 * worst_wrong / elements+ detail = (+ f"{worst_dropped} never written"+ if worst_dropped+ else "written with the wrong data"+ )+ print(+ f"{pair:>10} {str(mib) + ' MiB':>8} {access:>11} "+ f"CORRUPT: {worst_wrong}/{elements} elements wrong "+ f"({pct:.0f}%, {detail})"+ )+ else:+ print(f"{pair:>10} {str(mib) + ' MiB':>8} {access:>11} ok")++ print()+ if failures:+ print(+ f"FAIL: {failures} of {checked} peer-copy tests lost data.\n"+ "\n"+ "Peer-to-peer copies are NOT safe on this host. Do not set GGML_CUDA_P2P,\n"+ "and note that GGML_CUDA_P2P=0 does not disable it (llama.cpp tests the\n"+ "variable for presence, not value) -- it must be unset entirely.\n"+ "\n"+ "If CUDA reported success above while dropping data, check the kernel log\n"+ "for IOMMU faults: dmesg | grep -i -E 'DMAR|AMD-Vi'\n"+ "NVIDIA does not support bare-metal PCIe peer-to-peer with the IOMMU in\n"+ "translating mode; booting with intel_iommu=off / amd_iommu=off (or\n"+ "iommu=pt) is the usual remedy where that is acceptable."+ )+ return 1++ if inconclusive:+ # Exit 1 is for copies that were tested and lost data. Nothing was tested+ # here, so "unsafe" would push someone off a working optimisation.+ print(+ f"INCONCLUSIVE: {inconclusive} of {checked} tests could not allocate "+ "their buffers,\n"+ f"and {checked - inconclusive} completed intact. No peer copy was "+ "shown to lose data,\n"+ "but the host was not fully tested. Free the GPUs (stop any resident\n"+ "model) and re-run, or lower --sizes-mib."+ )+ return 2++ print(+ f"PASS: all {checked} peer-copy tests transferred intact.\n"+ "P2P is safe on this host. Unsloth Studio still gates GGML_CUDA_P2P on\n"+ "verified NVLink, so to enable it here anyway:\n"+ "\n"+ " data-center cards (A100, H100, L40S, RTX 6000 Ada, ...):\n"+ " UNSLOTH_FORCE_DC_P2P=1\n"+ " anything else, including GeForce:\n"+ " GGML_CUDA_P2P=1\n"+ "\n"+ "UNSLOTH_FORCE_DC_P2P only reaches the fabric check, which sits behind the\n"+ "data-center gate, so it does nothing on a consumer card. Setting\n"+ "GGML_CUDA_P2P yourself is passed through on every card (any value enables\n"+ "it upstream, so unset it to turn it back off; 0 will NOT)."+ )+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/studio/NVLINK_P2P.md b/studio/NVLINK_P2P.mdnew file mode 100644index 00000000000..4043b0607ec--- /dev/null+++ b/studio/NVLINK_P2P.md@@ -0,0 +1,121 @@+# GPU peer-to-peer (`GGML_CUDA_P2P`) and why it is gated on NVLink++Recorded so the gate in `_apply_datacenter_env` is not "fixed" back into a product-name+allowlist by someone who reasonably assumes a data-center GPU implies a working peer+fabric. It does not. Background: issue #10613.++## Summary++| | verdict |+|---|---|+| NVLink-connected multi-GPU (`NV#` in the topology matrix) | **P2P enabled** — the configuration PR #6098 benchmarked, +33-51% tensor-split |+| PCIe multi-GPU (`NODE` / `PHB` / `PXB` / `PIX` / `SYS`) | **P2P not enabled** — copies can be silently discarded |+| topology unreadable | **P2P not enabled** — unknown means no |+| single GPU | not applicable, no peer traffic |++`GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F` is unaffected by all of this and still applies to+every data-center part. It is a property of the silicon, not of the interconnect.++## The failure it prevents++On bare-metal Linux with the IOMMU in translating mode, a PCIe peer-to-peer copy between+two NVIDIA GPUs can be dropped by the chipset while CUDA reports `cudaSuccess`. The write+faults (`DMAR: [DMA Write NO_PASID] ... [fault reason 0x71]` in `dmesg`), the IOMMU+handles it as an Unsupported Request and discards it, and `cudaMemcpyPeerAsync` has no+return code for "the platform ate your DMA".++Inside llama.cpp with `GGML_CUDA_P2P` set, that means tensors that never arrive. The model+emits `!!!!!`, `/////`, a repeated Cyrillic token in reply to English, or fluent word+salad. Nothing is logged and no error is raised, so it looks exactly like a broken quant+or a wrong chat template. The reporter of #10613 spent a day on it.++It is also load-dependent: whether a given layer placement pushes enough traffic across+the bus varies, so the same configuration can look healthy on a short prompt and collapse+on a long one. A clean short test proves nothing here.++NVIDIA documents the configuration as unsupported ([CUDA C++ Programming Guide, IOMMU on+Linux](https://docs.nvidia.com/cuda/cuda-programming-guide/index.html#iommu-on-linux)):+bare-metal PCIe peer-to-peer copies require the IOMMU disabled. Most distributions enable+it by default, so "bare-metal Linux, no NVLink" is the common case, not a corner case.++## Why a name allowlist could not work++Four families on the data-center allowlist have no NVLink connector at all: RTX 6000 Ada,+RTX PRO 6000, L40 / L40S and L4. They ship in 2-, 4- and 8-way workstations and servers.+A product name cannot tell you whether the box has a bridge.++## Why the driver is not asked either++`torch.cuda.can_device_access_peer()` returns `True` and `nvidia-smi topo -p2p w` reports+`OK` on hosts where every peer copy drops. The driver's answer is the thing that is wrong;+asking it again is not verification. vLLM reached the same conclusion and performs a real+data-integrity check (`can_actually_p2p`, from vllm#2728).++Studio reads `nvidia-smi topo -m` instead and requires `NV#` between every selected pair.+NVLink traffic does not traverse the PCIe root complex, so the IOMMU fault class above+cannot apply to it. The probe fails closed: a missing `nvidia-smi`, a non-zero exit, a+timeout, an unparsable matrix, or a device mask that cannot be mapped to PCI indices all+mean no P2P.++### Partially bridged boxes, and which pairs get checked++Only the pairs actually selected are checked, so on a 4-way or 8-way box with NVLink+bridges over pairs (0-1 and 2-3, say) and PCIe between the islands, running on a bridged+pair keeps P2P while a selection spanning the islands does not.++Both index spaces here come from nvidia-smi: the GPU selection is sourced from+`nvidia-smi --query-gpu=index` and the matrix from `nvidia-smi topo -m`, one enumeration,+so the selection indexes the matrix directly. Do not "translate" it into CUDA ordinals.+CUDA enumerates in `FASTEST_FIRST` order by default, so under a permutation that remapping+would turn a PCIe-crossing selection into an NVLinked-looking one and enable the flag this+gate exists to withhold.++When no explicit selection is given, every pair on the visible box must be `NV#`.++The log line names the pair that vetoed it.++## Checking a host++```+nvidia-smi topo -m+```++`NV#` between the GPU pair means NVLink and you are unaffected. `NODE`, `PHB`, `PXB`,+`PIX` or `SYS` means the copy goes over PCIe.++To test whether peer copies on this host actually move data:++```+python scripts/p2p_integrity_probe.py+```++It fills the destination with a sentinel first, so a copy that transfers nothing is+distinguishable from one that legitimately writes zeros, and it sweeps every ordered pair+at several sizes. Exit 0 means intact, 1 means data was lost, 2 means it could not run.++## Environment variables++| variable | effect |+|---|---|+| `UNSLOTH_DISABLE_DC_TUNING=1` | disables all data-center tuning, including FP32 accumulate |+| `UNSLOTH_DISABLE_DC_P2P=1` | disables `GGML_CUDA_P2P` only, keeping FP32 accumulate and `CUDA_SCALE_LAUNCH_QUEUES`. Also removes an inherited `GGML_CUDA_P2P`, so it holds even if the variable is already set elsewhere in your environment |+| `UNSLOTH_FORCE_DC_P2P=1` | enables P2P on an unverified fabric (use after the probe passes). It cannot override `UNSLOTH_DISABLE_DC_P2P=1`, an off-meaning `GGML_CUDA_P2P` in your environment, or the data-center gate itself |++`CUDA_SCALE_LAUNCH_QUEUES` is deliberately not gated on the fabric: it sizes a CUDA command+buffer, moves no data between devices, and the reporter of #10613 measured it clean in+isolation on the affected host. The set of hosts that receive it is unchanged by this gate.++### `GGML_CUDA_P2P=0` does not disable P2P++llama.cpp tests the variable for presence, not value:++```c+// ggml/src/ggml-cuda/ggml-cuda.cu+if (getenv("GGML_CUDA_P2P") != nullptr) { ...+bool use_peer_access = getenv("GGML_CUDA_P2P") != nullptr;+```++So `GGML_CUDA_P2P=0` turns peer access **on**. The variable must be unset entirely.+Studio deletes it from the llama-server environment when its inherited value reads as off+(`0`, `false`, `no`, `off`, empty), on every backend, so the intuitive spelling of the+opt-out does what the user meant. Set it to `1` and it is honoured as a deliberate request.diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 8b65d4a94d2..97fe84e0cf6 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -9034,25 +9034,31 @@ class LlamaCppBackend:return Truereturn LlamaCppBackend._unified_memory_would_help(gpu_indices, need_bytes = need_bytes)- # Datacenter / professional NVIDIA parts that benefit from the llama.cpp- # FP32-accum / P2P tunings. Whole-word (\b) so short markers don't match- # workstation parts as substrings: "a100" must not fire on "RTX A1000".+ # Datacenter / professional NVIDIA parts that get the FP32-accum tuning.+ # Whole-word (\b) so "a100" does not fire on "RTX A1000"._DATACENTER_GPU_RE = re.compile(r"\b(?:a100|a30|h100|h200|h800|gh200|b200|b100|b300|gb200|gb300|"r"l40s?|l4|rtx pro 6000|rtx 6000 ada)\b")+ # The NVLink-capable subset. The rest (RTX 6000 Ada/PRO, L40/L40S, L4) have no+ # NVLink connector, so peer copies cross PCIe, where a translating IOMMU on+ # bare metal discards them silently while CUDA reports success (#10613).+ # NECESSARY, never SUFFICIENT: _p2p_veto_reason also demands a positive NV#+ # link, since a name cannot see an absent bridge.+ _NVLINK_FABRIC_GPU_RE = re.compile(+ r"\b(?:a100|a30|h100|h200|h800|gh200|b200|b100|b300|gb200|gb300)\b"+ )+@staticmethod- def _is_datacenter_gpu(gpu_indices = None) -> bool:- """True iff every selected NVIDIA GPU is a datacenter/professional part.- NVIDIA-only, fails open to False (consumer GeForce, ROCm, CPU and errors- are left untouched); a mixed DC+consumer selection counts as non-DC.-- gpu_indices are PHYSICAL ids (see _get_gpu_free_memory), but- get_device_properties wants mask-relative ordinals, so we rebuild the- ordinal->physical map from CUDA_VISIBLE_DEVICES and key names by physical- id. Otherwise a masked host (CUDA_VISIBLE_DEVICES=4,5,6,7, selection [4,5])- would drop the tuning or probe the wrong GPU."""+ def _all_selected_gpus_match(pattern, gpu_indices = None) -> bool:+ """True iff every selected NVIDIA GPU's name matches ``pattern``. False for+ consumer GeForce / ROCm / CPU / errors and for a mixed selection.++ gpu_indices are PHYSICAL ids (see _get_gpu_free_memory) but+ get_device_properties wants mask-relative ordinals, so names are keyed by+ physical id via CUDA_VISIBLE_DEVICES; otherwise a masked host would drop+ the tuning or probe the wrong GPU."""try:import torch@@ -9066,7 +9072,6 @@ class LlamaCppBackend:# CUDA_VISIBLE_DEVICES; unset/unparsable leaves physical id == ordinal.physical_ids = LlamaCppBackend._resolve_visible_physical_ids()- pattern = LlamaCppBackend._DATACENTER_GPU_REnames_by_id: dict[int, str] = {}for ordinal in range(count):try:@@ -9093,6 +9098,301 @@ class LlamaCppBackend:except Exception:return False+ @staticmethod+ def _is_datacenter_gpu(gpu_indices = None) -> bool:+ """True iff every selected GPU qualifies for the FP32-accum tuning. Mask+ handling: _all_selected_gpus_match. Peer-copy safety: _p2p_veto_reason."""+ return LlamaCppBackend._all_selected_gpus_match(+ LlamaCppBackend._DATACENTER_GPU_RE, gpu_indices+ )++ # `nvidia-smi topo -m` vocabulary: NV# is a bonded set of # NVLinks; SYS /+ # NODE / PHB / PXB / PIX all traverse PCIe and are unsafe. The header is+ # ANSI-underlined, so escapes are stripped before parsing.+ _TOPO_GPU_LABEL_RE = re.compile(r"^GPU(\d+)$")+ _TOPO_NVLINK_RE = re.compile(r"^NV\d+$")+ _ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")++ # One shell-out per process, not per launch. None = not yet probed; otherwise a+ # 1-tuple whose value may itself be None ("probed and unavailable").+ _NVLINK_TOPO_CACHE = None++ @staticmethod+ def _probe_nvlink_topology() -> Optional[dict]:+ """Parse ``nvidia-smi topo -m`` into {(i, j): "NV18" | "NODE" | ...}, keyed+ by the *nvidia-smi* index (PCI order, NOT the CUDA ordinal), both orders+ present. None when the topology cannot be established. Never raises.++ The table also carries NIC rows/columns, affinity columns and a Legend, so+ the GPU columns are the leading run of GPU<n> header labels and only GPU<n>+ rows are read."""+ try:+ result = subprocess.run(+ ["nvidia-smi", "topo", "-m"],+ capture_output = True,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ timeout = 10,+ env = child_env_without_native_path_secret(),+ **_windows_hidden_subprocess_kwargs(),+ )+ if result.returncode != 0:+ return None+ columns: list[int] = []+ matrix: dict = {}+ for raw in result.stdout.splitlines():+ line = LlamaCppBackend._ANSI_ESCAPE_RE.sub("", raw)+ if line.strip().lower().startswith("legend"):+ break+ tokens = line.split()+ if not tokens:+ continue+ if not columns:+ # Header: the leading run of GPU<n> labels names the columns.+ for token in tokens:+ m = LlamaCppBackend._TOPO_GPU_LABEL_RE.match(token)+ if m is None:+ break+ columns.append(int(m.group(1)))+ # Data rows lead with GPU<n> too, so one column is more likely+ # a missing header than a real table: keep looking.+ if len(columns) < 2:+ columns = []+ continue+ row = LlamaCppBackend._TOPO_GPU_LABEL_RE.match(tokens[0])+ if row is None:+ continue # NIC row, or the blank line before the legend+ source = int(row.group(1))+ labels = tokens[1 : 1 + len(columns)]+ if len(labels) != len(columns):+ return None # truncated row: refuse to guess+ for col, label in zip(columns, labels):+ if col != source:+ matrix[(source, col)] = label.upper()+ # Every directed pair the header promised, or nothing. Row-truncated+ # output still exits 0 and parses cleanly, and the rows that arrived can+ # be uniformly NV#, which reads as "the whole box is NVLinked" and would+ # enable P2P on cards whose links were never seen (#10613).+ if len(matrix) != len(columns) * (len(columns) - 1):+ logger.debug(+ f"nvidia-smi topo -m: {len(matrix)} pairs for {len(columns)} "+ "GPUs, expected "+ f"{len(columns) * (len(columns) - 1)}; treating as unreadable"+ )+ return None+ return matrix or None+ except Exception as e:+ logger.debug(f"nvidia-smi topo probe failed: {e}")+ return None++ @classmethod+ def _nvlink_topology(cls, refresh = False) -> Optional[dict]:+ """_probe_nvlink_topology, cached for the life of the process."""+ if refresh or cls._NVLINK_TOPO_CACHE is None:+ cls._NVLINK_TOPO_CACHE = (cls._probe_nvlink_topology(),)+ return cls._NVLINK_TOPO_CACHE[0]++ @staticmethod+ def _running_virtualized() -> bool:+ """True when this kernel runs under a hypervisor. CUDA supports peer copies+ via VM pass-through but NOT on bare metal behind a translating IOMMU, so the+ IOMMU veto applies only off this path. False when unknown (no flags line on+ non-x86), which keeps the veto in play."""+ try:+ with open("/proc/cpuinfo", "r", encoding = "utf-8", errors = "replace") as f:+ for line in f:+ if line.startswith("flags") and ":" in line:+ return "hypervisor" in line.split(":", 1)[1].split()+ except Exception:+ pass+ return False++ # One directory per IOMMU group, each with a `type`. Overridable for tests.+ _IOMMU_GROUPS_ROOT = "/sys/kernel/iommu_groups"++ # Whether _get_gpu_memory ids are nvidia-smi PCI indices (True) or torch CUDA+ # ordinals (False, the fallback); None until a probe has run. They coincide+ # only under PCI_BUS_ID, so a caller pinning the child's device order has to+ # know which it holds (#10613).+ _GPU_IDS_ARE_PCI_INDICES = None++ # Boot-time property: read once, not per launch (the #10613 host has 175+ # groups). Only the default root is cached; an explicit root (tests) re-reads.+ _IOMMU_CACHE = None++ @classmethod+ def _iommu_is_translating_cached(cls) -> Optional[bool]:+ if cls._IOMMU_CACHE is None:+ try:+ cls._IOMMU_CACHE = (cls._iommu_is_translating(),)+ except Exception as e:+ logger.debug(f"IOMMU probe failed: {e}")+ cls._IOMMU_CACHE = (None,)+ return cls._IOMMU_CACHE[0]++ @staticmethod+ def _iommu_is_translating(root = None) -> Optional[bool]:+ """Whether the IOMMU translates DMA: True for DMA / DMA-FQ groups, False+ when every group is identity/passthrough or there are none, None when the+ groups exist but none is readable. Never raises.++ CUDA C++ Programming Guide, "IOMMU on Linux": bare-metal PCIe peer-to-peer+ is unsupported while the IOMMU translates, and the dropped write looks like+ a successful copy rather than an error (#10613). Read per group, not per+ GPU: a host with ANY translating group is not one to infer P2P safety on,+ and resolving each GPU's PCI address needs a third index-space join."""+ root = root or LlamaCppBackend._IOMMU_GROUPS_ROOT+ try:+ groups = os.listdir(root)+ except Exception:+ # Absent = nothing translates; present but unreadable = unknown.+ return None if os.path.isdir(root) else False+ if not groups:+ return False # IOMMU compiled in but off+ kinds = []+ for group in groups:+ try:+ with open(+ os.path.join(root, group, "type"),+ "r",+ encoding = "utf-8",+ errors = "replace",+ ) as f:+ kinds.append(f.read().strip().upper())+ except OSError:+ continue # pre-5.x kernels have no `type`; other groups may+ if not kinds:+ return None+ return any(not kind.startswith("IDENTITY") for kind in kinds)++ @classmethod+ def _p2p_veto_reason(+ cls,+ gpu_indices = None,+ launch_order_pinned = False,+ ) -> Optional[str]:+ """Fail-closed wrapper around _p2p_veto_reason_inner: anything unexpected+ reads as "do not set P2P", since losing the tuning beats an exception+ escaping into load_model and failing the load."""+ try:+ return cls._p2p_veto_reason_inner(gpu_indices, launch_order_pinned)+ except Exception as e:+ logger.debug(f"peer-fabric check failed: {e}")+ return f"the peer-fabric check could not complete ({type(e).__name__})"++ @classmethod+ def _p2p_veto_reason_inner(+ cls,+ gpu_indices = None,+ launch_order_pinned = False,+ ) -> Optional[str]:+ """Why GGML_CUDA_P2P must NOT be set for this selection, or None once a+ working NVLink fabric is confirmed for every selected pair. Fails CLOSED on+ every unknown.++ Before #10613 the flag came from the device NAME alone, which cannot see an+ absent NVLink bridge: 2x RTX 6000 Ada matched the allowlist and every model+ returned garbage. The driver is no better, since cudaDeviceCanAccessPeer+ and `nvidia-smi topo -p2p w` both answered OK on that host, so the only+ evidence accepted here is a positive NV# link.++ Index spaces: gpu_indices are physical ids, the matrix is keyed by+ nvidia-smi index, and the two coincide only under+ CUDA_DEVICE_ORDER=PCI_BUS_ID (CUDA otherwise enumerates FASTEST_FIRST).+ Rather than lose the tuning on every host that never set it, an inexact+ mapping demands a uniformly NV# matrix, which holds under any permutation.++ Overrides come first: UNSLOTH_DISABLE_DC_P2P=1 drops P2P while KEEPING+ FP32-accum (all-or-nothing UNSLOTH_DISABLE_DC_TUNING would discard a tuning+ that is not implicated), UNSLOTH_FORCE_DC_P2P=1 opts back in for a real but+ unparsable fabric -- check it with scripts/p2p_integrity_probe.py."""+ if os.environ.get("UNSLOTH_DISABLE_DC_P2P") == "1":+ return "disabled by UNSLOTH_DISABLE_DC_P2P=1"+ if os.environ.get("UNSLOTH_FORCE_DC_P2P") == "1":+ return None++ def _pcie(reason: str) -> str:+ """Append the IOMMU diagnosis to a veto that implies a PCIe peer path.+ On bare-metal Linux with a translating IOMMU such a copy is not merely+ slow but unsupported, the write silently discarded. Hangs off EVERY such+ veto, not just the topology one: the #10613 reporter's 2x RTX 6000 Ada+ fails the NAME gate first and would otherwise never learn why its output+ was garbled."""+ if cls._iommu_is_translating_cached() and not cls._running_virtualized():+ reason += (+ "; bare-metal Linux with a translating IOMMU, where CUDA "+ "does not support PCIe peer copies at all"+ )+ return reason++ if not cls._all_selected_gpus_match(cls._NVLINK_FABRIC_GPU_RE, gpu_indices):+ return _pcie("no NVLink-capable part in the selection, so peer copies would cross PCIe")++ matrix = cls._nvlink_topology()+ if not matrix:+ return "nvidia-smi topo -m gave no usable interconnect matrix"++ gpu_ids = sorted({i for pair in matrix for i in pair})++ # gpu_indices are ALREADY nvidia-smi indices (_get_gpu_memory reads+ # `--query-gpu=index`, the same enumeration as `topo -m`), so they index the+ # matrix VERBATIM and remapping them as CUDA ordinals is wrong: under+ # FASTEST_FIRST that would turn a PCIe-crossing selection into an+ # NVLinked-looking one and enable the flag this gate exists to withhold. A+ # usable matrix means nvidia-smi answered, so the selection came from its+ # branch too.+ if gpu_indices is not None:+ # Verifying these ids means something only if the CHILD resolves the+ # same cards. Without a PCI_BUS_ID pin it reads them in FASTEST_FIRST+ # order, so [0,1] here can be [0,2] there: refuse rather than confirm+ # NVLink for a pair that is not the one about to run (#10613).+ # Unless the whole box is NVLinked, where the mapping cannot matter and+ # vetoing would lose P2P on every auto-fit NVSwitch launch (#6098).+ if not (+ launch_order_pinned+ or os.environ.get("CUDA_DEVICE_ORDER") == "PCI_BUS_ID"+ or all(cls._TOPO_NVLINK_RE.match(v) for v in matrix.values())+ ):+ return _pcie(+ "the child's device order is not pinned to PCI_BUS_ID and the "+ "box is not uniformly NVLinked, so the verified GPUs may not be "+ "the ones it runs on; set CUDA_DEVICE_ORDER=PCI_BUS_ID to "+ "enable P2P here"+ )+ selected = sorted(set(gpu_indices))+ else:+ # No explicit selection: the child uses every VISIBLE GPU. nvidia-smi+ # ignores CUDA_VISIBLE_DEVICES, so the matrix covers devices the child+ # never touches, and filtering it to the mask keeps a clean NVLinked+ # pair from being vetoed by a PCIe edge to a hidden device.+ # Sound only when mask and matrix share an index space: numeric mask+ # entries are CUDA ordinals, the matrix is PCI indices, and under+ # FASTEST_FIRST mask "0,1" can mean physical 0,2, which could confirm+ # NV# for a pair that is not the one in use (_cuda_compute_caps declines+ # the same mapping). Unmappable means check the whole box.+ visible = cls._resolve_visible_physical_ids()+ mappable = os.environ.get("CUDA_DEVICE_ORDER") == "PCI_BUS_ID"+ selected = (+ [i for i in gpu_ids if i in set(visible)]+ if visible is not None and mappable+ else list(gpu_ids)+ )++ if len(selected) < 2:+ return "fewer than two GPUs resolved in the interconnect matrix"++ for a in selected:+ for b in selected:+ if a == b:+ continue+ label = matrix.get((a, b))+ if label is None:+ return f"GPU {a} and GPU {b} are absent from the interconnect matrix"+ if not cls._TOPO_NVLINK_RE.match(label):+ return _pcie(f"GPU {a} to GPU {b} is {label}, not NVLink")+ return None+@staticmethoddef _effective_gpu_count(gpu_indices = None) -> int:"""GPUs llama-server will use: len(selection), else the visible CUDA@@ -9108,23 +9408,136 @@ class LlamaCppBackend:return 0return 0+ # Values a user writes to mean "off". ggml tests GGML_CUDA_P2P for PRESENCE, not+ # value, so passing any of these through ENABLES what they turned off; only+ # absence is off. Same trap as GGML_CUDA_ENABLE_UNIFIED_MEMORY (#8651).+ _FALSY_ENV_VALUES = frozenset({"", "0", "false", "off", "no"})++ # Property of the host, not the model, so warn once rather than per launch.+ _warned_no_nvlink = False+@staticmethod- def _apply_datacenter_env(env: dict, gpu_indices = None) -> bool:- """Inject DC llama.cpp tuning into env in place via setdefault (user- values win); return whether the box qualified. Opt out with- UNSLOTH_DISABLE_DC_TUNING=1; only datacenter NVIDIA parts qualify- (consumer/ROCm/CPU/error are a no-op). Sets GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F- for any qualifying GPU (FP32 accum: ~0% cost on B200, real cost on GeForce),- plus GGML_CUDA_P2P + CUDA_SCALE_LAUNCH_QUEUES=4x for multi-GPU (+33-51% pp- tensor-split, +8-16% pipeline split on B200)."""+ def _sanitize_p2p_env(env: dict) -> Optional[str]:+ """Drop an inherited GGML_CUDA_P2P that must not reach the child; return the+ removed value for logging, else None. ggml tests the variable for PRESENCE+ and not value, so both cases mean OFF and neither is honoured upstream: a+ falsy value ("0", "false", "off", ""), or UNSLOTH_DISABLE_DC_P2P=1, where+ suppressing our own default is not enough because an inherited truthy value+ would sail past it (#10613).++ Belongs at the call site on EVERY launch path: env is inherited from+ os.environ, and a consumer-GPU user who set the flag off by hand never+ reaches the datacenter gate at all."""+ value = env.get("GGML_CUDA_P2P")+ if value is None:+ return None+ if (+ str(value).strip().lower() in LlamaCppBackend._FALSY_ENV_VALUES+ or os.environ.get("UNSLOTH_DISABLE_DC_P2P") == "1"+ ):+ del env["GGML_CUDA_P2P"]+ logger.info(+ "Dropped inherited GGML_CUDA_P2P=%r from the llama-server "+ "environment: llama.cpp tests this variable for presence, not "+ "value, so passing it through would have ENABLED peer copies "+ "(#10613)",+ value,+ )+ return value+ return None++ @staticmethod+ def _p2p_user_opted_out() -> bool:+ """Whether the USER asked for peer copies off. Read from os.environ, not a+ child env _sanitize_p2p_env may already have cleaned, so the intent survives+ the variable and the default cannot put it back."""+ return (+ os.environ.get("UNSLOTH_DISABLE_DC_P2P") == "1"+ or str(os.environ.get("GGML_CUDA_P2P", "1")).strip().lower()+ in LlamaCppBackend._FALSY_ENV_VALUES+ )++ @staticmethod+ def _apply_datacenter_env(+ env: dict,+ gpu_indices = None,+ p2p_opted_out = False,+ launch_order_pinned = False,+ ) -> bool:+ """Inject DC llama.cpp tuning into env in place via setdefault (user values+ win); return whether the box qualified. Only datacenter NVIDIA parts qualify+ (consumer/ROCm/CPU/error are a no-op); opt out with+ UNSLOTH_DISABLE_DC_TUNING=1. Any qualifying GPU gets+ GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F (FP32 accum: ~0% cost on B200).++ GGML_CUDA_P2P (+33-51% pp tensor-split on B200) additionally needs a+ CONFIRMED NVLink fabric across the selection (_p2p_veto_reason): without one+ the peer copy is silently discarded and every model emits garbage (#10613).+ CUDA_SCALE_LAUNCH_QUEUES=4x (+8-16% pipeline split) is deliberately NOT+ gated with it: it sizes a command buffer, moves no data across the bus, and+ measured clean in isolation on the affected host.++ p2p_opted_out records that the call site already removed a falsy+ user-supplied GGML_CUDA_P2P (_sanitize_p2p_env), so the default must not put+ it straight back."""+ # Defence in depth, idempotent: called directly with a falsy value,+ # setdefault would keep "0" and log "GGML_CUDA_P2P=0 (user)", which reads as+ # off to a human and ON to ggml (#10613).+ if LlamaCppBackend._sanitize_p2p_env(env) is not None:+ p2p_opted_out = Trueif os.environ.get("UNSLOTH_DISABLE_DC_TUNING") == "1":return Falseif not LlamaCppBackend._is_datacenter_gpu(gpu_indices):return False- env.setdefault("GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F", "1")++ def _apply(name: str, value: str) -> str:+ supplied = name in env+ env.setdefault(name, value)+ return f"{name}={env[name]}" + (" (user)" if supplied else "")++ applied = [_apply("GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F", "1")]if LlamaCppBackend._effective_gpu_count(gpu_indices) > 1:- env.setdefault("GGML_CUDA_P2P", "1")- env.setdefault("CUDA_SCALE_LAUNCH_QUEUES", "4x")+ # Moves no data between GPUs, so it is not gated on the fabric.+ applied.append(_apply("CUDA_SCALE_LAUNCH_QUEUES", "4x"))+ # Asked for off: skip the probe rather than pay up to its 10s timeout to+ # decide something the user already decided.+ veto = (+ "GGML_CUDA_P2P was turned off in the environment"+ if p2p_opted_out+ else LlamaCppBackend._p2p_veto_reason(gpu_indices, launch_order_pinned)+ )+ if veto is None:+ applied.append(_apply("GGML_CUDA_P2P", "1"))+ elif "GGML_CUDA_P2P" in env:+ # Truthy and user-supplied: their value stands, but it steers+ # straight into the #10613 failure.+ logger.warning(+ "GGML_CUDA_P2P is set in the environment (%s), so peer copies stay "+ "ON despite: %s. Unset it entirely -- not =0, which reads as ON "+ "upstream -- if model output is garbled (#10613).",+ env["GGML_CUDA_P2P"],+ veto,+ )+ elif p2p_opted_out or os.environ.get("UNSLOTH_DISABLE_DC_P2P") == "1":+ # Asked for off and got off. The fabric was never inspected, so+ # calling it unconfirmed would warn about corruption on a box that+ # may be perfectly healthy.+ logger.info("GGML_CUDA_P2P left unset: %s", veto)+ elif not LlamaCppBackend._warned_no_nvlink:+ LlamaCppBackend._warned_no_nvlink = True+ logger.warning(+ "Multi-GPU data-center box without a confirmed NVLink fabric: %s. "+ "Leaving GGML_CUDA_P2P unset. A PCIe peer copy can be discarded "+ "silently while CUDA still reports success, which surfaces as "+ "garbled output rather than an error (#10613). Verify with "+ "scripts/p2p_integrity_probe.py; if it passes, set "+ "UNSLOTH_FORCE_DC_P2P=1 to opt back in.",+ veto,+ )+ logger.info(+ "Data-center GPU detected: applied DC llama.cpp env tuning (%s)",+ ", ".join(applied),+ )return True@staticmethod@@ -10026,6 +10439,7 @@ class LlamaCppBackend:# Match the docstring's sort-by-id guarantee (driver order isn't).gpus.sort(key = lambda g: g[0])if gpus:+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = Truereturn gpusexcept Exception as e:logger.debug(f"nvidia-smi probe failed: {e}")@@ -10045,6 +10459,8 @@ class LlamaCppBackend:return []if not hasattr(torch.cuda, "mem_get_info"):return []+ # CUDA ordinals from here on, not nvidia-smi PCI indices.+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = False# torch.cuda enumerates GPUs RELATIVE to the visibility mask. We# feed these IDs back into the subprocess as CVD, so visible ordinals# must be translated to physical indices first; otherwise CVD=2,3@@ -12021,6 +12437,11 @@ class LlamaCppBackend:WSL's librocdxg prepend is independent and is not gated by this flag."""env = child_env_without_native_path_secret()+ # Every llama-server child inherits os.environ, so an off-meaning+ # GGML_CUDA_P2P is stripped HERE, not at one call site: chat, the STT+ # sidecar and the embedding probe all build from this, and any path that+ # skips it keeps the #10613 bug (presence, not value).+ LlamaCppBackend._sanitize_p2p_env(env)# _llama_lib_dir resolves the llama-server symlink to the real build/bin.binary_dir = str(_llama_lib_dir(binary))@@ -23783,12 +24204,77 @@ class LlamaCppBackend:forced_cpu = _arch_gate_forced_cpu,)- # DC NVIDIA GPUs: FP32 accum (+ P2P / launch queues for multi-GPU).- # See _apply_datacenter_env; opt out with UNSLOTH_DISABLE_DC_TUNING=1.- if not is_vulkan_backend and self._apply_datacenter_env(env, gpu_indices):- multi_gpu = self._effective_gpu_count(gpu_indices) > 1- logger.info(- f"Data-center GPU detected: applied DC llama.cpp env tuning (multi_gpu={multi_gpu})"+ # An inherited GGML_CUDA_P2P=0 means OFF to the user and ON to+ # llama.cpp, which tests presence and not value (#10613).+ # _llama_server_env_for_binary already stripped it for every backend;+ # this covers an env assembled another way.+ self._sanitize_p2p_env(env)++ # A multi-GPU box that is NOT a datacenter part never reaches the+ # block below, so warn here or not at all: the 2x RTX 3090 case,+ # whose own truthy GGML_CUDA_P2P rides through to the child.+ # Computed here because a deliberate zero-offload load is masked to+ # CUDA_VISIBLE_DEVICES=-1 and moves no tensors between GPUs, so+ # probing or warning it is noise about traffic it never generates.+ _cpu_only_zero_offload = (+ gpu_memory_mode == "manual"+ and gpu_layers == 0+ and not is_vulkan_backend+ and not self._zero_offload_keeps_gpu_visible(cmd, env)+ )++ # Whether the child will resolve gpu_indices as the nvidia-smi ids+ # the P2P gate verifies. The launch below pins PCI_BUS_ID for an+ # explicit pick, and for an auto-fit selection only when NO mask was+ # inherited (nothing to reinterpret); anything else leaves CUDA on+ # FASTEST_FIRST, where those ids may name other cards.+ # That auto-fit arm also requires the ids to BE PCI indices: when the+ # nvidia-smi query fails _get_gpu_memory falls back to torch CUDA+ # ordinals, and pinning PCI order re-emits those numbers as a+ # different set of cards. An explicit pick is unaffected, its ids+ # come from the UI's own PCI-ordered picker.+ _p2p_launch_order_pinned = bool(gpu_ids) or (+ os.environ.get("CUDA_VISIBLE_DEVICES") is None+ and LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is True+ )++ # Only when the fabric is NOT confirmed: on a verified NV# pair the+ # flag is the benchmarked configuration. Datacenter boxes are+ # excluded because _apply_datacenter_env warns about the same+ # variable from its own veto branch.+ if (+ env.get("GGML_CUDA_P2P")+ and not _cpu_only_zero_offload+ and not is_vulkan_backend+ and not self._is_datacenter_gpu(gpu_indices)+ and self._effective_gpu_count(gpu_indices) > 1+ and not LlamaCppBackend._warned_no_nvlink+ ):+ _p2p_veto = self._p2p_veto_reason(gpu_indices, _p2p_launch_order_pinned)+ if _p2p_veto is not None:+ LlamaCppBackend._warned_no_nvlink = True+ logger.warning(+ "GGML_CUDA_P2P=%s is set in the environment but this "+ "selection has no confirmed NVLink fabric: %s. Peer copies "+ "over PCIe can be discarded silently while CUDA still "+ "reports success, which surfaces as garbled output rather "+ "than an error. Verify with scripts/p2p_integrity_probe.py "+ "and unset the variable entirely (not =0, which reads as ON "+ "upstream) if it fails (#10613).",+ env["GGML_CUDA_P2P"],+ _p2p_veto,+ )++ # DC NVIDIA GPUs: FP32 accum and launch queues, plus P2P once a+ # multi-GPU selection has a CONFIRMED NVLink fabric (#10613). Opt+ # out with UNSLOTH_DISABLE_DC_TUNING=1, or UNSLOTH_DISABLE_DC_P2P=1+ # for the peer flag alone.+ if not (is_vulkan_backend or _cpu_only_zero_offload):+ self._apply_datacenter_env(+ env,+ gpu_indices,+ p2p_opted_out = self._p2p_user_opted_out(),+ launch_order_pinned = _p2p_launch_order_pinned,)# Pin to selected GPU(s) (issue #7164; resolved above into gpu_indices).@@ -23803,12 +24289,6 @@ class LlamaCppBackend:# pin (extras or an inherited LLAMA_ARG_DEVICE) keeps control of its# own devices, the child aborting on a pin it cannot see. The# draft-device forms count too: parsed with no drafter loaded.- _cpu_only_zero_offload = (- gpu_memory_mode == "manual"- and gpu_layers == 0- and not is_vulkan_backend- and not self._zero_offload_keeps_gpu_visible(cmd, env)- )_child_gpu_physical_ids: Optional[tuple[int, ...]] = Noneif not is_vulkan_backend and _gpu_mem:_child_gpu_physical_ids = self._unmasked_child_gpu_physical_ids()@@ -23867,7 +24347,14 @@ class LlamaCppBackend:# When the user picked GPUs by index, align CUDA's ordering with# the PCI-bus order the picker enumerated, so "GPU 1" in the UI is# GPU 1 to llama.cpp, not CUDA's FASTEST_FIRST default (#5025).- if gpu_ids:+ # Extended to an auto-fit selection ONLY when the inherited mask+ # is ABSENT: a scheduler's CUDA_VISIBLE_DEVICES=0,1 meaning+ # physical 2,0 would be re-read as physical 0,1 once the order+ # changes, running on a device hidden on purpose. With no mask+ # there is nothing to preserve, and the one emitted below is our+ # own nvidia-smi indices, which pinning is what makes true. Where+ # we cannot pin, _p2p_veto_reason withholds the flag.+ if _p2p_launch_order_pinned:env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"# Mask on AMD at the ROCr/HSA layer: HIP-only masking still# enumerates every agent first, which segfaults on a deselecteddiff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.pyindex 0900a7c61a1..73195e03d47 100644--- a/studio/backend/core/rag/embed_llama_server.py+++ b/studio/backend/core/rag/embed_llama_server.py@@ -742,6 +742,12 @@ class LlamaServerBackend:def _build_env(self, binary: str, *, use_gpu: bool) -> dict[str, str]:env = child_env_without_native_path_secret()+ # Not routed through _llama_server_env_for_binary, so it needs its own strip:+ # an inherited GGML_CUDA_P2P=0 means OFF to the user and ON to llama.cpp, and+ # a corrupt embedding degrades retrieval quietly (#10613).+ from core.inference.llama_cpp import LlamaCppBackend++ LlamaCppBackend._sanitize_p2p_env(env)env["LLAMA_SET_ROWS"] = "1"if sys.platform == "darwin":# _llama_lib_dir, not Path(binary).parent: the managed install puts an entrypoint in front of thediff --git a/studio/backend/tests/test_datacenter_gpu_tuning.py b/studio/backend/tests/test_datacenter_gpu_tuning.pyindex fd9b291e8a8..cd0e59d1d67 100644--- a/studio/backend/tests/test_datacenter_gpu_tuning.py+++ b/studio/backend/tests/test_datacenter_gpu_tuning.py@@ -1,13 +1,18 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Data-center llama.cpp env tuning: FP32 accum (+ P2P / launch queues for-multi-GPU) must apply only to datacenter NVIDIA parts, never consumer GeForce,-AMD/ROCm, CPU or macOS. User values win; UNSLOTH_DISABLE_DC_TUNING=1 disables.+"""Data-center llama.cpp env tuning: datacenter NVIDIA parts only, never consumer+GeForce, ROCm, CPU or macOS. User values win; UNSLOTH_DISABLE_DC_TUNING=1 disables.++P2P has a second, stricter gate (#10613): a datacenter NAME is not evidence of an+NVLink fabric, and on a non-NVLink box the peer copy is silently discarded while+still reporting success, so every model emits garbage. It needs a confirmed NV#+link across the selection and fails CLOSED on unknowns."""from __future__ import annotations+import subprocessimport sysimport types@@ -22,7 +27,7 @@ def _fake_torch(hip = None,cuda_ok = True,):- """torch stub: version.hip, cuda.is_available/device_count, get_device_properties(i).name."""+ """torch stub: version.hip, cuda.*, get_device_properties(i).name."""t = types.ModuleType("torch")t.version = types.SimpleNamespace(hip = hip)t.cuda = types.SimpleNamespace(@@ -33,11 +38,94 @@ def _fake_torch(return t+# Real `nvidia-smi topo -m` from an 8x B200 NVLink host: ANSI-underlined header, NIC+# rows/columns, affinity columns and a Legend, none of which a toy string exercises.+TOPO_NVLINK_8X = (+ "\t\x1b[4mGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tNIC0\tNIC1\t"+ "CPU Affinity\tNUMA Affinity\tGPU NUMA ID\x1b[0m\n"+ "GPU0\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"+ "GPU1\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"+ "GPU2\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"+ "GPU3\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tNV18\tSYS\tSYS\t0-47,96-143\t0\t\tN/A\n"+ "GPU4\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tNV18\tPIX\tPIX\t48-95,144-191\t1\t\tN/A\n"+ "GPU5\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNV18\tPIX\tPIX\t48-95,144-191\t1\t\tN/A\n"+ "GPU6\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNV18\tNODE\tNODE\t48-95,144-191\t1\t\tN/A\n"+ "GPU7\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\tNV18\t X \tNODE\tNODE\t48-95,144-191\t1\t\tN/A\n"+ "NIC0\tSYS\tSYS\tSYS\tSYS\tPIX\tPIX\tNODE\tNODE\t X \tPIX\t\t\t\n"+ "NIC1\tSYS\tSYS\tSYS\tSYS\tPIX\tPIX\tNODE\tNODE\tPIX\t X \t\t\t\n"+ "\n"+ "Legend:\n"+ "\n"+ " X = Self\n"+ " SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes\n"+ " NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges\n"+ " NV# = Connection traversing a bonded set of # NVLinks\n"+)++# The reporter's host (#10613): 2x RTX 6000 Ada, NODE (PCIe via a host bridge), no+# NVLink. The driver still advertises P2P as available.+TOPO_PCIE_2X = (+ "\t\x1b[4mGPU0\tGPU1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\x1b[0m\n"+ "GPU0\t X \tNODE\t0-23\t0\t\tN/A\n"+ "GPU1\tNODE\t X \t0-23\t0\t\tN/A\n"+ "\n"+ "Legend:\n"+ "\n"+ " X = Self\n"+ " NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges\n"+)++@pytest.fixture(autouse = True)def _clear_cuda_visible_devices(monkeypatch):- """Detection reads CUDA_VISIBLE_DEVICES, so clear it by default (run unmasked,- physical id == ordinal) regardless of host; masked tests set it explicitly."""+ """Run unmasked by default (physical id == ordinal); masked tests set it."""monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ monkeypatch.delenv("CUDA_DEVICE_ORDER", raising = False)+++# Captured before the autouse fixture stubs it, so the probe can run for real.+_REAL_IOMMU_IS_TRANSLATING = LlamaCppBackend.__dict__["_iommu_is_translating"].__func__+++def _no_nvidia_smi(*a, **k):+ raise FileNotFoundError("nvidia-smi")++[email protected](autouse = True)+def _isolate_host_topology(monkeypatch):+ """Keep the P2P gate off the real host: only nvidia-smi's output is stubbed (the+ parser always runs), caches are dropped either side, and the platform probes are+ pinned so a CI box with GPUs cannot colour the results."""+ LlamaCppBackend._NVLINK_TOPO_CACHE = None+ LlamaCppBackend._IOMMU_CACHE = None+ monkeypatch.setattr(subprocess, "run", _no_nvidia_smi)+ monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(lambda *a: False))+ monkeypatch.setattr(LlamaCppBackend, "_running_virtualized", staticmethod(lambda: False))+ # Overrides off and the warn-once latch reset: no host env, no ordering leak.+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_P2P", raising = False)+ monkeypatch.delenv("UNSLOTH_FORCE_DC_P2P", raising = False)+ # Most tests are about the topology verdict, not device ordering, so default to+ # a pinned order; the unpinned-path tests delete this themselves.+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")+ LlamaCppBackend._warned_no_nvlink = False+ yield+ LlamaCppBackend._NVLINK_TOPO_CACHE = None+ LlamaCppBackend._IOMMU_CACHE = None+ LlamaCppBackend._warned_no_nvlink = False+++def _use_topo(+ monkeypatch,+ text,+ returncode = 0,+):+ """Feed the real parser canned `nvidia-smi topo -m` output."""+ monkeypatch.setattr(+ subprocess,+ "run",+ lambda *a, **k: types.SimpleNamespace(returncode = returncode, stdout = text, stderr = ""),+ )+ LlamaCppBackend._NVLINK_TOPO_CACHE = None# ---------------------------------------------------------------------------@@ -67,15 +155,14 @@ def _clear_cuda_visible_devices(monkeypatch):(["NVIDIA GeForce RTX 3090"], False),(["NVIDIA GeForce RTX 2080 Ti"], False),(["NVIDIA GeForce GTX 1080"], False),- # Workstation/laptop: short markers must not match as substrings- # ("a100" in "A1000", "a30" in "A3000").+ # Short markers must not match as substrings ("a100" in "A1000").(["NVIDIA RTX A1000 Laptop GPU"], False),(["NVIDIA RTX A1000 6GB Laptop GPU"], False),(["NVIDIA RTX A3000 Laptop GPU"], False),# Homogeneous multi-DC: all must match.(["NVIDIA B200", "NVIDIA B200"], True),(["NVIDIA H100 80GB HBM3", "NVIDIA H100 80GB HBM3"], True),- # Mixed DC + consumer: non-DC, so tuning never lands on the GeForce.+ # Mixed: non-DC, so tuning never lands on the GeForce.(["NVIDIA B200", "NVIDIA GeForce RTX 4090"], False),(["NVIDIA GeForce RTX 4090", "NVIDIA B200"], False),],@@ -86,7 +173,6 @@ def test_is_datacenter_gpu(monkeypatch, names, expected):def test_is_datacenter_gpu_respects_selection(monkeypatch):- # A mixed box where only the DC GPU is selected -> True; only consumer -> False.monkeypatch.setitem(sys.modules,"torch",@@ -99,15 +185,13 @@ def test_is_datacenter_gpu_respects_selection(monkeypatch):def test_is_datacenter_gpu_out_of_range_indices_skipped(monkeypatch):monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"]))- # Out-of-range / negative indices are skipped; the one valid DC GPU still wins.+ # Invalid indices are skipped; all invalid -> nothing seen -> False.assert LlamaCppBackend._is_datacenter_gpu([0, 5, -1]) is True- # Only invalid indices -> nothing seen -> False (fail closed for the flag).assert LlamaCppBackend._is_datacenter_gpu([5, 9]) is Falsedef test_is_datacenter_gpu_masked_host_physical_ids(monkeypatch):- # Mask 4,5,6,7 -> ordinals 0..3 == physical 4..7. PHYSICAL selection [4,5]- # must resolve, not index out of range (the pre-fix bug: 4 >= device_count).+ # PHYSICAL selection [4,5] must resolve (the pre-fix bug: 4 >= device_count).monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "4,5,6,7")monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))assert LlamaCppBackend._is_datacenter_gpu([4, 5]) is True@@ -117,15 +201,14 @@ def test_is_datacenter_gpu_masked_host_physical_ids(monkeypatch):def test_is_datacenter_gpu_masked_host_reordered(monkeypatch):- # Reordered mask preserves order: ordinal 0 -> physical 7, 1 -> 4, ...+ # Reordered mask preserves order: ordinal 0 -> physical 7.monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "7,4,5,6")monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100 80GB HBM3"] * 4))assert LlamaCppBackend._is_datacenter_gpu([7, 4]) is Truedef test_is_datacenter_gpu_masked_host_mixed_class(monkeypatch):- # Mask 4,5: physical 4 = GeForce, physical 5 = B200. Detection must follow the- # selected physical GPU, not a same-numbered ordinal.+ # Detection must follow the selected physical GPU, not a same-numbered ordinal.monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "4,5")monkeypatch.setitem(sys.modules,@@ -138,15 +221,14 @@ def test_is_datacenter_gpu_masked_host_mixed_class(monkeypatch):def test_is_datacenter_gpu_unparsable_mask_falls_back(monkeypatch):- # Unparsable (UUID) mask falls back to physical id == ordinal (mirrors- # _get_gpu_free_memory), so ordinal lookup still classifies the device.+ # Unparsable (UUID) mask falls back to physical id == ordinal.monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "GPU-abcdef12")monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"]))assert LlamaCppBackend._is_datacenter_gpu([0]) is Truedef test_is_datacenter_gpu_rocm_is_false(monkeypatch):- # ROCm reuses torch.cuda.*; an MI300X must not qualify.+ # ROCm reuses torch.cuda.*.monkeypatch.setitem(sys.modules,"torch",@@ -177,7 +259,6 @@ def test_effective_gpu_count_explicit_selection(monkeypatch):def test_effective_gpu_count_none_uses_visible(monkeypatch):- # None -> visible device count.monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))assert LlamaCppBackend._effective_gpu_count(None) == 4@@ -210,6 +291,7 @@ def test_apply_env_single_dc_gpu_sets_only_fp32(monkeypatch):def test_apply_env_multi_dc_gpu_sets_all(monkeypatch):monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)env: dict = {}assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is Trueassert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"@@ -218,9 +300,9 @@ def test_apply_env_multi_dc_gpu_sets_all(monkeypatch):def test_apply_env_none_indices_uses_visible_count(monkeypatch):- # None on a 2x DC box -> multi-GPU flags applied.monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100", "NVIDIA H100"]))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)env: dict = {}assert LlamaCppBackend._apply_datacenter_env(env, None) is Trueassert env["GGML_CUDA_P2P"] == "1"@@ -238,12 +320,13 @@ def test_apply_env_consumer_gpu_is_noop(monkeypatch):def test_apply_env_user_value_wins(monkeypatch):monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)env = {- "GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "0", # user explicitly disabled+ "GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "0", # user disabled"CUDA_SCALE_LAUNCH_QUEUES": "8x", # user override}assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True- # setdefault must not clobber user values; the unset one still defaults.+ # setdefault must not clobber user values; the unset one defaults.assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "0"assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "8x"assert env["GGML_CUDA_P2P"] == "1"@@ -266,13 +349,714 @@ def test_apply_env_fail_open_on_detection_error(monkeypatch):def test_apply_env_masked_host_multi_dc(monkeypatch):- # End-to-end masked host (mask 4,5,6,7, physical selection [4,5]): pre-fix- # applied no tuning; now all three multi-GPU flags must be set.+ # Masked host end-to-end: pre-fix this applied no tuning at all.monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "4,5,6,7")monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)env: dict = {}assert LlamaCppBackend._apply_datacenter_env(env, [4, 5]) is Trueassert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"assert env["GGML_CUDA_P2P"] == "1"assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"+++# ---------------------------------------------------------------------------+# nvidia-smi topo -m parsing+# ---------------------------------------------------------------------------+++def test_topo_parses_real_nvlink_table(monkeypatch):+ # 8 GPUs -> 8*7 ordered pairs; NIC rows, affinity columns and legend ignored.+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ matrix = LlamaCppBackend._nvlink_topology()+ assert len(matrix) == 56+ assert set(matrix.values()) == {"NV18"}+ assert (0, 0) not in matrix # self ("X") is not a pair+ assert matrix[(6, 7)] == "NV18"+++def test_topo_parses_pcie_table(monkeypatch):+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ assert LlamaCppBackend._nvlink_topology() == {(0, 1): "NODE", (1, 0): "NODE"}+++def test_topo_is_cached_then_refreshable(monkeypatch):+ calls = []++ def _once(*a, **k):+ calls.append(1)+ return types.SimpleNamespace(returncode = 0, stdout = TOPO_PCIE_2X, stderr = "")++ monkeypatch.setattr(subprocess, "run", _once)+ LlamaCppBackend._NVLINK_TOPO_CACHE = None+ LlamaCppBackend._nvlink_topology()+ LlamaCppBackend._nvlink_topology()+ assert len(calls) == 1 # one shell-out per process+ LlamaCppBackend._nvlink_topology(refresh = True)+ assert len(calls) == 2+++def test_topo_unavailable_paths_are_none(monkeypatch):+ # Missing binary, non-zero exit and a GPU-less table mean "unknown", never NVLink.+ assert LlamaCppBackend._nvlink_topology() is None+ _use_topo(monkeypatch, TOPO_NVLINK_8X, returncode = 9)+ assert LlamaCppBackend._nvlink_topology() is None+ _use_topo(monkeypatch, "Legend:\n X = Self\n")+ assert LlamaCppBackend._nvlink_topology() is None+++def test_topo_truncated_row_refuses_to_guess(monkeypatch):+ _use_topo(+ monkeypatch,+ "\tGPU0\tGPU1\tCPU Affinity\n"+ "GPU0\t X \tNV18\t0-23\n"+ "GPU1\tNV18\n", # row cut short: fewer labels than columns+ )+ assert LlamaCppBackend._nvlink_topology() is None+++# ---------------------------------------------------------------------------+# _p2p_veto_reason (the #10613 gate)+# ---------------------------------------------------------------------------+++def test_p2p_vetoed_on_rtx_6000_ada(monkeypatch):+ # A datacenter NAME with no NVLink.+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ reason = LlamaCppBackend._p2p_veto_reason([0, 1])+ assert reason is not None and "NVLink-capable" in reason+++ "name",+ ["NVIDIA L40S", "NVIDIA L40", "NVIDIA L4", "NVIDIA RTX PRO 6000 Blackwell Server Edition"],+)+def test_p2p_vetoed_on_other_connectorless_parts(monkeypatch, name):+ monkeypatch.setitem(sys.modules, "torch", _fake_torch([name] * 2))+ _use_topo(monkeypatch, TOPO_NVLINK_8X) # even a lying matrix must not rescue them+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None+++def test_p2p_allowed_on_confirmed_nvlink(monkeypatch):+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None+++def test_p2p_vetoed_when_topology_unknown(monkeypatch):+ # NVLink-capable parts but no nvidia-smi: fail CLOSED (the old code trusted the+ # name alone).+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ reason = LlamaCppBackend._p2p_veto_reason([0, 1])+ assert reason is not None and "interconnect matrix" in reason+++def test_p2p_vetoed_on_pcie_pair_between_nvlink_parts(monkeypatch):+ # A100s with no bridge fitted is real, and the name gate would allow it.+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 2))+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ reason = LlamaCppBackend._p2p_veto_reason([0, 1])+ assert reason is not None and "NODE" in reason+++def test_p2p_veto_names_the_iommu_on_bare_metal(monkeypatch):+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 2))+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(lambda *a: True))+ reason = LlamaCppBackend._p2p_veto_reason([0, 1])+ assert "translating IOMMU" in reason+ # Under a hypervisor CUDA supports pass-through P2P, so the IOMMU is not it.+ monkeypatch.setattr(LlamaCppBackend, "_running_virtualized", staticmethod(lambda: True))+ assert "IOMMU" not in LlamaCppBackend._p2p_veto_reason([0, 1])+++def test_p2p_exact_mapping_consults_only_the_selected_pair(monkeypatch):+ # PCI_BUS_ID makes the index spaces the same, so a partially linked box can still+ # use its NVLinked pair.+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100"] * 3))+ _use_topo(+ monkeypatch,+ "\tGPU0\tGPU1\tGPU2\tCPU Affinity\n"+ "GPU0\t X \tNV18\tNODE\t0-23\n"+ "GPU1\tNV18\t X \tNODE\t0-23\n"+ "GPU2\tNODE\tNODE\t X \t0-23\n",+ )+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None+ assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None+ assert LlamaCppBackend._p2p_veto_reason([0, 1, 2]) is not None+++def test_p2p_checks_the_selected_pair_on_a_partially_linked_box(monkeypatch):+ # Selection and matrix are both nvidia-smi indices, so a linked pair is allowed+ # and an unlinked one refused on the same box, whatever CUDA_DEVICE_ORDER says.+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA H100"] * 3))+ _use_topo(+ monkeypatch,+ "\tGPU0\tGPU1\tGPU2\tCPU Affinity\n"+ "GPU0\t X \tNV18\tNODE\t0-23\n"+ "GPU1\tNV18\t X \tNODE\t0-23\n"+ "GPU2\tNODE\tNODE\t X \t0-23\n",+ )+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None+ assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None+ # No selection: the whole visible box has to qualify.+ assert LlamaCppBackend._p2p_veto_reason(None) is not None+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None+++# ---------------------------------------------------------------------------+# End-to-end: the reported bug, and the opt-out that did not work+# ---------------------------------------------------------------------------+++def test_apply_env_rtx_6000_ada_gets_fp32_but_not_p2p(monkeypatch):+ """#10613 exactly: 2x RTX 6000 Ada keeps FP32 accum, harmless per the reporter's+ own isolation, and loses the P2P that garbled every model."""+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ env: dict = {}+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True+ assert "GGML_CUDA_P2P" not in env+ # Launch-queue depth moves no data across the bus and #10613 measured it clean on+ # that host, so it is deliberately NOT gated with P2P.+ assert env == {"GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "1", "CUDA_SCALE_LAUNCH_QUEUES": "4x"}+++def test_apply_env_l40s_multi_gpu_gets_fp32_but_not_p2p(monkeypatch):+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA L40S"] * 4))+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ env: dict = {}+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1, 2, 3]) is True+ assert "GGML_CUDA_P2P" not in env+ assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"+++def test_sanitize_falsy_user_p2p_is_removed():+ """ggml tests GGML_CUDA_P2P for presence, so passing a user's "0" through ENABLES+ peer copies; the opt-out has to unset it."""+ for value in ("0", "false", "OFF", "no", "", " 0 "):+ env = {"GGML_CUDA_P2P": value, "OTHER": "kept"}+ assert LlamaCppBackend._sanitize_p2p_env(env) == value+ assert env == {"OTHER": "kept"}, value+++def test_sanitize_leaves_a_truthy_user_p2p_alone():+ env = {"GGML_CUDA_P2P": "1"}+ assert LlamaCppBackend._sanitize_p2p_env(env) is None+ assert env == {"GGML_CUDA_P2P": "1"}+++def test_opted_out_p2p_is_not_reintroduced_by_the_default(monkeypatch):+ # The call site strips it; the DC block must not put it back. The rest stands.+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ env: dict = {}+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1], p2p_opted_out = True) is True+ assert "GGML_CUDA_P2P" not in env+ assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"+ assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"+++def test_disable_dc_p2p_drops_peer_flag_but_keeps_fp32(monkeypatch):+ # The surgical opt-out: UNSLOTH_DISABLE_DC_TUNING is all-or-nothing and discards+ # a tuning that is not implicated.+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ env: dict = {}+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True+ assert "GGML_CUDA_P2P" not in env+ assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"+++def test_force_dc_p2p_opts_back_in_over_an_unreadable_topology(monkeypatch):+ # For a real but unparsable fabric, once p2p_integrity_probe.py passes.+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_P2P", raising = False)+ monkeypatch.setenv("UNSLOTH_FORCE_DC_P2P", "1")+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ env: dict = {}+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True+ assert env["GGML_CUDA_P2P"] == "1"+++def test_disable_dc_p2p_beats_force_dc_p2p(monkeypatch):+ # Both set: the safe direction wins.+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")+ monkeypatch.setenv("UNSLOTH_FORCE_DC_P2P", "1")+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ env: dict = {}+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True+ assert "GGML_CUDA_P2P" not in env+++def test_sanitize_applies_to_a_consumer_box_that_never_reaches_the_dc_gate():+ # A 2x RTX 3090 never matches the allowlist, so only the call-site strip helps.+ env = {"GGML_CUDA_P2P": "0"}+ assert LlamaCppBackend._sanitize_p2p_env(env) == "0"+ assert env == {}+++def test_apply_env_truthy_user_p2p_still_wins(monkeypatch):+ # An explicit opt-in survives the veto.+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ env = {"GGML_CUDA_P2P": "1"}+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True+ assert env["GGML_CUDA_P2P"] == "1"+++def test_apply_env_single_nvlink_gpu_still_skips_p2p(monkeypatch):+ # One GPU has no peer.+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 4))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ env: dict = {}+ assert LlamaCppBackend._apply_datacenter_env(env, [0]) is True+ assert env == {"GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F": "1"}+++def test_apply_env_multi_dc_without_nvidia_smi_withholds_p2p(monkeypatch):+ # Unknown topology must cost the optimisation, not the correctness.+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ env: dict = {}+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True+ assert "GGML_CUDA_P2P" not in env+ assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"+++# ---------------------------------------------------------------------------+# Platform probes+# ---------------------------------------------------------------------------+++def _iommu_tree(tmp_path, types_by_group):+ root = tmp_path / "iommu_groups"+ for group, kind in types_by_group.items():+ (root / str(group)).mkdir(parents = True)+ if kind is not None:+ (root / str(group) / "type").write_text(f"{kind}\n")+ return str(root)+++def test_iommu_identity_groups_are_not_translating(tmp_path):+ root = _iommu_tree(tmp_path, {0: "identity", 1: "identity"})+ assert _REAL_IOMMU_IS_TRANSLATING(root) is False+++def test_iommu_dma_group_is_translating(tmp_path):+ # The reporter's host: 175 groups in DMA-FQ (translating) mode.+ root = _iommu_tree(tmp_path, {0: "identity", 1: "DMA-FQ"})+ assert _REAL_IOMMU_IS_TRANSLATING(root) is True+++def test_iommu_absent_or_empty_is_not_translating(tmp_path):+ assert _REAL_IOMMU_IS_TRANSLATING(str(tmp_path / "nope")) is False+ (tmp_path / "empty").mkdir()+ assert _REAL_IOMMU_IS_TRANSLATING(str(tmp_path / "empty")) is False+++def test_iommu_unreadable_types_are_unknown(tmp_path):+ # Pre-5.x kernels expose groups with no `type`: unknown, not "safe".+ root = _iommu_tree(tmp_path, {0: None, 1: None})+ assert _REAL_IOMMU_IS_TRANSLATING(root) is None+++# ---------------------------------------------------------------------------+# Gaps found reviewing the #10613 fix+# ---------------------------------------------------------------------------+++def test_disable_dc_p2p_also_drops_an_inherited_truthy_value(monkeypatch):+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ env = {"GGML_CUDA_P2P": "1"}+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])+ # Presence is truth upstream, so "disabled" has to mean absent.+ assert "GGML_CUDA_P2P" not in env+++def test_apply_env_does_not_leave_a_falsy_value_present_when_called_directly(monkeypatch):+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ env = {"GGML_CUDA_P2P": "0"}+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])+ assert "GGML_CUDA_P2P" not in env+++def test_shared_llama_server_env_builder_sanitizes_p2p(monkeypatch, tmp_path):+ """The STT sidecar and the embedding probe build from this, so stripping here+ covers every llama-server child, not just chat."""+ monkeypatch.setenv("GGML_CUDA_P2P", "0")+ binary = tmp_path / "llama-server"+ binary.write_text("", encoding = "utf-8")+ env = LlamaCppBackend._llama_server_env_for_binary(str(binary))+ assert "GGML_CUDA_P2P" not in env+++def test_stt_sidecar_env_sanitizes_p2p(monkeypatch, tmp_path):+ # Spawns llama-server with -ngl 99, so it is exposed like chat.+ from core.inference import stt_mtmd_sidecar++ monkeypatch.setenv("GGML_CUDA_P2P", "0")+ binary = tmp_path / "llama-server"+ binary.write_text("", encoding = "utf-8")+ assert "GGML_CUDA_P2P" not in stt_mtmd_sidecar._llama_server_child_env(str(binary))+++def test_embedding_server_env_sanitizes_p2p(monkeypatch, tmp_path):+ """Bypasses the shared builder, and a corrupt embedding degrades retrieval with+ no garbled text to notice."""+ from core.rag.embed_llama_server import LlamaServerBackend++ monkeypatch.setenv("GGML_CUDA_P2P", "0")+ binary = tmp_path / "llama-server"+ binary.write_text("", encoding = "utf-8")+ server = LlamaServerBackend.__new__(LlamaServerBackend)+ env = server._build_env(str(binary), use_gpu = False)+ assert "GGML_CUDA_P2P" not in env+++# ---------------------------------------------------------------------------+# Found by the platform/hardware simulation matrix (temp/sim_10613)+# ---------------------------------------------------------------------------+++# 4x A100 bridged over (0,1) and (2,3), PCIe between the islands: the standard+# bridged build, which the whole-matrix fallback used to veto outright.+TOPO_BRIDGED_4X = (+ "\t\x1b[4mGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\tNUMA Affinity\x1b[0m\n"+ "GPU0\t X \tNV12\tSYS\tSYS\t0-23\t0\n"+ "GPU1\tNV12\t X \tSYS\tSYS\t0-23\t0\n"+ "GPU2\tSYS\tSYS\t X \tNV12\t24-47\t1\n"+ "GPU3\tSYS\tSYS\tNV12\t X \t24-47\t1\n"+ "\nLegend:\n\n X = Self\n"+)+++def test_bridged_pair_keeps_p2p_on_a_partially_bridged_box(monkeypatch):+ """A genuinely NVLinked pair on a partially bridged box keeps P2P: gpu_indices+ are already nvidia-smi indices, so the pair checked is the pair selected."""+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None+ assert LlamaCppBackend._p2p_veto_reason([2, 3]) is None+ # Across the islands the copy really would cross PCIe.+ assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None+ assert LlamaCppBackend._p2p_veto_reason([1, 3]) is not None+++def test_selection_is_not_remapped_out_of_the_nvidia_smi_index_space(monkeypatch):+ """The selection indexes the topology matrix VERBATIM: both come from nvidia-smi,+ one enumeration. Remapping them as CUDA ordinals would, under FASTEST_FIRST on a+ bridged box, turn a PCIe-crossing selection into an NVLinked-looking one and+ enable the flag this gate exists to withhold."""+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)+ for pair in ([0, 1], [2, 3]):+ assert LlamaCppBackend._p2p_veto_reason(pair) is None, pair+ for pair in ([0, 2], [0, 3], [1, 2], [1, 3]):+ assert LlamaCppBackend._p2p_veto_reason(pair) is not None, pair+ # An unset CUDA_DEVICE_ORDER must not change any of the above.+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "FASTEST_FIRST")+ LlamaCppBackend._NVLINK_TOPO_CACHE = None+ assert LlamaCppBackend._p2p_veto_reason([0, 2]) is not None+++def test_name_gate_veto_still_names_the_iommu(monkeypatch):+ """The reporter's host fails the NAME gate first, so without this the one+ actionable diagnosis never reaches them (#10613)."""+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))+ monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(lambda *a: True))+ LlamaCppBackend._IOMMU_CACHE = None+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ reason = LlamaCppBackend._p2p_veto_reason([0, 1])+ assert "translating IOMMU" in reason+++def test_a_raising_probe_never_fails_the_model_load(monkeypatch):+ # Losing the tuning is acceptable; taking the model load down is not.+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))++ def boom(*a, **k):+ raise OSError("sysfs exploded")++ monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(boom))+ monkeypatch.setattr(LlamaCppBackend, "_nvlink_topology", classmethod(boom))+ LlamaCppBackend._IOMMU_CACHE = None+ env: dict = {}+ assert LlamaCppBackend._apply_datacenter_env(env, [0, 1]) is True+ assert "GGML_CUDA_P2P" not in env+ assert env["GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F"] == "1"+++def test_iommu_scan_is_cached_across_loads(monkeypatch):+ # A boot-time property, and 175 groups on the reporter's host.+ calls = []++ def counted(*a, **k):+ calls.append(1)+ return True++ monkeypatch.setattr(LlamaCppBackend, "_iommu_is_translating", staticmethod(counted))+ LlamaCppBackend._IOMMU_CACHE = None+ for _ in range(5):+ LlamaCppBackend._iommu_is_translating_cached()+ assert len(calls) == 1+++def _capture_warnings(monkeypatch):+ """Collect logger.warning calls: caplog does not see structlog here, so asserting+ on it would pass vacuously."""+ from core.inference import llama_cpp as _mod++ seen: list[str] = []+ real = _mod.logger.warning+ monkeypatch.setattr(+ _mod.logger,+ "warning",+ lambda msg, *a, **k: (seen.append(str(msg)), real(msg, *a, **k))[0],+ )+ return seen+++def test_no_pcie_warning_when_the_user_opts_in_on_a_verified_fabric(monkeypatch):+ """A deliberate GGML_CUDA_P2P=1 on a confirmed NV# pair is the benchmarked+ configuration, so warning there would push users off a working optimisation."""+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None+ seen = _capture_warnings(monkeypatch)+ env = {"GGML_CUDA_P2P": "1"}+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])+ assert env["GGML_CUDA_P2P"] == "1"+ assert not [m for m in seen if "peer copies" in m], seen+++def test_pcie_warning_still_fires_on_an_unverified_fabric(monkeypatch):+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ seen = _capture_warnings(monkeypatch)+ env = {"GGML_CUDA_P2P": "1"}+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])+ assert [m for m in seen if "peer copies stay" in m], seen+++def test_masked_visible_devices_filter_needs_a_shared_index_space(monkeypatch):+ """nvidia-smi ignores CUDA_VISIBLE_DEVICES, so the matrix covers cards the child+ never touches and filtering it to the mask recovers a clean NVLinked pair. Sound+ only under PCI_BUS_ID: numeric mask entries are CUDA ordinals, and under+ FASTEST_FIRST "0,1" can mean physical 0,2."""+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 2))+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")+ assert LlamaCppBackend._p2p_veto_reason(None) is None+ # A mask spanning the islands is still correctly refused.+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,2")+ LlamaCppBackend._NVLINK_TOPO_CACHE = None+ assert LlamaCppBackend._p2p_veto_reason(None) is not None+ # Without a shared index space the mask is not trusted: the whole box has to+ # qualify, which the bridged fixture does not.+ monkeypatch.delenv("CUDA_DEVICE_ORDER")+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")+ LlamaCppBackend._NVLINK_TOPO_CACHE = None+ assert LlamaCppBackend._p2p_veto_reason(None) is not None+++def test_explicit_p2p_opt_out_does_not_warn_about_corruption(monkeypatch):+ """UNSLOTH_DISABLE_DC_P2P=1 returns before the topology is inspected, so calling+ the fabric unconfirmed would warn about corruption on a healthy box."""+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setenv("UNSLOTH_DISABLE_DC_P2P", "1")+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ seen = _capture_warnings(monkeypatch)+ env: dict = {}+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])+ assert "GGML_CUDA_P2P" not in env+ assert not [m for m in seen if "without a confirmed NVLink" in m], seen+++def test_auto_fit_selection_without_a_pinned_order_refuses_p2p(monkeypatch):+ """The gate verifies nvidia-smi physical ids, and the child resolves the same+ cards only under a pinned CUDA_DEVICE_ORDER. On a box that is NOT uniformly+ NVLinked, [0,1] here can be [0,2] there, so refuse rather than confirm NVLink for+ a pair that is not the one about to run."""+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)+ monkeypatch.delenv("CUDA_DEVICE_ORDER")+ # Auto-fit (the launch will not pin the order): withheld.+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None+ # Explicit user pick, so the launch pins PCI_BUS_ID: allowed.+ assert LlamaCppBackend._p2p_veto_reason([0, 1], launch_order_pinned = True) is None+ # Already pinned in this process: allowed either way.+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")+ LlamaCppBackend._NVLINK_TOPO_CACHE = None+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is None+++def test_auto_fit_launch_does_not_rewrite_an_inherited_device_order(monkeypatch):+ """Forcing PCI_BUS_ID for an auto-fit launch would re-read an inherited numeric+ mask: a scheduler's CUDA_VISIBLE_DEVICES=0,1 meaning physical 2,0 would become+ physical 0,1, running on a GPU that was hidden on purpose."""+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ branch = src[src.index("elif gpu_indices is not None and not is_vulkan_backend") :]+ branch = branch[: branch.index("_launch_pinned_ids")]+ pin = 'env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"'+ assert pin in branch+ # The pin must stay behind a guard, never unconditional.+ assert "if _p2p_launch_order_pinned:" in branch[: branch.index(pin)]+ # The guard must require an ABSENT inherited mask: an inherited numeric mask is+ # the thing that must not be re-read.+ start = src.index("_p2p_launch_order_pinned = ")+ guard = src[start : src.index("\n\n", start)]+ assert 'os.environ.get("CUDA_VISIBLE_DEVICES") is None' in guard, guard+++def test_datacenter_box_warns_once_not_twice(monkeypatch):+ """The call-site warning is for hosts that never reach _apply_datacenter_env; a+ datacenter box does, and its veto branch warns about the same variable."""+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA RTX 6000 Ada Generation"] * 2))+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ assert LlamaCppBackend._is_datacenter_gpu([0, 1]) is True+ seen = _capture_warnings(monkeypatch)+ env = {"GGML_CUDA_P2P": "1"}+ LlamaCppBackend._apply_datacenter_env(env, [0, 1])+ assert len([m for m in seen if "peer copies stay" in m]) == 1, seen+++def test_uniform_nvlink_box_keeps_p2p_without_a_pinned_order(monkeypatch):+ """The NVSwitch case PR #6098 benchmarked. When every pair on the box is NV# the+ index mapping is irrelevant, since whichever cards the child resolves are linked,+ and requiring a pinned order would strip P2P from every DGX/HGX auto-fit load."""+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ monkeypatch.delenv("CUDA_DEVICE_ORDER")+ for sel in ([0, 1], [0, 7], list(range(8)), None):+ assert LlamaCppBackend._p2p_veto_reason(sel) is None, sel+++def test_partially_bridged_box_still_needs_a_pinned_order(monkeypatch):+ # Not uniform, so a permutation really can change which pair runs.+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)+ monkeypatch.delenv("CUDA_DEVICE_ORDER")+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None+++def test_unmasked_auto_fit_keeps_p2p(monkeypatch):+ """An Auto load that fits on a subset sets gpu_indices with gpu_ids empty. With no+ inherited CUDA_VISIBLE_DEVICES there is nothing to reinterpret, so the launch pins+ PCI_BUS_ID and the flag is kept, sparing ordinary Auto loads on NVLink boxes."""+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA A100-SXM4-80GB"] * 4))+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)+ monkeypatch.delenv("CUDA_DEVICE_ORDER")+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ # Not uniform, so this only passes because the launch can pin the order.+ assert LlamaCppBackend._p2p_veto_reason([0, 1], launch_order_pinned = True) is None+++def test_p2p_opt_out_skips_the_topology_probe(monkeypatch):+ """A user who turned P2P off should not pay the nvidia-smi probe, up to a 10s+ timeout, to decide something they already decided."""+ monkeypatch.delenv("UNSLOTH_DISABLE_DC_TUNING", raising = False)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ probed = []+ monkeypatch.setattr(+ LlamaCppBackend,+ "_nvlink_topology",+ classmethod(lambda cls, *a, **k: (probed.append(1), None)[1]),+ )+ env: dict = {}+ LlamaCppBackend._apply_datacenter_env(env, [0, 1], p2p_opted_out = True)+ assert "GGML_CUDA_P2P" not in env+ assert env["CUDA_SCALE_LAUNCH_QUEUES"] == "4x"+ assert probed == [], "topology probed despite an explicit opt-out"+++def test_row_truncated_matrix_is_rejected(monkeypatch):+ """Row-truncated output still exits 0 and parses cleanly, and the rows that+ arrived can be uniformly NV#, which would read as "the whole box is NVLinked" and+ enable P2P on cards whose links were never seen. Partial must mean none."""+ header, *rows = TOPO_NVLINK_8X.splitlines()+ truncated = "\n".join([header] + rows[:4]) + "\n"+ _use_topo(monkeypatch, truncated)+ assert LlamaCppBackend._nvlink_topology() is None++ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 8))+ monkeypatch.delenv("CUDA_DEVICE_ORDER")+ LlamaCppBackend._NVLINK_TOPO_CACHE = None+ # The pairs that DID parse are all NV18, so without the completeness check the+ # uniform escape would allow this.+ assert LlamaCppBackend._p2p_veto_reason([0, 1]) is not None+++def test_complete_matrix_is_still_accepted(monkeypatch):+ # The completeness check must not reject the real thing.+ _use_topo(monkeypatch, TOPO_NVLINK_8X)+ assert len(LlamaCppBackend._nvlink_topology()) == 8 * 7+ _use_topo(monkeypatch, TOPO_BRIDGED_4X)+ assert len(LlamaCppBackend._nvlink_topology()) == 4 * 3+ _use_topo(monkeypatch, TOPO_PCIE_2X)+ assert len(LlamaCppBackend._nvlink_topology()) == 2 * 1+++def test_pin_requires_ids_to_be_pci_indices(monkeypatch):+ """When the nvidia-smi query fails, _get_gpu_memory falls back to torch CUDA+ ordinals, and pinning PCI order re-emits those numbers as a different set of+ cards, so Auto could run on GPUs other than the ones it measured."""+ import inspect++ src = inspect.getsource(LlamaCppBackend.load_model)+ start = src.index("_p2p_launch_order_pinned = ")+ guard = src[start : src.index("\n\n", start)]+ assert "_GPU_IDS_ARE_PCI_INDICES is True" in guard, guard+ assert 'os.environ.get("CUDA_VISIBLE_DEVICES") is None' in guard, guard+++def test_gpu_id_provenance_is_recorded(monkeypatch):+ # The nvidia-smi branch yields PCI indices; the torch fallback yields ordinals.+ monkeypatch.setattr(+ subprocess,+ "run",+ lambda *a, **k: types.SimpleNamespace(+ returncode = 0,+ stdout = "0, 1000, 2000\n1, 1000, 2000\n",+ stderr = "",+ ),+ )+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b: False))+ monkeypatch.setattr(+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "llama-server")+ )+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = None+ assert LlamaCppBackend._get_gpu_memory("llama-server")+ assert LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is True++ # nvidia-smi absent -> torch fallback -> ordinals.+ monkeypatch.setattr(subprocess, "run", _no_nvidia_smi)+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(["NVIDIA B200"] * 2))+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = None+ LlamaCppBackend._get_gpu_memory("llama-server")+ assert LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not True