Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: do not lose a DGX Spark's memory pool to the page cache when fitting context (#10704)
Daniel Han61eefe60cdbe
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 0aa089ccbca..ea86b1e9575 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -9594,6 +9594,32 @@ class LlamaCppBackend:except Exception:return False+ # PHYSICAL integrated ids per visibility mask. Integratedness cannot change under+ # a fixed mask, and the mask decides which devices the answer is about.+ _INTEGRATED_CUDA_IDS: dict[tuple, set[int]] = {}++ @staticmethod+ def _integrated_cuda_mask_key() -> tuple:+ return tuple(+ os.environ.get(name)+ for name in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES")+ )++ @staticmethod+ def _integrated_cuda_probe_is_free() -> bool:+ """True when ``_integrated_cuda_gpu_ids()`` costs no NEW CUDA context.++ That probe pins a ~700 MiB primary context per visible card for the life of+ this process, and the launch preflight runs AFTER the VRAM budget was taken, so+ probing there can OOM a tightly fitted child against a stale budget. So the+ preflight never probes: it reads an answer only if one is cached for this mask.+ ``_get_gpu_memory`` fills that cache on its torch arm before reading any+ free/total figure, which puts the cost inside the snapshot. That is the arm an+ integrated SoC takes anyway (nvidia-smi answers ``[N/A]`` on a Spark, so the CLI+ probe parses nothing); a host whose nvidia-smi answers never pays for it.+ """+ return LlamaCppBackend._integrated_cuda_mask_key() in LlamaCppBackend._INTEGRATED_CUDA_IDS+@staticmethoddef _integrated_cuda_gpu_ids() -> set[int]:"""PHYSICAL ids of visible CUDA GPUs whose "VRAM" is shared system RAM.@@ -9604,6 +9630,12 @@ class LlamaCppBackend:``torch.cuda.*`` and ``_rocm_unified_memory_gpu_ids`` already answers for it.Empty off CUDA and on error, so every caller keeps its discrete-GPU default."""+ cached = LlamaCppBackend._INTEGRATED_CUDA_IDS.get(+ LlamaCppBackend._integrated_cuda_mask_key()+ )+ if cached is not None:+ # Settled: a second pass over the devices could learn nothing.+ return set(cached)try:import torch@@ -9615,10 +9647,15 @@ class LlamaCppBackend:# (CUDA_VISIBLE_DEVICES=2) does not answer for the wrong card.physical_ids = LlamaCppBackend._resolve_visible_physical_ids()integrated: set[int] = set()+ # A device that did not answer leaves this incomplete: still returned, so+ # an unqueryable card keeps its discrete default, but not remembered, since+ # a caller reading it as settled would retry the query that failed.+ complete = Truefor ordinal in range(torch.cuda.device_count()):try:props = torch.cuda.get_device_properties(ordinal)except Exception:+ complete = Falsecontinue# Both spellings: the attribute was `integrated` before torch renamed# it, and an old wheel exposing neither reads discrete.@@ -9631,6 +9668,12 @@ class LlamaCppBackend:if physical_ids is not None and ordinal < len(physical_ids)else ordinal)+ # Cached for the preflight, but only a pass that reached every device: a+ # torch that raised says nothing about the hardware.+ if complete:+ LlamaCppBackend._INTEGRATED_CUDA_IDS[+ LlamaCppBackend._integrated_cuda_mask_key()+ ] = integratedreturn integratedexcept Exception:return set()@@ -9652,6 +9695,31 @@ class LlamaCppBackend:except Exception:return False+ @staticmethod+ def _integrated_cuda_selection_is_all_shared(gpu_indices = None) -> bool:+ """True when EVERY credited CUDA device is an integrated SoC.++ Stricter than ``_integrated_cuda_unified_memory``, which answers for ANY, and+ that difference matters here: this guard charges the WHOLE model against system+ RAM, and layers placed on a discrete card in a mixed selection never touch the+ shared pool. Answering for any would call an otherwise fitting load oversized,+ warn about it, and override an explicitly unmapped load to use mmap.+ """+ try:+ integrated = LlamaCppBackend._integrated_cuda_gpu_ids()+ if not integrated:+ return False+ if gpu_indices is not None:+ return bool(gpu_indices) and all(_i in integrated for _i in gpu_indices)+ # Nothing pinned, so the credited set is every visible device.+ visible = LlamaCppBackend._resolve_visible_physical_ids()+ if visible is None:+ import torch+ visible = list(range(torch.cuda.device_count()))+ return bool(visible) and all(_i in integrated for _i in visible)+ except Exception:+ return False+# "Off" spellings ggml itself ignores (it tests presence); we honour them._UNIFIED_MEMORY_OFF = frozenset({"", "0", "false", "no", "off"})@@ -11428,8 +11496,15 @@ class LlamaCppBackend:# Shared-pool APU: same as the Vulkan iGPU path. Hold back the host# margin, and report total 0 since that "total" is system RAM.unified_ids = LlamaCppBackend._rocm_unified_memory_gpu_ids()+ # An integrated CUDA SoC shares one pool too, and its free reading is wrong+ # in the OPPOSITE direction to ROCm's -- see below.+ integrated_ids = LlamaCppBackend._integrated_cuda_gpu_ids()# Same #7624 arch gate the amd-smi branch applies, from the one helper.arch_keeps = LlamaCppBackend._rocm_arch_gate_keep(binary, torch, for_llama_server)+ # How many devices the shared pool is about to be divided between. Read+ # once, from the same set the loop tests, so the count and the per-device+ # decision cannot disagree and produce a division by zero.+ shared_cuda_count = sum(1 for _i in integrated_ids if arch_keeps(_i))gpus = []# Windows ROCm's free reading is an over-report on discrete cards too,# not only on the shared pool handled below (#8403). It is capped@@ -11447,7 +11522,10 @@ class LlamaCppBackend:if not arch_keeps(idx):continueshared = idx in unified_ids+ integrated = idx in integrated_ids+ cgroup_bound = Falseraw_mib = free_bytes // (1024 * 1024)+ total_mib = total_bytes // (1024 * 1024)if shared:# ROCm's free is unreliable on a shared pool (Windows HIP# reports free==total, #7072), and system RAM is the real@@ -11455,14 +11533,56 @@ class LlamaCppBackend:avail = LlamaCppBackend._available_system_memory_mib()if avail is not None:raw_mib = min(raw_mib, avail)- free_mib = _apply_igpu_host_reserve_mib(raw_mib, shared)+ elif integrated:+ # cudaMemGetInfo's free half here is MemFree, which counts the+ # page cache as used, so a GGUF's own download or mmap collapses it+ # and the context is fitted against its own cached bytes (#9889).+ # MemAvailable prices that reclaimable cache back in.+ avail = LlamaCppBackend._available_system_memory_mib()+ # A zero total is a probe that could not size the pool, not a pool+ # of zero: capping against it would take the device to nothing.+ if avail is not None and total_mib > 0:+ raw_mib = min(total_mib, max(raw_mib, avail))+ # Again as a CEILING: `avail` is cgroup-capped already, but as a+ # lower bound that is thrown away whenever host-wide MemFree is+ # larger, the normal case in a container. Allocations here are+ # charged to the cgroup, so an oversized fit dies at memory.max.+ # `<=` below, not `<`: _available_system_memory_mib is itself+ # cgroup-capped, so it hands back exactly the remainder whenever+ # MemAvailable exceeds it, which makes equality the ordinary result+ # on a constrained Spark rather than a coincidence.+ cgroup_mib = LlamaCppBackend._cgroup_available_memory_mib()+ if cgroup_mib is not None and cgroup_mib <= raw_mib:+ raw_mib = cgroup_mib+ # ...and then the pool is the container's, not the device's.+ # _vram_usable_mib takes its occupancy reserve as+ # (1 - frac) * total, so a host-wide total against a container's+ # free reading reserves memory this process cannot reach: a+ # 16 GiB cgroup on a 121 GiB Spark loses most of its budget to+ # a card it does not have. Publishing no total is the same+ # answer the ROCm shared pool gives, and prices the fit off the+ # free reading, which IS the container's ceiling.+ cgroup_bound = True+ # One pool, however many integrated devices draw on it. Every+ # figure above is a whole-pool figure, so handing it to each+ # device in turn lets a caller that sums across cards commit the+ # same bytes twice. Share it instead. Division by 1 on every+ # shipping part, since no product pairs two integrated SoCs.+ if shared_cuda_count > 1:+ raw_mib //= shared_cuda_count+ total_mib //= shared_cuda_count+ free_mib = _apply_igpu_host_reserve_mib(raw_mib, shared or integrated)if free_mib < raw_mib:logger.info(- f"ROCm device {idx} is a unified-memory APU sharing system "+ f"{'CUDA' if integrated else 'ROCm'} device {idx} is a "+ f"unified-memory {'SoC' if integrated else 'APU'} sharing system "f"RAM; reserving {raw_mib - free_mib}MiB host headroom "f"({raw_mib}->{free_mib}MiB usable)")- gpus.append((idx, free_mib, 0 if shared else total_bytes // (1024 * 1024)))+ # ROCm publishes 0 because that "total" is system RAM of unknown+ # scope. An integrated part's total IS the pool, so it is a real+ # ceiling; zeroing it would drop the fit to free*frac.+ gpus.append((idx, free_mib, 0 if shared or cgroup_bound else total_mib))# Match the nvidia-smi path's docstring guarantee of sorted-by-id.return sorted(gpus, key = lambda g: g[0])except Exception as e:@@ -12081,27 +12201,32 @@ class LlamaCppBackend:model_size_bytes: int,avail_mib: Optional[int],headroom_mib: int = _HOST_RAM_HEADROOM_MIB,+ *,+ part: str = "APU",) -> Optional[str]:- """On a unified-memory APU, return a user-facing WARNING when the weights+ """On a unified-memory part, return a user-facing WARNING when the weightsdo not fit in available system RAM (else None). Weights only: KV/contextauto-reduce, so counting them too would warn about loads that are fine.None avail (unknown RAM) never warns.Advisory, never a refusal: the load goes ahead and llama.cpp reports whatactually happens rather than Studio pre-empting a failure it predicted.++ ``part`` names the hardware. An integrated CUDA SoC has the same shortfall and+ is not an APU, and the WSL hint cannot apply to a Jetson or a DGX Spark."""if avail_mib is None:return Noneneed_mib = model_size_bytes / (1024 * 1024)if need_mib <= avail_mib - headroom_mib:return None+ free_hint = " (on WSL, raise the memory limit in .wslconfig)" if part == "APU" else ""return (f"This model needs about {need_mib / 1024:.0f} GB but only about "f"{avail_mib / 1024:.0f} GB of memory is available. On a unified-memory "- "APU the weights load into system RAM, so the OS may stop the load. "+ f"{part} the weights load into system RAM, so the OS may stop the load. ""Loading anyway. If it does not complete, use a smaller or more "- "quantized GGUF, or free memory (on WSL, raise the memory limit in "- ".wslconfig)."+ f"quantized GGUF, or free memory{free_hint}.")@staticmethod@@ -12560,6 +12685,7 @@ class LlamaCppBackend:model_size: Optional[int],pinned_bytes: int,avail_mib: Optional[int],+ part: str = "APU",) -> None:"""Re-price the APU RAM advisory once a text-only retry drops a CPU-pinnedvision projector.@@ -12602,7 +12728,7 @@ class LlamaCppBackend:# it would charge ``model_size`` against ``avail - model_size`` and report a# shortfall for a load that demonstrably just started. Same pool, one term# removed, is the only comparison that answers the question being asked.- repriced = self._apu_ram_shortfall_message(model_size, avail_mib)+ repriced = self._apu_ram_shortfall_message(model_size, avail_mib, part = part)if repriced:self._last_load_warning = repriced + suffixelif host_msg:@@ -23670,6 +23796,8 @@ class LlamaCppBackend:_host_ram_msg: Optional[str] = None# The RAM figure those notices were priced against, kept with them._apu_avail_mib: Optional[int] = None+ # ...and which part they describe, for the same reason.+ _apu_ram_part = "APU"# Unified-memory APUs load weights into system RAM (under WSL the VM# cap, not the ROCm-reported VRAM, is the real ceiling); refuse an@@ -23679,19 +23807,35 @@ class LlamaCppBackend:if (model_size is not Noneand not is_vulkan_backend- and self._amd_apu_wants_unified_memory(gpu_indices)+ # An integrated CUDA SoC loads into system RAM like an APU, and+ # _shared_gpu_ids is Vulkan-only, so without this its pool is+ # credited as dedicated VRAM and the spill prices out at zero.+ and (+ self._amd_apu_wants_unified_memory(gpu_indices)+ or (+ self._integrated_cuda_probe_is_free()+ and self._integrated_cuda_selection_is_all_shared(gpu_indices)+ )+ )):# Read ONCE and kept, because the text-only fallback re-prices this# same decision much later, with the weights already resident. A# second live reading there would be the pool MINUS the model the# reprice is asking about, which double-charges it._apu_avail_mib = self._available_system_memory_mib()+ # Kept beside the notice: the text-only fallback rebuilds it later+ # and would otherwise turn a Spark's into an APU's, .wslconfig hint+ # and all.+ _apu_ram_part = (+ "APU" if self._amd_apu_wants_unified_memory(gpu_indices) else "SoC"+ )_ram_msg = self._apu_ram_shortfall_message(# A pinned projector left model_size but not system RAM, and# this guard exists to stop an oversize load being OOM-killed# mid-read, so it has to weigh the projector either way.model_size + _mmproj_pinned_bytes,_apu_avail_mib,+ part = _apu_ram_part,)# gpu_indices is None for a launch nothing pinned, so the guard# priced every visible card, including an APU the gate is about to@@ -27401,6 +27545,7 @@ class LlamaCppBackend:model_size = model_size,pinned_bytes = _mmproj_pinned_bytes,avail_mib = _apu_avail_mib,+ part = _apu_ram_part,)else:# Read the exit code before _kill_process() clears it, sodiff --git a/studio/backend/tests/test_dgx_spark_gguf_context.py b/studio/backend/tests/test_dgx_spark_gguf_context.pynew file mode 100644index 00000000000..ed876cf983b--- /dev/null+++ b/studio/backend/tests/test_dgx_spark_gguf_context.py@@ -0,0 +1,530 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A GGUF context fit on a DGX Spark must not lose the pool to the page cache.++``cudaMemGetInfo``'s free half on an integrated SoC is the kernel's ``MemFree``, which+counts the page cache as used. Measured on a GB10: writing a 60 GiB file took it from+103.25 GiB to 41.52 GiB while ``MemAvailable`` never moved off 115.6 GiB. Downloading or+mmap'ing a GGUF is that same write, so a model the machine can hold gets its context+fitted against the bytes its own weights left in cache, and a 262k-native model comes up+at ``_FIT_MIN_CTX`` (#9889).++Hermetic: torch, nvidia-smi and host memory are stubbed, so these run anywhere.+"""++from __future__ import annotations++import sys+import types++import pytest++from core.inference.llama_cpp import LlamaCppBackend++GIB = 1 << 30+MIB = 1 << 20+# What a DGX Spark actually reports.+SPARK_TOTAL_BYTES = 124609 * MIB+SPARK_TOTAL_GB = round(SPARK_TOTAL_BYTES / GIB, 2)+++class _SparkProps:+ """cudaDeviceProp as torch surfaces it for a GB10."""++ name = "NVIDIA GB10"+ total_memory = SPARK_TOTAL_BYTES+ is_integrated = 1+ gcnArchName = ""+++class _DiscreteProps:+ name = "NVIDIA GB200"+ total_memory = 183 * GIB+ is_integrated = 0+ gcnArchName = ""++[email protected](autouse = True)+def _forget_the_last_machine(monkeypatch):+ """The integrated classification is cached for the life of the process, which is+ right for one machine and wrong for a file that describes several. Each case starts+ with an empty cache so it cannot inherit the hardware the previous one invented."""+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})+++def _spark_torch(driver_free_mib: int, total_mib: int) -> types.ModuleType:+ """torch as it answers on a GB10: mem_get_info's free half is MemFree."""+ module = types.ModuleType("torch")+ module.version = types.SimpleNamespace(hip = None)+ module.cuda = types.SimpleNamespace(+ is_available = lambda: True,+ device_count = lambda: 1,+ mem_get_info = lambda ordinal: (driver_free_mib * MIB, total_mib * MIB),+ get_device_properties = lambda ordinal: _SparkProps(),+ )+ return module+++def _spark_gpu_memory(+ monkeypatch,+ driver_free_mib,+ available_mib,+ total_mib = 124609,+ cgroup_mib = None,+):+ from core.inference.llama_cpp import LlamaCppBackend++ import sys as _sys++ for mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(mask, raising = False)+ monkeypatch.setitem(_sys.modules, "torch", _spark_torch(driver_free_mib, total_mib))+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b: False))+ monkeypatch.setattr(+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "llama-server")+ )+ monkeypatch.setattr(+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: available_mib)+ )+ # The cgroup probe is a SECOND, independent read of the host, so leaving it live+ # would let the machine running the suite decide the answer: under a container with+ # a memory.max below the mocked pool these cases fail while claiming to be hermetic.+ # Stubbed to the value the case is about, None (unconstrained) unless it says.+ monkeypatch.setattr(+ LlamaCppBackend, "_cgroup_available_memory_mib", staticmethod(lambda: cgroup_mib)+ )+ # nvidia-smi answers [N/A] for both memory columns on a Spark, so every row is+ # unparseable and the probe falls through to torch. Absent is the same path.+ monkeypatch.setattr(+ "core.inference.llama_cpp.subprocess.run",+ lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError("no nvidia-smi")),+ )+ return LlamaCppBackend._get_gpu_memory()+++def test_gguf_fit_does_not_lose_the_pool_to_the_page_cache(monkeypatch):+ """The measured case: a 60 GiB download leaves the driver reporting 29.5 GiB free."""+ gpus = _spark_gpu_memory(monkeypatch, driver_free_mib = 29509, available_mib = 118451)++ index, free_mib, total_mib = gpus[0]+ assert index == 0+ # 118451 available, less the 1 GiB integrated host reserve. Was 28485.+ assert free_mib == 118451 - 1024+ # An integrated part keeps its total: unlike a ROCm APU's, it IS the whole pool.+ assert total_mib == 124609+++def test_gguf_fit_keeps_the_host_reserve(monkeypatch):+ """A genuinely full Spark is not talked up, and still gives the OS its margin."""+ gpus = _spark_gpu_memory(monkeypatch, driver_free_mib = 4096, available_mib = 4096)++ assert gpus[0][1] == 4096 - 1024+++def test_gguf_fit_never_exceeds_the_pool(monkeypatch):+ """MemAvailable can exceed a masked or smaller device total; the pool is the cap."""+ gpus = _spark_gpu_memory(+ monkeypatch, driver_free_mib = 1024, available_mib = 200000, total_mib = 124609+ )++ assert gpus[0][1] == 124609 - 1024+++# ── a host that is not one of these parts must be untouched ──────────────────+++def test_discrete_cuda_keeps_the_whole_free_reading(monkeypatch):+ """No host reserve, no MemAvailable credit, and the driver's own total."""+ from core.inference.llama_cpp import LlamaCppBackend++ import sys as _sys++ module = _spark_torch(29509, 81559)+ module.cuda.get_device_properties = lambda ordinal: _DiscreteProps()+ for mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(mask, raising = False)+ monkeypatch.setitem(_sys.modules, "torch", module)+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b: False))+ monkeypatch.setattr(+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "llama-server")+ )+ monkeypatch.setattr(LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 4096))+ monkeypatch.setattr(+ "core.inference.llama_cpp.subprocess.run",+ lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError("no nvidia-smi")),+ )++ # Untouched by both the 1 GiB reserve and the low host figure beside it.+ assert LlamaCppBackend._get_gpu_memory() == [(0, 29509, 81559)]+++def test_gguf_fit_is_bounded_by_an_enforcing_cgroup(monkeypatch):+ """A container's limit is a ceiling, not a floor.++ ``_available_system_memory_mib`` already caps host MemAvailable by the cgroup+ remainder, but reading it only as a lower bound throws that away whenever the+ driver's host-wide MemFree is larger, which is the normal case in a container.+ Host-backed GPU allocations are charged to the cgroup here, so a fit sized above it+ is killed at memory.max.+ """+ gpus = _spark_gpu_memory(+ monkeypatch, driver_free_mib = 102400, available_mib = 16384, cgroup_mib = 16384+ )++ assert gpus[0][1] == 16384 - 1024+++def test_an_unconstrained_host_is_not_capped(monkeypatch):+ """No cgroup limit means no ceiling: the credited pool stands."""+ gpus = _spark_gpu_memory(+ monkeypatch, driver_free_mib = 29509, available_mib = 118451, cgroup_mib = None+ )++ assert gpus[0][1] == 118451 - 1024+++def test_the_unified_preflight_reaches_an_integrated_cuda_soc(monkeypatch):+ """The oversize-load guard was AMD-only, so a Spark's pool read as dedicated VRAM.++ ``_shared_gpu_ids`` is populated for Vulkan alone, so without this the downstream+ guard credits the SoC's reported pool against the weights, finds no spill, and the+ unmapped oversize load that would have been remapped is not.+ """+ from core.inference.llama_cpp import LlamaCppBackend++ import sys as _sys++ monkeypatch.setitem(_sys.modules, "torch", _spark_torch(29509, 124609))+ for mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(mask, raising = False)++ assert LlamaCppBackend._integrated_cuda_unified_memory(None) is True+ assert LlamaCppBackend._integrated_cuda_unified_memory([0]) is True+ # 180 GiB of weights against 118 GiB of pool: the message the preflight now reaches.+ message = LlamaCppBackend._apu_ram_shortfall_message(180 * GIB, 118 * 1024, part = "SoC")+ assert message is not None+ assert "unified-memory SoC" in message+ # A Spark is aarch64 Linux and a Jetson is not a PC: neither runs under WSL.+ assert ".wslconfig" not in message+++def test_the_apu_message_is_unchanged(monkeypatch):+ """The AMD wording and its WSL hint are what they were."""+ from core.inference.llama_cpp import LlamaCppBackend++ message = LlamaCppBackend._apu_ram_shortfall_message(64 * GIB, 46 * 1024)++ assert "unified-memory APU" in message+ assert ".wslconfig" in message+++def test_a_discrete_cuda_host_reaches_no_unified_preflight(monkeypatch):+ from core.inference.llama_cpp import LlamaCppBackend++ import sys as _sys++ module = _spark_torch(29509, 81559)+ module.cuda.get_device_properties = lambda ordinal: _DiscreteProps()+ monkeypatch.setitem(_sys.modules, "torch", module)+ for mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(mask, raising = False)++ assert LlamaCppBackend._integrated_cuda_unified_memory(None) is False+++def test_the_preflight_never_probes_a_device_itself(monkeypatch):+ """An unprobed host answers "not free" rather than paying for a CUDA context.++ _integrated_cuda_gpu_ids calls get_device_properties on every visible card, which+ pins a primary context per device for the life of this process. The preflight runs+ after the VRAM budget was taken, so a probe there can OOM a tightly fitted child+ against a stale budget, on a host whose answer is False regardless. Covers an ARM+ host with discrete cards and an x86 host that initialised CUDA on one GPU only:+ neither is evidence that every per-device probe is already paid for.+ """+ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)++ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False+++def test_the_memory_probe_pays_for_the_classification_up_front(monkeypatch):+ """_get_gpu_memory's torch arm classifies BEFORE it reads any free figure.++ That is the arm a Spark takes: nvidia-smi reports [N/A] for both memory columns+ there, so the CLI probe parses nothing and falls through. Whatever the property+ probe costs is therefore inside the snapshot that follows it, and the preflight+ later reads the answer for nothing.+ """+ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})+ _spark_gpu_memory(monkeypatch, driver_free_mib = 29509, available_mib = 118451)++ assert LlamaCppBackend._integrated_cuda_probe_is_free() is True+ assert LlamaCppBackend._integrated_cuda_unified_memory([0]) is True+++def test_a_different_mask_is_a_different_question(monkeypatch):+ """The cache is keyed by the visibility mask, which decides which devices it is+ about. A cached answer for one mask must not be read as an answer for another."""+ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})+ _spark_gpu_memory(monkeypatch, driver_free_mib = 29509, available_mib = 118451)+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is True++ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False+++def test_a_failed_probe_is_not_remembered(monkeypatch):+ """A torch that raised says nothing about the hardware, so caching its empty answer+ would make the miss permanent for the life of the process."""+ import sys as _sys++ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ broken = types.ModuleType("torch")+ broken.version = types.SimpleNamespace(hip = None)++ def _raise():+ raise RuntimeError("driver not loaded")++ broken.cuda = types.SimpleNamespace(is_available = _raise)+ monkeypatch.setitem(_sys.modules, "torch", broken)++ assert LlamaCppBackend._integrated_cuda_gpu_ids() == set()+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False+++def test_repricing_keeps_the_soc_wording(monkeypatch):+ """A text-only retry must not turn a Spark's notice into a .wslconfig hint.++ The repriced message is rebuilt from scratch after the CPU-pinned projector is+ dropped, so the hardware kind has to travel with it.+ """+ from core.inference.llama_cpp import LlamaCppBackend++ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ original = LlamaCppBackend._apu_ram_shortfall_message(200 * GIB, 118 * 1024, part = "SoC")+ backend._last_load_warning = original++ backend._reprice_after_dropping_pinned_projector(+ apu_msg = original,+ host_msg = None,+ model_size = 180 * GIB,+ pinned_bytes = 20 * GIB,+ avail_mib = 118 * 1024,+ part = "SoC",+ )++ assert backend._last_load_warning is not None+ assert "unified-memory SoC" in backend._last_load_warning+ assert ".wslconfig" not in backend._last_load_warning+++def test_repricing_still_says_apu_for_an_apu():+ """The AMD path keeps the wording and the WSL hint it has always had."""+ from core.inference.llama_cpp import LlamaCppBackend++ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ original = LlamaCppBackend._apu_ram_shortfall_message(64 * GIB, 46 * 1024)+ backend._last_load_warning = original++ backend._reprice_after_dropping_pinned_projector(+ apu_msg = original,+ host_msg = None,+ model_size = 60 * GIB,+ pinned_bytes = 4 * GIB,+ avail_mib = 46 * 1024,+ )++ assert "unified-memory APU" in backend._last_load_warning+ assert ".wslconfig" in backend._last_load_warning+++def test_a_device_that_did_not_answer_is_not_settled(monkeypatch):+ """An incomplete probe must not be remembered as a finished one.++ If one ordinal raised, a later caller reading the answer as settled could retry the+ query that failed; a retry that succeeds initialises that device after the budget+ was taken, which is the allocation this guard exists to keep out of the launch.+ """+ import sys as _sys++ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)++ def _properties(ordinal):+ if ordinal == 1:+ raise RuntimeError("device 1 did not answer")+ return _SparkProps()++ module = _spark_torch(29509, 124609)+ module.cuda.device_count = lambda: 2+ module.cuda.get_device_properties = _properties+ monkeypatch.setitem(_sys.modules, "torch", module)++ # The answer still comes back, so a card that cannot be queried keeps its default.+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == {0}+ # ...but the preflight is told there is nothing free to read.+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False+++def test_a_settled_answer_is_never_probed_again(monkeypatch):+ """Integratedness cannot change under a fixed mask, so one pass is the whole cost."""+ import sys as _sys++ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ calls = []+ module = _spark_torch(29509, 124609)+ original = module.cuda.get_device_properties++ def _counted(ordinal):+ calls.append(ordinal)+ return original(ordinal)++ module.cuda.get_device_properties = _counted+ monkeypatch.setitem(_sys.modules, "torch", module)++ assert LlamaCppBackend._integrated_cuda_gpu_ids() == {0}+ assert len(calls) == 1+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == {0}+ assert LlamaCppBackend._integrated_cuda_unified_memory([0]) is True+ assert len(calls) == 1+++def test_a_cgroup_bound_row_publishes_no_total(monkeypatch):+ """_vram_usable_mib reserves (1 - frac) * total, so a host-wide total against a+ container's free reading reserves memory this process cannot reach: a 16 GiB cgroup+ on a 121 GiB Spark loses most of its budget to a card it does not have. No total is+ what the ROCm shared pool already publishes, and prices the fit off free alone."""+ gpus = _spark_gpu_memory(+ monkeypatch, driver_free_mib = 102400, available_mib = 16384, cgroup_mib = 16384+ )++ index, free_mib, total_mib = gpus[0]+ assert (index, free_mib) == (0, 16384 - 1024)+ assert total_mib == 0+++def test_an_unconstrained_row_keeps_its_total(monkeypatch):+ """The pool is only the container's where a container is what bounds it."""+ gpus = _spark_gpu_memory(monkeypatch, driver_free_mib = 29509, available_mib = 118451)++ assert gpus[0][2] == 124609+++def test_the_preflight_needs_every_credited_device_to_share_the_pool(monkeypatch):+ """The guard charges the WHOLE model against system RAM, and layers placed on a+ discrete card never touch the shared pool. Answering for ANY integrated device in a+ mixed selection calls an otherwise fitting load oversized."""+ import sys as _sys++ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ module = _spark_torch(29509, 124609)+ module.cuda.device_count = lambda: 2+ module.cuda.get_device_properties = (+ lambda ordinal: _SparkProps() if ordinal == 0 else _DiscreteProps()+ )+ monkeypatch.setitem(_sys.modules, "torch", module)++ # Any: true, which is right for the double-count question the tensor-spill guard asks.+ assert LlamaCppBackend._integrated_cuda_unified_memory([0, 1]) is True+ # Every: false, which is what this guard needs.+ assert LlamaCppBackend._integrated_cuda_selection_is_all_shared([0, 1]) is False+ assert LlamaCppBackend._integrated_cuda_selection_is_all_shared([0]) is True+ # Nothing pinned means every visible device, and one of them is discrete.+ assert LlamaCppBackend._integrated_cuda_selection_is_all_shared(None) is False+++def test_an_all_integrated_selection_still_reaches_the_preflight(monkeypatch):+ import sys as _sys++ from core.inference.llama_cpp import LlamaCppBackend++ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ monkeypatch.setitem(_sys.modules, "torch", _spark_torch(29509, 124609))++ assert LlamaCppBackend._integrated_cuda_selection_is_all_shared(None) is True+ assert LlamaCppBackend._integrated_cuda_selection_is_all_shared([0]) is True+++def test_an_equal_cgroup_remainder_is_still_the_ceiling(monkeypatch):+ """_available_system_memory_mib is cgroup-capped, so it hands back exactly the+ remainder whenever MemAvailable exceeds it: equality is the ordinary result on a+ constrained Spark whose driver-free reading is no larger, not a coincidence."""+ gpus = _spark_gpu_memory(+ monkeypatch, driver_free_mib = 8192, available_mib = 16384, cgroup_mib = 16384+ )++ assert gpus[0][1] == 16384 - 1024+ # The host-wide total here would reserve several GiB the container cannot reach.+ assert gpus[0][2] == 0+++# ── one pool, more than one device drawing on it ──+++def _two_integrated_torch(free_mib: int, total_mib: int) -> types.ModuleType:+ """Two devices that both report integrated, sharing ONE host pool."""+ module = types.ModuleType("torch")+ module.version = types.SimpleNamespace(hip = None)+ module.cuda = types.SimpleNamespace(+ is_available = lambda: True,+ device_count = lambda: 2,+ mem_get_info = lambda ordinal: (free_mib * MIB, total_mib * MIB),+ get_device_properties = lambda ordinal: _SparkProps(),+ )+ return module+++def test_two_integrated_devices_do_not_each_claim_the_whole_pool(monkeypatch):+ # Every figure the integrated arm computes -- MemAvailable, the cgroup+ # remainder, the pool total -- describes the WHOLE host. Handing it to each+ # device in turn lets a caller that sums across cards commit the same bytes+ # twice, and the fit then sizes a load against memory that does not exist.+ # No shipping product pairs two integrated SoCs, so this is a guard rather+ # than a reproduction, and it costs a division by 1 everywhere else.+ for mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(mask, raising = False)+ monkeypatch.setitem(sys.modules, "torch", _two_integrated_torch(1590, 124609))+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b: False))+ monkeypatch.setattr(+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "llama-server")+ )+ monkeypatch.setattr(+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 61850)+ )+ monkeypatch.setattr(LlamaCppBackend, "_cgroup_available_memory_mib", staticmethod(lambda: None))+ monkeypatch.setattr(+ "core.inference.llama_cpp.subprocess.run",+ lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError("no nvidia-smi")),+ )+ gpus = LlamaCppBackend._get_gpu_memory()++ assert len(gpus) == 2+ # The pool is offered once, not once per device.+ assert sum(free for _idx, free, _total in gpus) <= 61850+ assert sum(total for _idx, _free, total in gpus) <= 124609+++def test_a_single_integrated_device_is_not_divided(monkeypatch):+ # The guard above must be invisible on every machine that actually exists.+ gpus = _spark_gpu_memory(monkeypatch, driver_free_mib = 1590, available_mib = 61850)+ assert gpus == [(0, 61850 - 1024, 124609)]