Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: do not lose a DGX Spark's memory pool to the page cache when fitting context (#10704)

Daniel Han61eefe60cdbe
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 0aa089ccbca..ea86b1e9575 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -9594,6 +9594,32 @@ class LlamaCppBackend:
except Exception:
return False
+ # PHYSICAL integrated ids per visibility mask. Integratedness cannot change under
+ # a fixed mask, and the mask decides which devices the answer is about.
+ _INTEGRATED_CUDA_IDS: dict[tuple, set[int]] = {}
+
+ @staticmethod
+ def _integrated_cuda_mask_key() -> tuple:
+ return tuple(
+ os.environ.get(name)
+ for name in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES")
+ )
+
+ @staticmethod
+ def _integrated_cuda_probe_is_free() -> bool:
+ """True when ``_integrated_cuda_gpu_ids()`` costs no NEW CUDA context.
+
+ That probe pins a ~700 MiB primary context per visible card for the life of
+ this process, and the launch preflight runs AFTER the VRAM budget was taken, so
+ probing there can OOM a tightly fitted child against a stale budget. So the
+ preflight never probes: it reads an answer only if one is cached for this mask.
+ ``_get_gpu_memory`` fills that cache on its torch arm before reading any
+ free/total figure, which puts the cost inside the snapshot. That is the arm an
+ integrated SoC takes anyway (nvidia-smi answers ``[N/A]`` on a Spark, so the CLI
+ probe parses nothing); a host whose nvidia-smi answers never pays for it.
+ """
+ return LlamaCppBackend._integrated_cuda_mask_key() in LlamaCppBackend._INTEGRATED_CUDA_IDS
+
@staticmethod
def _integrated_cuda_gpu_ids() -> set[int]:
"""PHYSICAL ids of visible CUDA GPUs whose "VRAM" is shared system RAM.
@@ -9604,6 +9630,12 @@ class LlamaCppBackend:
``torch.cuda.*`` and ``_rocm_unified_memory_gpu_ids`` already answers for it.
Empty off CUDA and on error, so every caller keeps its discrete-GPU default.
"""
+ cached = LlamaCppBackend._INTEGRATED_CUDA_IDS.get(
+ LlamaCppBackend._integrated_cuda_mask_key()
+ )
+ if cached is not None:
+ # Settled: a second pass over the devices could learn nothing.
+ return set(cached)
try:
import torch
@@ -9615,10 +9647,15 @@ class LlamaCppBackend:
# (CUDA_VISIBLE_DEVICES=2) does not answer for the wrong card.
physical_ids = LlamaCppBackend._resolve_visible_physical_ids()
integrated: set[int] = set()
+ # A device that did not answer leaves this incomplete: still returned, so
+ # an unqueryable card keeps its discrete default, but not remembered, since
+ # a caller reading it as settled would retry the query that failed.
+ complete = True
for ordinal in range(torch.cuda.device_count()):
try:
props = torch.cuda.get_device_properties(ordinal)
except Exception:
+ complete = False
continue
# Both spellings: the attribute was `integrated` before torch renamed
# it, and an old wheel exposing neither reads discrete.
@@ -9631,6 +9668,12 @@ class LlamaCppBackend:
if physical_ids is not None and ordinal < len(physical_ids)
else ordinal
)
+ # Cached for the preflight, but only a pass that reached every device: a
+ # torch that raised says nothing about the hardware.
+ if complete:
+ LlamaCppBackend._INTEGRATED_CUDA_IDS[
+ LlamaCppBackend._integrated_cuda_mask_key()
+ ] = integrated
return integrated
except Exception:
return set()
@@ -9652,6 +9695,31 @@ class LlamaCppBackend:
except Exception:
return False
+ @staticmethod
+ def _integrated_cuda_selection_is_all_shared(gpu_indices = None) -> bool:
+ """True when EVERY credited CUDA device is an integrated SoC.
+
+ Stricter than ``_integrated_cuda_unified_memory``, which answers for ANY, and
+ that difference matters here: this guard charges the WHOLE model against system
+ RAM, and layers placed on a discrete card in a mixed selection never touch the
+ shared pool. Answering for any would call an otherwise fitting load oversized,
+ warn about it, and override an explicitly unmapped load to use mmap.
+ """
+ try:
+ integrated = LlamaCppBackend._integrated_cuda_gpu_ids()
+ if not integrated:
+ return False
+ if gpu_indices is not None:
+ return bool(gpu_indices) and all(_i in integrated for _i in gpu_indices)
+ # Nothing pinned, so the credited set is every visible device.
+ visible = LlamaCppBackend._resolve_visible_physical_ids()
+ if visible is None:
+ import torch
+ visible = list(range(torch.cuda.device_count()))
+ return bool(visible) and all(_i in integrated for _i in visible)
+ except Exception:
+ return False
+
# "Off" spellings ggml itself ignores (it tests presence); we honour them.
_UNIFIED_MEMORY_OFF = frozenset({"", "0", "false", "no", "off"})
@@ -11428,8 +11496,15 @@ class LlamaCppBackend:
# Shared-pool APU: same as the Vulkan iGPU path. Hold back the host
# margin, and report total 0 since that "total" is system RAM.
unified_ids = LlamaCppBackend._rocm_unified_memory_gpu_ids()
+ # An integrated CUDA SoC shares one pool too, and its free reading is wrong
+ # in the OPPOSITE direction to ROCm's -- see below.
+ integrated_ids = LlamaCppBackend._integrated_cuda_gpu_ids()
# Same #7624 arch gate the amd-smi branch applies, from the one helper.
arch_keeps = LlamaCppBackend._rocm_arch_gate_keep(binary, torch, for_llama_server)
+ # How many devices the shared pool is about to be divided between. Read
+ # once, from the same set the loop tests, so the count and the per-device
+ # decision cannot disagree and produce a division by zero.
+ shared_cuda_count = sum(1 for _i in integrated_ids if arch_keeps(_i))
gpus = []
# Windows ROCm's free reading is an over-report on discrete cards too,
# not only on the shared pool handled below (#8403). It is capped
@@ -11447,7 +11522,10 @@ class LlamaCppBackend:
if not arch_keeps(idx):
continue
shared = idx in unified_ids
+ integrated = idx in integrated_ids
+ cgroup_bound = False
raw_mib = free_bytes // (1024 * 1024)
+ total_mib = total_bytes // (1024 * 1024)
if shared:
# ROCm's free is unreliable on a shared pool (Windows HIP
# reports free==total, #7072), and system RAM is the real
@@ -11455,14 +11533,56 @@ class LlamaCppBackend:
avail = LlamaCppBackend._available_system_memory_mib()
if avail is not None:
raw_mib = min(raw_mib, avail)
- free_mib = _apply_igpu_host_reserve_mib(raw_mib, shared)
+ elif integrated:
+ # cudaMemGetInfo's free half here is MemFree, which counts the
+ # page cache as used, so a GGUF's own download or mmap collapses it
+ # and the context is fitted against its own cached bytes (#9889).
+ # MemAvailable prices that reclaimable cache back in.
+ avail = LlamaCppBackend._available_system_memory_mib()
+ # A zero total is a probe that could not size the pool, not a pool
+ # of zero: capping against it would take the device to nothing.
+ if avail is not None and total_mib > 0:
+ raw_mib = min(total_mib, max(raw_mib, avail))
+ # Again as a CEILING: `avail` is cgroup-capped already, but as a
+ # lower bound that is thrown away whenever host-wide MemFree is
+ # larger, the normal case in a container. Allocations here are
+ # charged to the cgroup, so an oversized fit dies at memory.max.
+ # `<=` below, not `<`: _available_system_memory_mib is itself
+ # cgroup-capped, so it hands back exactly the remainder whenever
+ # MemAvailable exceeds it, which makes equality the ordinary result
+ # on a constrained Spark rather than a coincidence.
+ cgroup_mib = LlamaCppBackend._cgroup_available_memory_mib()
+ if cgroup_mib is not None and cgroup_mib <= raw_mib:
+ raw_mib = cgroup_mib
+ # ...and then the pool is the container's, not the device's.
+ # _vram_usable_mib takes its occupancy reserve as
+ # (1 - frac) * total, so a host-wide total against a container's
+ # free reading reserves memory this process cannot reach: a
+ # 16 GiB cgroup on a 121 GiB Spark loses most of its budget to
+ # a card it does not have. Publishing no total is the same
+ # answer the ROCm shared pool gives, and prices the fit off the
+ # free reading, which IS the container's ceiling.
+ cgroup_bound = True
+ # One pool, however many integrated devices draw on it. Every
+ # figure above is a whole-pool figure, so handing it to each
+ # device in turn lets a caller that sums across cards commit the
+ # same bytes twice. Share it instead. Division by 1 on every
+ # shipping part, since no product pairs two integrated SoCs.
+ if shared_cuda_count > 1:
+ raw_mib //= shared_cuda_count
+ total_mib //= shared_cuda_count
+ free_mib = _apply_igpu_host_reserve_mib(raw_mib, shared or integrated)
if free_mib < raw_mib:
logger.info(
- f"ROCm device {idx} is a unified-memory APU sharing system "
+ f"{'CUDA' if integrated else 'ROCm'} device {idx} is a "
+ f"unified-memory {'SoC' if integrated else 'APU'} sharing system "
f"RAM; reserving {raw_mib - free_mib}MiB host headroom "
f"({raw_mib}->{free_mib}MiB usable)"
)
- gpus.append((idx, free_mib, 0 if shared else total_bytes // (1024 * 1024)))
+ # ROCm publishes 0 because that "total" is system RAM of unknown
+ # scope. An integrated part's total IS the pool, so it is a real
+ # ceiling; zeroing it would drop the fit to free*frac.
+ gpus.append((idx, free_mib, 0 if shared or cgroup_bound else total_mib))
# Match the nvidia-smi path's docstring guarantee of sorted-by-id.
return sorted(gpus, key = lambda g: g[0])
except Exception as e:
@@ -12081,27 +12201,32 @@ class LlamaCppBackend:
model_size_bytes: int,
avail_mib: Optional[int],
headroom_mib: int = _HOST_RAM_HEADROOM_MIB,
+ *,
+ part: str = "APU",
) -> Optional[str]:
- """On a unified-memory APU, return a user-facing WARNING when the weights
+ """On a unified-memory part, return a user-facing WARNING when the weights
do not fit in available system RAM (else None). Weights only: KV/context
auto-reduce, so counting them too would warn about loads that are fine.
None avail (unknown RAM) never warns.
Advisory, never a refusal: the load goes ahead and llama.cpp reports what
actually happens rather than Studio pre-empting a failure it predicted.
+
+ ``part`` names the hardware. An integrated CUDA SoC has the same shortfall and
+ is not an APU, and the WSL hint cannot apply to a Jetson or a DGX Spark.
"""
if avail_mib is None:
return None
need_mib = model_size_bytes / (1024 * 1024)
if need_mib <= avail_mib - headroom_mib:
return None
+ free_hint = " (on WSL, raise the memory limit in .wslconfig)" if part == "APU" else ""
return (
f"This model needs about {need_mib / 1024:.0f} GB but only about "
f"{avail_mib / 1024:.0f} GB of memory is available. On a unified-memory "
- "APU the weights load into system RAM, so the OS may stop the load. "
+ f"{part} the weights load into system RAM, so the OS may stop the load. "
"Loading anyway. If it does not complete, use a smaller or more "
- "quantized GGUF, or free memory (on WSL, raise the memory limit in "
- ".wslconfig)."
+ f"quantized GGUF, or free memory{free_hint}."
)
@staticmethod
@@ -12560,6 +12685,7 @@ class LlamaCppBackend:
model_size: Optional[int],
pinned_bytes: int,
avail_mib: Optional[int],
+ part: str = "APU",
) -> None:
"""Re-price the APU RAM advisory once a text-only retry drops a CPU-pinned
vision projector.
@@ -12602,7 +12728,7 @@ class LlamaCppBackend:
# it would charge ``model_size`` against ``avail - model_size`` and report a
# shortfall for a load that demonstrably just started. Same pool, one term
# removed, is the only comparison that answers the question being asked.
- repriced = self._apu_ram_shortfall_message(model_size, avail_mib)
+ repriced = self._apu_ram_shortfall_message(model_size, avail_mib, part = part)
if repriced:
self._last_load_warning = repriced + suffix
elif host_msg:
@@ -23670,6 +23796,8 @@ class LlamaCppBackend:
_host_ram_msg: Optional[str] = None
# The RAM figure those notices were priced against, kept with them.
_apu_avail_mib: Optional[int] = None
+ # ...and which part they describe, for the same reason.
+ _apu_ram_part = "APU"
# Unified-memory APUs load weights into system RAM (under WSL the VM
# cap, not the ROCm-reported VRAM, is the real ceiling); refuse an
@@ -23679,19 +23807,35 @@ class LlamaCppBackend:
if (
model_size is not None
and not is_vulkan_backend
- and self._amd_apu_wants_unified_memory(gpu_indices)
+ # An integrated CUDA SoC loads into system RAM like an APU, and
+ # _shared_gpu_ids is Vulkan-only, so without this its pool is
+ # credited as dedicated VRAM and the spill prices out at zero.
+ and (
+ self._amd_apu_wants_unified_memory(gpu_indices)
+ or (
+ self._integrated_cuda_probe_is_free()
+ and self._integrated_cuda_selection_is_all_shared(gpu_indices)
+ )
+ )
):
# Read ONCE and kept, because the text-only fallback re-prices this
# same decision much later, with the weights already resident. A
# second live reading there would be the pool MINUS the model the
# reprice is asking about, which double-charges it.
_apu_avail_mib = self._available_system_memory_mib()
+ # Kept beside the notice: the text-only fallback rebuilds it later
+ # and would otherwise turn a Spark's into an APU's, .wslconfig hint
+ # and all.
+ _apu_ram_part = (
+ "APU" if self._amd_apu_wants_unified_memory(gpu_indices) else "SoC"
+ )
_ram_msg = self._apu_ram_shortfall_message(
# A pinned projector left model_size but not system RAM, and
# this guard exists to stop an oversize load being OOM-killed
# mid-read, so it has to weigh the projector either way.
model_size + _mmproj_pinned_bytes,
_apu_avail_mib,
+ part = _apu_ram_part,
)
# gpu_indices is None for a launch nothing pinned, so the guard
# priced every visible card, including an APU the gate is about to
@@ -27401,6 +27545,7 @@ class LlamaCppBackend:
model_size = model_size,
pinned_bytes = _mmproj_pinned_bytes,
avail_mib = _apu_avail_mib,
+ part = _apu_ram_part,
)
else:
# Read the exit code before _kill_process() clears it, so
diff --git a/studio/backend/tests/test_dgx_spark_gguf_context.py b/studio/backend/tests/test_dgx_spark_gguf_context.py
new file mode 100644
index 00000000000..ed876cf983b
--- /dev/null
+++ b/studio/backend/tests/test_dgx_spark_gguf_context.py
@@ -0,0 +1,530 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A GGUF context fit on a DGX Spark must not lose the pool to the page cache.
+
+``cudaMemGetInfo``'s free half on an integrated SoC is the kernel's ``MemFree``, which
+counts the page cache as used. Measured on a GB10: writing a 60 GiB file took it from
+103.25 GiB to 41.52 GiB while ``MemAvailable`` never moved off 115.6 GiB. Downloading or
+mmap'ing a GGUF is that same write, so a model the machine can hold gets its context
+fitted against the bytes its own weights left in cache, and a 262k-native model comes up
+at ``_FIT_MIN_CTX`` (#9889).
+
+Hermetic: torch, nvidia-smi and host memory are stubbed, so these run anywhere.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import pytest
+
+from core.inference.llama_cpp import LlamaCppBackend
+
+GIB = 1 << 30
+MIB = 1 << 20
+# What a DGX Spark actually reports.
+SPARK_TOTAL_BYTES = 124609 * MIB
+SPARK_TOTAL_GB = round(SPARK_TOTAL_BYTES / GIB, 2)
+
+
+class _SparkProps:
+ """cudaDeviceProp as torch surfaces it for a GB10."""
+
+ name = "NVIDIA GB10"
+ total_memory = SPARK_TOTAL_BYTES
+ is_integrated = 1
+ gcnArchName = ""
+
+
+class _DiscreteProps:
+ name = "NVIDIA GB200"
+ total_memory = 183 * GIB
+ is_integrated = 0
+ gcnArchName = ""
+
+
[email protected](autouse = True)
+def _forget_the_last_machine(monkeypatch):
+ """The integrated classification is cached for the life of the process, which is
+ right for one machine and wrong for a file that describes several. Each case starts
+ with an empty cache so it cannot inherit the hardware the previous one invented."""
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+
+
+def _spark_torch(driver_free_mib: int, total_mib: int) -> types.ModuleType:
+ """torch as it answers on a GB10: mem_get_info's free half is MemFree."""
+ module = types.ModuleType("torch")
+ module.version = types.SimpleNamespace(hip = None)
+ module.cuda = types.SimpleNamespace(
+ is_available = lambda: True,
+ device_count = lambda: 1,
+ mem_get_info = lambda ordinal: (driver_free_mib * MIB, total_mib * MIB),
+ get_device_properties = lambda ordinal: _SparkProps(),
+ )
+ return module
+
+
+def _spark_gpu_memory(
+ monkeypatch,
+ driver_free_mib,
+ available_mib,
+ total_mib = 124609,
+ cgroup_mib = None,
+):
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ import sys as _sys
+
+ for mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(mask, raising = False)
+ monkeypatch.setitem(_sys.modules, "torch", _spark_torch(driver_free_mib, total_mib))
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b: False))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "llama-server")
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: available_mib)
+ )
+ # The cgroup probe is a SECOND, independent read of the host, so leaving it live
+ # would let the machine running the suite decide the answer: under a container with
+ # a memory.max below the mocked pool these cases fail while claiming to be hermetic.
+ # Stubbed to the value the case is about, None (unconstrained) unless it says.
+ monkeypatch.setattr(
+ LlamaCppBackend, "_cgroup_available_memory_mib", staticmethod(lambda: cgroup_mib)
+ )
+ # nvidia-smi answers [N/A] for both memory columns on a Spark, so every row is
+ # unparseable and the probe falls through to torch. Absent is the same path.
+ monkeypatch.setattr(
+ "core.inference.llama_cpp.subprocess.run",
+ lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError("no nvidia-smi")),
+ )
+ return LlamaCppBackend._get_gpu_memory()
+
+
+def test_gguf_fit_does_not_lose_the_pool_to_the_page_cache(monkeypatch):
+ """The measured case: a 60 GiB download leaves the driver reporting 29.5 GiB free."""
+ gpus = _spark_gpu_memory(monkeypatch, driver_free_mib = 29509, available_mib = 118451)
+
+ index, free_mib, total_mib = gpus[0]
+ assert index == 0
+ # 118451 available, less the 1 GiB integrated host reserve. Was 28485.
+ assert free_mib == 118451 - 1024
+ # An integrated part keeps its total: unlike a ROCm APU's, it IS the whole pool.
+ assert total_mib == 124609
+
+
+def test_gguf_fit_keeps_the_host_reserve(monkeypatch):
+ """A genuinely full Spark is not talked up, and still gives the OS its margin."""
+ gpus = _spark_gpu_memory(monkeypatch, driver_free_mib = 4096, available_mib = 4096)
+
+ assert gpus[0][1] == 4096 - 1024
+
+
+def test_gguf_fit_never_exceeds_the_pool(monkeypatch):
+ """MemAvailable can exceed a masked or smaller device total; the pool is the cap."""
+ gpus = _spark_gpu_memory(
+ monkeypatch, driver_free_mib = 1024, available_mib = 200000, total_mib = 124609
+ )
+
+ assert gpus[0][1] == 124609 - 1024
+
+
+# ── a host that is not one of these parts must be untouched ──────────────────
+
+
+def test_discrete_cuda_keeps_the_whole_free_reading(monkeypatch):
+ """No host reserve, no MemAvailable credit, and the driver's own total."""
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ import sys as _sys
+
+ module = _spark_torch(29509, 81559)
+ module.cuda.get_device_properties = lambda ordinal: _DiscreteProps()
+ for mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(mask, raising = False)
+ monkeypatch.setitem(_sys.modules, "torch", module)
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b: False))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "llama-server")
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 4096))
+ monkeypatch.setattr(
+ "core.inference.llama_cpp.subprocess.run",
+ lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError("no nvidia-smi")),
+ )
+
+ # Untouched by both the 1 GiB reserve and the low host figure beside it.
+ assert LlamaCppBackend._get_gpu_memory() == [(0, 29509, 81559)]
+
+
+def test_gguf_fit_is_bounded_by_an_enforcing_cgroup(monkeypatch):
+ """A container's limit is a ceiling, not a floor.
+
+ ``_available_system_memory_mib`` already caps host MemAvailable by the cgroup
+ remainder, but reading it only as a lower bound throws that away whenever the
+ driver's host-wide MemFree is larger, which is the normal case in a container.
+ Host-backed GPU allocations are charged to the cgroup here, so a fit sized above it
+ is killed at memory.max.
+ """
+ gpus = _spark_gpu_memory(
+ monkeypatch, driver_free_mib = 102400, available_mib = 16384, cgroup_mib = 16384
+ )
+
+ assert gpus[0][1] == 16384 - 1024
+
+
+def test_an_unconstrained_host_is_not_capped(monkeypatch):
+ """No cgroup limit means no ceiling: the credited pool stands."""
+ gpus = _spark_gpu_memory(
+ monkeypatch, driver_free_mib = 29509, available_mib = 118451, cgroup_mib = None
+ )
+
+ assert gpus[0][1] == 118451 - 1024
+
+
+def test_the_unified_preflight_reaches_an_integrated_cuda_soc(monkeypatch):
+ """The oversize-load guard was AMD-only, so a Spark's pool read as dedicated VRAM.
+
+ ``_shared_gpu_ids`` is populated for Vulkan alone, so without this the downstream
+ guard credits the SoC's reported pool against the weights, finds no spill, and the
+ unmapped oversize load that would have been remapped is not.
+ """
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ import sys as _sys
+
+ monkeypatch.setitem(_sys.modules, "torch", _spark_torch(29509, 124609))
+ for mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(mask, raising = False)
+
+ assert LlamaCppBackend._integrated_cuda_unified_memory(None) is True
+ assert LlamaCppBackend._integrated_cuda_unified_memory([0]) is True
+ # 180 GiB of weights against 118 GiB of pool: the message the preflight now reaches.
+ message = LlamaCppBackend._apu_ram_shortfall_message(180 * GIB, 118 * 1024, part = "SoC")
+ assert message is not None
+ assert "unified-memory SoC" in message
+ # A Spark is aarch64 Linux and a Jetson is not a PC: neither runs under WSL.
+ assert ".wslconfig" not in message
+
+
+def test_the_apu_message_is_unchanged(monkeypatch):
+ """The AMD wording and its WSL hint are what they were."""
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ message = LlamaCppBackend._apu_ram_shortfall_message(64 * GIB, 46 * 1024)
+
+ assert "unified-memory APU" in message
+ assert ".wslconfig" in message
+
+
+def test_a_discrete_cuda_host_reaches_no_unified_preflight(monkeypatch):
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ import sys as _sys
+
+ module = _spark_torch(29509, 81559)
+ module.cuda.get_device_properties = lambda ordinal: _DiscreteProps()
+ monkeypatch.setitem(_sys.modules, "torch", module)
+ for mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(mask, raising = False)
+
+ assert LlamaCppBackend._integrated_cuda_unified_memory(None) is False
+
+
+def test_the_preflight_never_probes_a_device_itself(monkeypatch):
+ """An unprobed host answers "not free" rather than paying for a CUDA context.
+
+ _integrated_cuda_gpu_ids calls get_device_properties on every visible card, which
+ pins a primary context per device for the life of this process. The preflight runs
+ after the VRAM budget was taken, so a probe there can OOM a tightly fitted child
+ against a stale budget, on a host whose answer is False regardless. Covers an ARM
+ host with discrete cards and an x86 host that initialised CUDA on one GPU only:
+ neither is evidence that every per-device probe is already paid for.
+ """
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+
+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False
+
+
+def test_the_memory_probe_pays_for_the_classification_up_front(monkeypatch):
+ """_get_gpu_memory's torch arm classifies BEFORE it reads any free figure.
+
+ That is the arm a Spark takes: nvidia-smi reports [N/A] for both memory columns
+ there, so the CLI probe parses nothing and falls through. Whatever the property
+ probe costs is therefore inside the snapshot that follows it, and the preflight
+ later reads the answer for nothing.
+ """
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+ _spark_gpu_memory(monkeypatch, driver_free_mib = 29509, available_mib = 118451)
+
+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is True
+ assert LlamaCppBackend._integrated_cuda_unified_memory([0]) is True
+
+
+def test_a_different_mask_is_a_different_question(monkeypatch):
+ """The cache is keyed by the visibility mask, which decides which devices it is
+ about. A cached answer for one mask must not be read as an answer for another."""
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+ _spark_gpu_memory(monkeypatch, driver_free_mib = 29509, available_mib = 118451)
+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is True
+
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "1")
+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False
+
+
+def test_a_failed_probe_is_not_remembered(monkeypatch):
+ """A torch that raised says nothing about the hardware, so caching its empty answer
+ would make the miss permanent for the life of the process."""
+ import sys as _sys
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ broken = types.ModuleType("torch")
+ broken.version = types.SimpleNamespace(hip = None)
+
+ def _raise():
+ raise RuntimeError("driver not loaded")
+
+ broken.cuda = types.SimpleNamespace(is_available = _raise)
+ monkeypatch.setitem(_sys.modules, "torch", broken)
+
+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == set()
+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False
+
+
+def test_repricing_keeps_the_soc_wording(monkeypatch):
+ """A text-only retry must not turn a Spark's notice into a .wslconfig hint.
+
+ The repriced message is rebuilt from scratch after the CPU-pinned projector is
+ dropped, so the hardware kind has to travel with it.
+ """
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ original = LlamaCppBackend._apu_ram_shortfall_message(200 * GIB, 118 * 1024, part = "SoC")
+ backend._last_load_warning = original
+
+ backend._reprice_after_dropping_pinned_projector(
+ apu_msg = original,
+ host_msg = None,
+ model_size = 180 * GIB,
+ pinned_bytes = 20 * GIB,
+ avail_mib = 118 * 1024,
+ part = "SoC",
+ )
+
+ assert backend._last_load_warning is not None
+ assert "unified-memory SoC" in backend._last_load_warning
+ assert ".wslconfig" not in backend._last_load_warning
+
+
+def test_repricing_still_says_apu_for_an_apu():
+ """The AMD path keeps the wording and the WSL hint it has always had."""
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ original = LlamaCppBackend._apu_ram_shortfall_message(64 * GIB, 46 * 1024)
+ backend._last_load_warning = original
+
+ backend._reprice_after_dropping_pinned_projector(
+ apu_msg = original,
+ host_msg = None,
+ model_size = 60 * GIB,
+ pinned_bytes = 4 * GIB,
+ avail_mib = 46 * 1024,
+ )
+
+ assert "unified-memory APU" in backend._last_load_warning
+ assert ".wslconfig" in backend._last_load_warning
+
+
+def test_a_device_that_did_not_answer_is_not_settled(monkeypatch):
+ """An incomplete probe must not be remembered as a finished one.
+
+ If one ordinal raised, a later caller reading the answer as settled could retry the
+ query that failed; a retry that succeeds initialises that device after the budget
+ was taken, which is the allocation this guard exists to keep out of the launch.
+ """
+ import sys as _sys
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+
+ def _properties(ordinal):
+ if ordinal == 1:
+ raise RuntimeError("device 1 did not answer")
+ return _SparkProps()
+
+ module = _spark_torch(29509, 124609)
+ module.cuda.device_count = lambda: 2
+ module.cuda.get_device_properties = _properties
+ monkeypatch.setitem(_sys.modules, "torch", module)
+
+ # The answer still comes back, so a card that cannot be queried keeps its default.
+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == {0}
+ # ...but the preflight is told there is nothing free to read.
+ assert LlamaCppBackend._integrated_cuda_probe_is_free() is False
+
+
+def test_a_settled_answer_is_never_probed_again(monkeypatch):
+ """Integratedness cannot change under a fixed mask, so one pass is the whole cost."""
+ import sys as _sys
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ calls = []
+ module = _spark_torch(29509, 124609)
+ original = module.cuda.get_device_properties
+
+ def _counted(ordinal):
+ calls.append(ordinal)
+ return original(ordinal)
+
+ module.cuda.get_device_properties = _counted
+ monkeypatch.setitem(_sys.modules, "torch", module)
+
+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == {0}
+ assert len(calls) == 1
+ assert LlamaCppBackend._integrated_cuda_gpu_ids() == {0}
+ assert LlamaCppBackend._integrated_cuda_unified_memory([0]) is True
+ assert len(calls) == 1
+
+
+def test_a_cgroup_bound_row_publishes_no_total(monkeypatch):
+ """_vram_usable_mib reserves (1 - frac) * total, so a host-wide total against a
+ container's free reading reserves memory this process cannot reach: a 16 GiB cgroup
+ on a 121 GiB Spark loses most of its budget to a card it does not have. No total is
+ what the ROCm shared pool already publishes, and prices the fit off free alone."""
+ gpus = _spark_gpu_memory(
+ monkeypatch, driver_free_mib = 102400, available_mib = 16384, cgroup_mib = 16384
+ )
+
+ index, free_mib, total_mib = gpus[0]
+ assert (index, free_mib) == (0, 16384 - 1024)
+ assert total_mib == 0
+
+
+def test_an_unconstrained_row_keeps_its_total(monkeypatch):
+ """The pool is only the container's where a container is what bounds it."""
+ gpus = _spark_gpu_memory(monkeypatch, driver_free_mib = 29509, available_mib = 118451)
+
+ assert gpus[0][2] == 124609
+
+
+def test_the_preflight_needs_every_credited_device_to_share_the_pool(monkeypatch):
+ """The guard charges the WHOLE model against system RAM, and layers placed on a
+ discrete card never touch the shared pool. Answering for ANY integrated device in a
+ mixed selection calls an otherwise fitting load oversized."""
+ import sys as _sys
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ module = _spark_torch(29509, 124609)
+ module.cuda.device_count = lambda: 2
+ module.cuda.get_device_properties = (
+ lambda ordinal: _SparkProps() if ordinal == 0 else _DiscreteProps()
+ )
+ monkeypatch.setitem(_sys.modules, "torch", module)
+
+ # Any: true, which is right for the double-count question the tensor-spill guard asks.
+ assert LlamaCppBackend._integrated_cuda_unified_memory([0, 1]) is True
+ # Every: false, which is what this guard needs.
+ assert LlamaCppBackend._integrated_cuda_selection_is_all_shared([0, 1]) is False
+ assert LlamaCppBackend._integrated_cuda_selection_is_all_shared([0]) is True
+ # Nothing pinned means every visible device, and one of them is discrete.
+ assert LlamaCppBackend._integrated_cuda_selection_is_all_shared(None) is False
+
+
+def test_an_all_integrated_selection_still_reaches_the_preflight(monkeypatch):
+ import sys as _sys
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ monkeypatch.setitem(_sys.modules, "torch", _spark_torch(29509, 124609))
+
+ assert LlamaCppBackend._integrated_cuda_selection_is_all_shared(None) is True
+ assert LlamaCppBackend._integrated_cuda_selection_is_all_shared([0]) is True
+
+
+def test_an_equal_cgroup_remainder_is_still_the_ceiling(monkeypatch):
+ """_available_system_memory_mib is cgroup-capped, so it hands back exactly the
+ remainder whenever MemAvailable exceeds it: equality is the ordinary result on a
+ constrained Spark whose driver-free reading is no larger, not a coincidence."""
+ gpus = _spark_gpu_memory(
+ monkeypatch, driver_free_mib = 8192, available_mib = 16384, cgroup_mib = 16384
+ )
+
+ assert gpus[0][1] == 16384 - 1024
+ # The host-wide total here would reserve several GiB the container cannot reach.
+ assert gpus[0][2] == 0
+
+
+# ── one pool, more than one device drawing on it ──
+
+
+def _two_integrated_torch(free_mib: int, total_mib: int) -> types.ModuleType:
+ """Two devices that both report integrated, sharing ONE host pool."""
+ module = types.ModuleType("torch")
+ module.version = types.SimpleNamespace(hip = None)
+ module.cuda = types.SimpleNamespace(
+ is_available = lambda: True,
+ device_count = lambda: 2,
+ mem_get_info = lambda ordinal: (free_mib * MIB, total_mib * MIB),
+ get_device_properties = lambda ordinal: _SparkProps(),
+ )
+ return module
+
+
+def test_two_integrated_devices_do_not_each_claim_the_whole_pool(monkeypatch):
+ # Every figure the integrated arm computes -- MemAvailable, the cgroup
+ # remainder, the pool total -- describes the WHOLE host. Handing it to each
+ # device in turn lets a caller that sums across cards commit the same bytes
+ # twice, and the fit then sizes a load against memory that does not exist.
+ # No shipping product pairs two integrated SoCs, so this is a guard rather
+ # than a reproduction, and it costs a division by 1 everywhere else.
+ for mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(mask, raising = False)
+ monkeypatch.setitem(sys.modules, "torch", _two_integrated_torch(1590, 124609))
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b: False))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "llama-server")
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 61850)
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_cgroup_available_memory_mib", staticmethod(lambda: None))
+ monkeypatch.setattr(
+ "core.inference.llama_cpp.subprocess.run",
+ lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError("no nvidia-smi")),
+ )
+ gpus = LlamaCppBackend._get_gpu_memory()
+
+ assert len(gpus) == 2
+ # The pool is offered once, not once per device.
+ assert sum(free for _idx, free, _total in gpus) <= 61850
+ assert sum(total for _idx, _free, total in gpus) <= 124609
+
+
+def test_a_single_integrated_device_is_not_divided(monkeypatch):
+ # The guard above must be invisible on every machine that actually exists.
+ gpus = _spark_gpu_memory(monkeypatch, driver_free_mib = 1590, available_mib = 61850)
+ assert gpus == [(0, 61850 - 1024, 124609)]