Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: take GGML_CUDA_ENABLE_UNIFIED_MEMORY only when the weights outgrow the APU carve-out (#10351)

Daniel Han9a21df4172f3
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index eb15d8d172f..f1fb53faabe 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -4788,6 +4788,27 @@ def _paravirtual_draft_ngl_flag(server_caps: Mapping[str, object]) -> Optional[s
return None
+def _argv_keeps_projector_on_gpu(
+ args: Sequence[str], env: Optional[Mapping[str, str]] = None
+) -> bool:
+ """Whether *args* still load a projector onto the device. arg.cpp applies
+ ``LLAMA_ARG_MMPROJ(_URL)`` before argv, so the env alone still loads one."""
+ tokens = [str(a) for a in args]
+ env_map = env or {}
+
+ def _flag(token: str) -> str:
+ return token.split("=", 1)[0].replace("_", "-").lower()
+
+ from_argv = any(_flag(a) in ("--mmproj", "-mm") for a in tokens)
+ from_env = any(
+ str(env_map.get(name) or "").strip()
+ for name in ("LLAMA_ARG_MMPROJ", "LLAMA_ARG_MMPROJ_URL")
+ )
+ if not from_argv and not from_env:
+ return False
+ return _resolved_mmproj_offload(tokens, env_map) is not False
+
+
def _paravirtual_mmproj_pinnable(server_caps: Mapping[str, object]) -> bool:
"""True when --no-mmproj-offload can be emitted, including on an unanswered probe."""
return bool(
@@ -8715,17 +8736,13 @@ class LlamaCppBackend:
return None
@staticmethod
- def _unified_memory_would_help(gpu_indices = None) -> bool:
- """Whether managed allocation is the larger pool for the selected APUs.
-
- On HIP it draws host RAM instead of the selected APUs' carve-outs, so the
- decision is a direct comparison of the two pools. Available host RAM against
- the carve-out's TOTAL is deliberate: a carve-out another process is holding
- is not credited back to the host side, because the two pools fail
- differently. Over-asking the carve-out returns hipErrorOutOfMemory and the
- load fails cleanly, while over-asking host RAM is the OOM kill this gate
- exists to stop, so both halves of the asymmetry lean toward the pool a miss
- is recoverable in. Missing data and mixed-device selections fail closed.
+ def _unified_memory_would_help(gpu_indices = None, need_bytes = None) -> bool:
+ """Whether managed allocation is worth taking for the selected APUs.
+
+ Host RAM must be the larger pool, against the carve-out's TOTAL (over-asking
+ the carve-out fails cleanly, over-asking host RAM is an OOM kill), AND the
+ weights must not fit that carve-out: managed pages fault in k_set_rows on
+ Linux ROCm gfx1151 (HF Qwen3.8-Flash-Next-GGUF discussion 30, #10330).
"""
try:
pool_mib = LlamaCppBackend._rocm_selected_pool_mib(gpu_indices)
@@ -8734,7 +8751,11 @@ class LlamaCppBackend:
host_mib = LlamaCppBackend._available_system_memory_mib()
if not host_mib:
return False
- return int(host_mib) > int(pool_mib)
+ if int(host_mib) <= int(pool_mib):
+ return False
+ if need_bytes is None:
+ return False
+ return int(need_bytes) > int(pool_mib) * 1024 * 1024
except Exception:
return False
@@ -8877,6 +8898,29 @@ class LlamaCppBackend:
except Exception:
return False
+ @staticmethod
+ def _unified_memory_opted_in(env = None) -> bool:
+ """True when UNSLOTH_ENABLE_UNIFIED_MEMORY=1 asks for managed allocation even
+ though the weights fit. Exact "1"; the opt-out wins over it."""
+ try:
+ source = os.environ if env is None else env
+ return str(source.get("UNSLOTH_ENABLE_UNIFIED_MEMORY", "")).strip() == "1"
+ except Exception:
+ return False
+
+ @staticmethod
+ def _unified_memory_for_launch(
+ gpu_indices,
+ need_bytes,
+ *,
+ opted_in = False,
+ ) -> bool:
+ """The launch-time decision behind GGML_CUDA_ENABLE_UNIFIED_MEMORY. It is
+ process-wide and hurts discrete cards, so a mixed selection is always False."""
+ if opted_in and LlamaCppBackend._rocm_selected_pool_mib(gpu_indices) is not None:
+ return True
+ return LlamaCppBackend._unified_memory_would_help(gpu_indices, need_bytes = need_bytes)
+
# Datacenter / professional NVIDIA parts that benefit from the llama.cpp
# FP32-accum / P2P tunings. Whole-word (\b) so short markers don't match
# workstation parts as substrings: "a100" must not fire on "RTX A1000".
@@ -10320,6 +10364,42 @@ class LlamaCppBackend:
n_layers = self.n_layers
return requested == -1 or (bool(n_layers) and requested > n_layers)
+ def _launch_forces_full_offload(
+ self,
+ argv: Iterable[str],
+ env: Optional[Mapping[str, str]] = None,
+ ) -> bool:
+ """Whether the child puts every layer on a GPU whatever the fitter says. The
+ fitter cannot lower a count the user fixed (common/fit.cpp "n_gpu_layers
+ already set by user", downgraded to a warning), so a count above the block
+ count stands under ``--fit on``, as does no count at all with the fitter off
+ (llama.cpp's default is ``-1``)."""
+ args = [str(a) for a in argv or ()]
+ if self._argv_offloads_every_layer(args, env):
+ return True
+ if _device_selection_is_cpu(args, env):
+ return False
+ if _args_place_tensors_on_cpu(args) or _env_places_tensors_on_cpu(env):
+ return False
+ try:
+ requested = parse_gpu_layers_override(args)
+ except ValueError:
+ return False
+ if requested is None and env and _env_fixes_gpu_layers(env):
+ raw = str(env.get("LLAMA_ARG_N_GPU_LAYERS", "")).strip().lower()
+ if raw == "all":
+ return True
+ try:
+ requested = int(raw)
+ except ValueError:
+ return False
+ if requested is None:
+ return not fit_is_effectively_on(args, env)
+ n_layers = self.n_layers
+ if not n_layers:
+ return False
+ return requested > n_layers
+
@staticmethod
def _rows_the_child_can_reach(detected_gpus, pinned_ids) -> list:
"""``detected_gpus`` narrowed to the cards THIS launch pinned the child to.
@@ -19042,6 +19122,8 @@ class LlamaCppBackend:
total_by_idx: dict[int, int] = {}
_gpu_mem: list[tuple[int, int, int]] = []
model_size = None # set in the fit try; used by the APU RAM guard
+ _mtp_will_engage = False
+ _separate_draft_launches = False # a sidecar displaces an embedded head
# "none" once the fit proves the load needs no demand paging, else None
# for llama.cpp's own default. Bound before the try like the verdict
# flags above: the except path falls through to the launch, which reads it.
@@ -22817,6 +22899,60 @@ class LlamaCppBackend:
# preserve inherited values.
_unified_env_applied = False
_unified_opt_out = self._unified_memory_opted_out(env)
+ _unified_opt_in = self._unified_memory_opted_in(env)
+ # The arch gate and arch-crash retry re-select without rebinding gpu_indices.
+ _unified_gpu_indices = gpu_indices
+
+ def _unified_need_now(argv = None, mtp_engages = None):
+ """Bytes a forced full offload puts on the device, or None."""
+ run_argv = cmd if argv is None else argv
+ # The trailing blocks load only when the EMBEDDED head drafts: a
+ # sidecar wins over it (llama.cpp has_dft()), wherever it lands.
+ if mtp_engages is None:
+ engages = _mtp_will_engage and not _separate_draft_launches
+ else:
+ engages = mtp_engages
+ if model_size is None or not self._launch_forces_full_offload(run_argv, env):
+ return None
+ need = int(model_size)
+ if mmproj_size and not _argv_keeps_projector_on_gpu(run_argv, env):
+ need -= int(mmproj_size)
+ # dev_input is CPU-pinned even at full offload, so token_embd never
+ # reaches the device; a tied file duplicates the matrix onto it instead.
+ layout = self._tensor_spill_layout(model_path, all_shards = True)
+ if layout is None or not getattr(layout, "complete", True):
+ return None
+ if int(getattr(layout, "lm_head_bytes", 0) or 0):
+ need -= int(getattr(layout, "token_embd_bytes", 0) or 0)
+ if not engages and self._nextn_predict_layers:
+ need -= int(getattr(layout, "excluded_block_bytes", 0) or 0)
+ return need
+
+ def _unified_withdraw_if_unneeded(
+ run_cmd,
+ *,
+ mtp_engages = None,
+ why = "",
+ ):
+ """Drop the variable THIS launch set once a respawn stops needing it."""
+ nonlocal _unified_env_applied
+ if not _unified_env_applied:
+ return
+ if self._unified_memory_for_launch(
+ _unified_gpu_indices,
+ _unified_need_now(argv = run_cmd, mtp_engages = mtp_engages),
+ opted_in = _unified_opt_in,
+ ):
+ return
+ env.pop("GGML_CUDA_ENABLE_UNIFIED_MEMORY", None)
+ _unified_env_applied = False
+ logger.info(
+ "%s no longer needs managed memory; dropped "
+ "GGML_CUDA_ENABLE_UNIFIED_MEMORY.",
+ why or "The retry",
+ )
+
+ _unified_need = _unified_need_now()
if _unified_opt_out:
# ggml tests presence, so passing a user's "0" through would
# ENABLE what they turned off (#8651). Only absence is off.
@@ -22824,12 +22960,16 @@ class LlamaCppBackend:
logger.info(
"Unified memory opted out: unset GGML_CUDA_ENABLE_UNIFIED_MEMORY"
)
- elif not is_vulkan_backend and self._unified_memory_would_help(gpu_indices):
+ elif not is_vulkan_backend and self._unified_memory_for_launch(
+ gpu_indices, _unified_need, opted_in = _unified_opt_in
+ ):
_unified_env_applied = "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
env.setdefault("GGML_CUDA_ENABLE_UNIFIED_MEMORY", "1")
logger.info(
- "AMD unified-memory APU whose carve-out is smaller than host "
- "RAM: set GGML_CUDA_ENABLE_UNIFIED_MEMORY=1"
+ "AMD unified-memory APU: set GGML_CUDA_ENABLE_UNIFIED_MEMORY=1 (%s)",
+ "UNSLOTH_ENABLE_UNIFIED_MEMORY=1"
+ if _unified_opt_in
+ else "the weights outgrow the carve-out and host RAM is the larger pool",
)
# DC NVIDIA GPUs: FP32 accum (+ P2P / launch queues for multi-GPU).
@@ -22981,7 +23121,10 @@ class LlamaCppBackend:
self._clear_split_placement_env(env)
# The setting is process-wide, so recompute it after changing
# the selected devices. Ownership protects inherited values.
- _survivors_gain_unified = self._unified_memory_would_help(_survivors)
+ _unified_gpu_indices = _survivors
+ _survivors_gain_unified = self._unified_memory_for_launch(
+ _survivors, _unified_need_now(), opted_in = _unified_opt_in
+ )
if _unified_env_applied and not _survivors_gain_unified:
env.pop("GGML_CUDA_ENABLE_UNIFIED_MEMORY", None)
_unified_env_applied = False
@@ -22999,7 +23142,7 @@ class LlamaCppBackend:
_unified_env_applied = True
logger.info(
"Arch gate narrowed the launch onto a unified-memory "
- "APU whose carve-out is smaller than host RAM; set "
+ "APU the weights outgrow; set "
"GGML_CUDA_ENABLE_UNIFIED_MEMORY=1."
)
self._emit_child_gpu_visibility(
@@ -23824,7 +23967,9 @@ class LlamaCppBackend:
# APU presence controls the RAM guard; relative pool sizes
# determine whether managed allocations help.
_retry_wants_unified = self._amd_apu_wants_unified_memory(_remaining)
- _retry_unified_helps = self._unified_memory_would_help(_remaining)
+ _retry_unified_helps = self._unified_memory_for_launch(
+ _remaining, _unified_need_now(), opted_in = _unified_opt_in
+ )
# Everything recorded so far priced the CRASHED selection, and
# that placement is gone: the canonical #7624 shape pins the APU
# whose shared-pool "free memory" outranked the dGPU, warns that
@@ -23879,6 +24024,7 @@ class LlamaCppBackend:
)
self._kill_process()
gpu_indices = _remaining
+ _unified_gpu_indices = _remaining
# GGML_CUDA_ENABLE_UNIFIED_MEMORY was decided for the CRASHED
# set. The canonical #7624 shape crashes on the APU and retries
# on the dGPU, where it is harmful, so withdraw it, but only
@@ -23901,8 +24047,8 @@ class LlamaCppBackend:
env["GGML_CUDA_ENABLE_UNIFIED_MEMORY"] = "1"
_unified_env_applied = True
logger.info(
- "Arch-crash retry targets a unified-memory APU whose "
- "carve-out is smaller than host RAM; set "
+ "Arch-crash retry targets a unified-memory APU the "
+ "weights outgrow; set "
"GGML_CUDA_ENABLE_UNIFIED_MEMORY=1 for the respawn."
)
self._emit_child_gpu_visibility(
@@ -24292,6 +24438,11 @@ class LlamaCppBackend:
strip_split_mode = False,
)
fallback_cmd = cmd[:_spec_at] + ["--spec-default"] + _fb_tail
+ _unified_withdraw_if_unneeded(
+ fallback_cmd,
+ mtp_engages = False,
+ why = "The retry without speculative decoding",
+ )
healthy = _spawn_and_wait(fallback_cmd, label = "-retry")
if healthy:
self._speculative_type = "default"
@@ -24375,6 +24526,9 @@ class LlamaCppBackend:
"projector on CPU to preserve image input."
)
cmd = _cpu_projector_cmd
+ _unified_withdraw_if_unneeded(
+ cmd, why = "The retry with the projector on CPU"
+ )
healthy = _spawn_and_wait(cmd, label = "-mmproj-cpu")
if healthy:
self._mmproj_fallback_reason = "cpu_offload"
@@ -24448,6 +24602,7 @@ class LlamaCppBackend:
)
self._mmproj_fallback_reason = "projector_startup_failure"
cmd = self._strip_mmproj_args(_vision_gpu_cmd)
+ _unified_withdraw_if_unneeded(cmd, why = "The text-only retry")
# This retry bypasses _spawn_and_wait, so refresh the
# launched-argv snapshot itself -- the zero-offload
# classification below must not see the stripped --mmproj.
@@ -26715,7 +26870,12 @@ class LlamaCppBackend:
logger.info("Tensor spill: dropping the plan for the %s retry; %s", why, "using --fit on")
return [*stripped, "--fit", "on"]
- def _tensor_spill_layout(self, model_path: "Optional[str]") -> "Optional[ModelLayout]":
+ def _tensor_spill_layout(
+ self,
+ model_path: "Optional[str]",
+ *,
+ all_shards: bool = False,
+ ) -> "Optional[ModelLayout]":
"""The GGUF's placement buckets, read once per path and cached.
Per BLOCK, not per bucket total, so the planner can spill the minimum set
@@ -26737,14 +26897,18 @@ class LlamaCppBackend:
# (size, mtime_ns) stat identity _slot_launch_fingerprint uses.
try:
st = os.stat(model_path)
- key = (model_path, st.st_size, st.st_mtime_ns)
+ key = (model_path, st.st_size, st.st_mtime_ns, all_shards)
except OSError:
- key = (model_path, None, None)
+ key = (model_path, None, None, all_shards)
cached = getattr(self, "_spill_layout_cache", None)
if cached is not None and cached[0] == key:
return cached[1]
try:
- layout = layout_from_gguf(model_path)
+ layout = (
+ layout_from_gguf(model_path, all_shards = True)
+ if all_shards
+ else layout_from_gguf(model_path)
+ )
except Exception as e: # unreadable, truncated, or an arch we cannot bucket
logger.debug("Tensor spill: cannot read layout from %s (%s)", model_path, e)
layout = None
diff --git a/studio/backend/core/inference/offload_layout.py b/studio/backend/core/inference/offload_layout.py
index ce885e1f3fb..bf1779a3984 100644
--- a/studio/backend/core/inference/offload_layout.py
+++ b/studio/backend/core/inference/offload_layout.py
@@ -14,6 +14,7 @@ import logging
import re
from dataclasses import dataclass, field
from typing import Optional
+import os
logger = logging.getLogger(__name__)
@@ -123,33 +124,63 @@ def _field(
return default
-def layout_from_gguf(path: str) -> ModelLayout:
+_SPLIT_SHARD_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})\.gguf$", re.IGNORECASE)
+
+
+def split_shard_paths(path: str) -> Optional[list[str]]:
+ """Every shard of the split *path* belongs to, in order, or None when the name
+ is not llama.cpp's ``<prefix>-NNNNN-of-MMMMM.gguf`` (llama_split_path)."""
+ directory, name = os.path.split(path)
+ match = _SPLIT_SHARD_RE.match(name)
+ if not match:
+ return None
+ prefix, _index, total = match.groups()
+ return [
+ os.path.join(directory, f"{prefix}-{i:05d}-of-{int(total):05d}.gguf")
+ for i in range(1, int(total) + 1)
+ ]
+
+
+def layout_from_gguf(path: str, *, all_shards: bool = False) -> ModelLayout:
"""Read ``path`` into a :class:`ModelLayout`.
Returns an incomplete layout (``complete = False``) rather than raising when
anything required is missing, so a surprising GGUF makes the planner abstain
instead of failing a load that llama.cpp would have handled.
+ ``all_shards`` reads every sibling shard; all of them must be present.
"""
try:
from gguf import GGUFReader
- reader = GGUFReader(path)
+ readers = [GGUFReader(path)]
+ if all_shards and int(_field(readers[0], "split.count") or 0) > 1:
+ shards = split_shard_paths(path)
+ if not shards or not all(os.path.isfile(p) for p in shards):
+ logger.debug("offload layout: split %s is missing a shard", path)
+ return ModelLayout()
+ readers = [GGUFReader(p) for p in shards]
except Exception as exc:
logger.debug("offload layout: cannot read %s (%s)", path, exc)
return ModelLayout()
try:
- return _layout_from_reader(reader)
+ return _layout_from_readers(readers)
except Exception as exc:
logger.debug("offload layout: cannot interpret %s (%s)", path, exc)
return ModelLayout()
def _layout_from_reader(reader) -> ModelLayout:
+ return _layout_from_readers([reader])
+
+
+def _layout_from_readers(readers) -> ModelLayout:
+ """One reader per shard, the first carrying the metadata."""
+ reader = readers[0]
# Split GGUF: llama.cpp loads every sibling shard (llama-model-loader.cpp:590-618), but GGUFReader memmaps only the
# ONE path it was given. Shard 1 still carries the metadata, so the layout would look complete while undercounting
# resident and spillable by most of the model -- an overstated fit, too few -ot patterns, and a startup OOM with
- # --fit off. Abstain instead; the seam then reproduces --fit on exactly.
- if int(_field(reader, "split.count") or 0) > 1:
+ # --fit off. Abstain unless every shard was handed over; the seam then reproduces --fit on exactly.
+ if (int(_field(reader, "split.count") or 0) or 1) != len(readers):
return ModelLayout()
arch = str(_field(reader, "general.architecture") or "")
@@ -213,7 +244,7 @@ def _layout_from_reader(reader) -> ModelLayout:
token_embd = 0
other_resident = 0
- for tensor in reader.tensors:
+ for tensor in (t for r in readers for t in r.tensors):
name = str(tensor.name)
nbytes = int(tensor.n_bytes)
match = _BLOCK_RE.match(name)
diff --git a/studio/backend/tests/test_amd_apu_unified_memory.py b/studio/backend/tests/test_amd_apu_unified_memory.py
index 65f52233ea1..265144df908 100644
--- a/studio/backend/tests/test_amd_apu_unified_memory.py
+++ b/studio/backend/tests/test_amd_apu_unified_memory.py
@@ -420,7 +420,8 @@ _GIB = 1024**3
class TestTheUnifiedMemorySwapIsOnlyTakenWhenItPays:
- """Managed allocation is enabled only when host RAM is the larger pool."""
+ """Managed allocation is enabled only when host RAM is the larger pool. Every arm
+ prices weights that outgrow the carve-out, so only the pools decide."""
def _host(self, monkeypatch, specs, ram_mib):
monkeypatch.setitem(sys.modules, "torch", _fake_torch_sized(specs))
@@ -430,23 +431,23 @@ class TestTheUnifiedMemorySwapIsOnlyTakenWhenItPays:
def test_a_small_carve_out_still_gets_it(self, monkeypatch):
self._host(monkeypatch, [("gfx1151", 16 * _GIB)], 117_000)
- assert LlamaCppBackend._unified_memory_would_help() is True
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is True
def test_a_carve_out_larger_than_host_ram_does_not(self, monkeypatch):
self._host(monkeypatch, [("gfx1151", 64 * _GIB)], 58_880)
- assert LlamaCppBackend._unified_memory_would_help() is False
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is False
def test_an_exact_tie_does_not(self, monkeypatch):
self._host(monkeypatch, [("gfx1151", 32 * _GIB)], 32 * 1024)
- assert LlamaCppBackend._unified_memory_would_help() is False
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is False
def test_a_discrete_card_is_never_offered_it(self, monkeypatch):
self._host(monkeypatch, [("gfx1100", 16 * _GIB)], 117_000)
- assert LlamaCppBackend._unified_memory_would_help() is False
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is False
def test_unreadable_host_ram_fails_closed(self, monkeypatch):
self._host(monkeypatch, [("gfx1151", 16 * _GIB)], None)
- assert LlamaCppBackend._unified_memory_would_help() is False
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is False
def test_an_unreported_pool_size_fails_closed(self, monkeypatch):
monkeypatch.setitem(sys.modules, "torch", _fake_torch("6.2.0", ["gfx1151"]))
@@ -454,41 +455,149 @@ class TestTheUnifiedMemorySwapIsOnlyTakenWhenItPays:
LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 117_000)
)
assert LlamaCppBackend._rocm_selected_pool_mib() is None
- assert LlamaCppBackend._unified_memory_would_help() is False
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is False
def test_a_missing_torch_fails_closed(self, monkeypatch):
monkeypatch.setitem(sys.modules, "torch", None)
assert LlamaCppBackend._rocm_selected_pool_mib() is None
- assert LlamaCppBackend._unified_memory_would_help() is False
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is False
def test_two_apus_are_weighed_together(self, monkeypatch):
self._host(monkeypatch, [("gfx1151", 8 * _GIB), ("gfx1150", 24 * _GIB)], 40_000)
assert LlamaCppBackend._rocm_selected_pool_mib() == 32 * 1024
- assert LlamaCppBackend._unified_memory_would_help() is True
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is True
self._host(monkeypatch, [("gfx1151", 8 * _GIB), ("gfx1150", 24 * _GIB)], 30_000)
- assert LlamaCppBackend._unified_memory_would_help() is False
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is False
def test_the_answer_is_scoped_to_the_selected_gpu(self, monkeypatch):
self._host(monkeypatch, [("gfx1151", 8 * _GIB), ("gfx1150", 64 * _GIB)], 40_000)
- assert LlamaCppBackend._unified_memory_would_help([0]) is True
- assert LlamaCppBackend._unified_memory_would_help([1]) is False
+ assert LlamaCppBackend._unified_memory_would_help([0], need_bytes = 100 * _GIB) is True
+ assert LlamaCppBackend._unified_memory_would_help([1], need_bytes = 100 * _GIB) is False
def test_a_discrete_card_in_the_selection_fails_closed(self, monkeypatch):
self._host(monkeypatch, [("gfx1151", 8 * _GIB), ("gfx1100", 64 * _GIB)], 40_000)
assert LlamaCppBackend._rocm_selected_pool_mib() is None
- assert LlamaCppBackend._unified_memory_would_help() is False
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is False
def test_the_same_mixed_host_pinned_to_the_apu_still_gains(self, monkeypatch):
self._host(monkeypatch, [("gfx1151", 8 * _GIB), ("gfx1100", 64 * _GIB)], 40_000)
assert LlamaCppBackend._rocm_selected_pool_mib([0]) == 8 * 1024
- assert LlamaCppBackend._unified_memory_would_help([0]) is True
+ assert LlamaCppBackend._unified_memory_would_help([0], need_bytes = 100 * _GIB) is True
def test_an_id_this_host_does_not_enumerate_fails_closed(self, monkeypatch):
self._host(monkeypatch, [("gfx1151", 8 * _GIB)], 117_000)
assert LlamaCppBackend._rocm_selected_pool_mib([0, 3]) is None
- assert LlamaCppBackend._unified_memory_would_help([0, 3]) is False
+ assert LlamaCppBackend._unified_memory_would_help([0, 3], need_bytes = 100 * _GIB) is False
def test_an_empty_selection_fails_closed(self, monkeypatch):
self._host(monkeypatch, [("gfx1151", 8 * _GIB)], 117_000)
assert LlamaCppBackend._rocm_selected_pool_mib([]) is None
- assert LlamaCppBackend._unified_memory_would_help([]) is False
+ assert LlamaCppBackend._unified_memory_would_help([], need_bytes = 100 * _GIB) is False
+
+
+class TestManagedMemoryIsTakenOnlyWhenTheWeightsOutgrowTheCarveOut:
+ """The second condition: managed pages fault in k_set_rows on Linux ROCm gfx1151
+ (HF Qwen3.8-Flash-Next-GGUF discussion 30, #10330), so a fitting load never takes them."""
+
+ def _strix_halo(
+ self,
+ monkeypatch,
+ carve_out = 64 * _GIB,
+ ram_mib = 117_000,
+ ):
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch_sized([("gfx1151", carve_out)]))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: ram_mib)
+ )
+
+ def test_weights_that_fit_the_carve_out_do_not_get_it(self, monkeypatch):
+ self._strix_halo(monkeypatch)
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 45 * _GIB) is False
+
+ def test_weights_that_outgrow_it_do(self, monkeypatch):
+ self._strix_halo(monkeypatch)
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is True
+
+ def test_weights_exactly_the_carve_out_do_not(self, monkeypatch):
+ self._strix_halo(monkeypatch)
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 64 * _GIB) is False
+
+ def test_one_byte_over_does(self, monkeypatch):
+ """Bytes against bytes: flooring the need to MiB hid a sub-MiB overrun."""
+ self._strix_halo(monkeypatch)
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 64 * _GIB + 1) is True
+
+ def test_unpriced_weights_fail_closed(self, monkeypatch):
+ self._strix_halo(monkeypatch)
+ assert LlamaCppBackend._unified_memory_would_help() is False
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = None) is False
+
+ def test_the_pool_comparison_still_comes_first(self, monkeypatch):
+ self._strix_halo(monkeypatch, carve_out = 96 * _GIB, ram_mib = 30_000)
+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 120 * _GIB) is False
+
+
+class TestTheEnableSwitch:
+ """UNSLOTH_ENABLE_UNIFIED_MEMORY=1 takes managed allocation even when weights fit."""
+
+ @pytest.mark.parametrize(
+ "env, expected",
+ [
+ ({"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"}, True),
+ ({"UNSLOTH_ENABLE_UNIFIED_MEMORY": " 1 "}, True),
+ ({"UNSLOTH_ENABLE_UNIFIED_MEMORY": "0"}, False),
+ ({"UNSLOTH_ENABLE_UNIFIED_MEMORY": "yes"}, False),
+ ({}, False),
+ ],
+ )
+ def test_opt_in_decisions(self, env, expected):
+ assert LlamaCppBackend._unified_memory_opted_in(env) is expected
+
+ def test_none_reads_the_process_env(self, monkeypatch):
+ monkeypatch.delenv("UNSLOTH_ENABLE_UNIFIED_MEMORY", raising = False)
+ assert LlamaCppBackend._unified_memory_opted_in() is False
+ monkeypatch.setenv("UNSLOTH_ENABLE_UNIFIED_MEMORY", "1")
+ assert LlamaCppBackend._unified_memory_opted_in() is True
+
+ def test_a_hostile_env_fails_closed(self):
+ class _Exploding:
+ def get(self, *_a, **_k):
+ raise RuntimeError("no")
+
+ assert LlamaCppBackend._unified_memory_opted_in(_Exploding()) is False
+
+ def _strix_halo(self, monkeypatch):
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch_sized([("gfx1151", 64 * _GIB)]))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 117_000)
+ )
+
+ def test_the_switch_takes_it_for_weights_that_fit(self, monkeypatch):
+ self._strix_halo(monkeypatch)
+ assert LlamaCppBackend._unified_memory_for_launch([0], 45 * _GIB) is False
+ assert LlamaCppBackend._unified_memory_for_launch([0], 45 * _GIB, opted_in = True) is True
+
+ def test_the_switch_never_reaches_a_discrete_card(self, monkeypatch):
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch_sized([("gfx1100", 16 * _GIB)]))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 117_000)
+ )
+ assert LlamaCppBackend._unified_memory_for_launch([0], 45 * _GIB, opted_in = True) is False
+
+ def test_the_switch_refuses_a_mixed_selection(self, monkeypatch):
+ monkeypatch.setitem(
+ sys.modules,
+ "torch",
+ _fake_torch_sized([("gfx1151", 64 * _GIB), ("gfx1100", 16 * _GIB)]),
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 117_000)
+ )
+ assert LlamaCppBackend._unified_memory_for_launch([0, 1], 45 * _GIB, opted_in = True) is False
+ assert LlamaCppBackend._unified_memory_for_launch(None, 45 * _GIB, opted_in = True) is False
+ assert LlamaCppBackend._unified_memory_for_launch([0], 45 * _GIB, opted_in = True) is True
+
+ def test_without_the_switch_the_priced_gate_decides(self, monkeypatch):
+ self._strix_halo(monkeypatch)
+ assert LlamaCppBackend._unified_memory_for_launch([0], 100 * _GIB) is True
+ assert LlamaCppBackend._unified_memory_for_launch([0], None) is False
diff --git a/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py b/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py
index ef3d6ad3703..159647d278b 100644
--- a/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py
+++ b/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py
@@ -770,6 +770,15 @@ class TestArchStringRobustness:
) == frozenset({"gfx1030"})
+def _priced_layout(**fields):
+ """A readable layout: untied embeddings, nothing CPU-pinned, no MTP blocks."""
+ from core.inference.offload_layout import ModelLayout
+
+ values = {"complete": True, "lm_head_bytes": 1}
+ values.update(fields)
+ return ModelLayout(**values)
+
+
def _run_auto_load(
monkeypatch,
tmp_path,
@@ -785,6 +794,8 @@ def _run_auto_load(
apu_ram_stub = None,
host_offload_stub = None,
backend = None,
+ mmproj_bytes = 0,
+ server_caps = None,
):
"""Drive a real automatic (no explicit GPU pick) llama-server load with the real
``_get_gpu_memory`` behind it, and return the spawned (cmd, env) list.
@@ -817,8 +828,17 @@ def _run_auto_load(
# model_bytes drives the placement decision: a model no GPU can hold makes
# _select_gpus return (None, True), so `--fit on` owns placement.
backend._get_gguf_size_bytes = lambda _path: model_bytes
- backend._mmproj_vram_bytes = lambda _path: 0
- backend._resolve_launch_mmproj_path = lambda **_kwargs: None
+ if mmproj_bytes:
+ mmproj = tmp_path / "mmproj.gguf"
+ mmproj.write_bytes(b"GGUF")
+ backend._mmproj_vram_bytes = lambda _path: mmproj_bytes
+ backend._resolve_launch_mmproj_path = lambda **_kwargs: str(mmproj)
+ else:
+ backend._mmproj_vram_bytes = lambda _path: 0
+ backend._resolve_launch_mmproj_path = lambda **_kwargs: None
+ # A header-only GGUF prices nothing, so hand the load a readable layout.
+ if "_tensor_spill_layout" not in vars(backend):
+ backend._tensor_spill_layout = lambda _path, **_kw: _priced_layout()
# Off by default: the APU RAM preflight is not what most of these cells are
# about. A test that IS about it passes its own recording stub.
backend._apu_ram_shortfall_message = apu_ram_stub or (lambda *_args, **_kwargs: None)
@@ -826,7 +846,7 @@ def _run_auto_load(
backend._host_offload_shortfall_message = host_offload_stub or (lambda *_args, **_kwargs: None)
backend._find_llama_server_binary = lambda include_denied = False: binary
backend._fit_off_retry_eligible = lambda *_args, **_kwargs: False
- backend.probe_server_capabilities = lambda _binary: {"found": True}
+ backend.probe_server_capabilities = lambda _binary: {"found": True, **(server_caps or {})}
backend._record_server_pid = lambda _pid: None
backend._clear_server_pid = lambda: None
# env_extra seeds the child env the way an inherited / user-set variable
@@ -1144,7 +1164,7 @@ class TestArchCrashRetryEnv:
)
return _fake_torch(
[
- _device("gfx1151", free_mib = 40000, is_integrated = 1),
+ _device("gfx1151", free_mib = 40000, total_bytes = 8 * GIB, is_integrated = 1),
_device("gfx1030", free_mib = 12049),
],
vendor = "amd",
@@ -1164,6 +1184,7 @@ class TestArchCrashRetryEnv:
None, # no marker: the proactive gate fails open, so the crash path runs
returncode = 1,
output = "ROCm error: device kernel image is invalid",
+ model_bytes = 10 * GIB, # outgrows the APU's 8 GiB carve-out
)
# The APU is pinned first, so the crashed spawn carries the env and the
# respawns are masked onto the discrete card. The unrelated --fit off retry
@@ -1715,6 +1736,7 @@ class TestGatedNarrowingDropsUnifiedMemory:
["gfx1151"], # covers the APU, not the gfx1200 dGPU
returncode = None,
model_bytes = 400 * 1024**3,
+ env_extra = {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"}, # the fitter owns the layers
)
_cmd, env = launches[0]
assert _visibility(env) == {"ROCR_VISIBLE_DEVICES": "0", "CUDA_VISIBLE_DEVICES": "0"}
@@ -1911,7 +1933,7 @@ class TestArchCrashRetryOntoAnApu:
return _fake_torch(
[
_device("gfx1030", free_mib = 40000),
- _device("gfx1151", free_mib = 12000, is_integrated = 1),
+ _device("gfx1151", free_mib = 12000, total_bytes = 8 * GIB, is_integrated = 1),
],
vendor = "amd",
)
@@ -1925,6 +1947,7 @@ class TestArchCrashRetryOntoAnApu:
None, # no marker: the proactive gate fails open, so the crash path runs
returncode = 1,
output = "ROCm error: device kernel image is invalid",
+ model_bytes = 10 * GIB, # outgrows the APU's 8 GiB carve-out
)
assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in launches[0][1]
_retry = [env for _c, env in launches if env.get("ROCR_VISIBLE_DEVICES") == "1"]
@@ -1957,21 +1980,312 @@ class TestUnifiedMemoryOptOut:
tmp_path,
monkeypatch,
env_extra = None,
+ model_bytes = 40 * GIB, # outgrows the fake 32 GiB carve-out
+ backend = None,
+ extra_args = None,
+ returncode = None,
+ output = "",
+ intent_extra = None,
+ mmproj_bytes = 0,
+ server_caps = None,
):
+ # Manual mode above the block count: the only launch that needs managed pages.
+ backend = backend or LlamaCppBackend()
+ backend._n_layers = 8
+ intent_kwargs = {"gpu_memory_mode": "manual", "gpu_layers": 9}
+ if extra_args:
+ intent_kwargs["extra_args"] = list(extra_args)
+ intent_kwargs.update(intent_extra or {})
return _run_auto_load(
monkeypatch,
tmp_path,
self._strix_halo(monkeypatch),
None,
- returncode = None,
+ returncode = returncode,
+ output = output,
env_extra = env_extra,
+ model_bytes = model_bytes,
+ backend = backend,
+ intent_kwargs = intent_kwargs,
+ mmproj_bytes = mmproj_bytes,
+ server_caps = server_caps,
)
- def test_the_apu_still_gets_it_by_default(self, tmp_path, monkeypatch, probe_env):
+ def test_a_forced_full_offload_that_outgrows_the_carve_out_gets_it(
+ self, tmp_path, monkeypatch, probe_env
+ ):
"""Baseline: #5301 added the variable for exactly this hardware."""
_cmd, env = self._load(tmp_path, monkeypatch)[0]
+ assert "--gpu-layers" in _cmd and _cmd[_cmd.index("--gpu-layers") + 1] == "9"
+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+
+ def test_weights_that_fit_the_carve_out_keep_it_unset(self, tmp_path, monkeypatch, probe_env):
+ """Managed pages fault Qwen3.8-Flash-Next on Linux gfx1151 (#10330)."""
+ _cmd, env = self._load(tmp_path, monkeypatch, model_bytes = 20 * GIB)[0]
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ def test_the_enable_switch_takes_it_for_a_fitting_model(self, tmp_path, monkeypatch, probe_env):
+ _cmd, env = self._load(
+ tmp_path, monkeypatch, {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"}, model_bytes = 20 * GIB
+ )[0]
+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+
+ def test_a_manual_partial_placement_never_takes_it(self, tmp_path, monkeypatch, probe_env):
+ launches = _run_auto_load(
+ monkeypatch,
+ tmp_path,
+ self._strix_halo(monkeypatch),
+ None,
+ returncode = None,
+ model_bytes = 400 * GIB,
+ intent_kwargs = {"gpu_memory_mode": "manual", "gpu_layers": 1},
+ )
+ _cmd, env = launches[0]
+ assert "--gpu-layers" in _cmd and _cmd[_cmd.index("--gpu-layers") + 1] == "1"
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ def test_the_fitter_owning_the_layer_count_never_takes_it(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ launches = _run_auto_load(
+ monkeypatch,
+ tmp_path,
+ self._strix_halo(monkeypatch),
+ None,
+ returncode = None,
+ model_bytes = 400 * GIB,
+ )
+ _cmd, env = launches[0]
+ assert "--fit" in _cmd and _cmd[_cmd.index("--fit") + 1] == "on"
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ def _auto_mode_with(
+ self,
+ tmp_path,
+ monkeypatch,
+ *,
+ extra_args = None,
+ env_extra = None,
+ ):
+ backend = LlamaCppBackend()
+ backend._n_layers = 8
+ return _run_auto_load(
+ monkeypatch,
+ tmp_path,
+ self._strix_halo(monkeypatch),
+ None,
+ returncode = None,
+ model_bytes = 40 * GIB,
+ backend = backend,
+ intent_kwargs = {"extra_args": list(extra_args or [])},
+ env_extra = env_extra,
+ )[0]
+
+ def test_a_user_count_above_the_block_count_stands_under_the_fitter(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ _cmd, env = self._auto_mode_with(tmp_path, monkeypatch, extra_args = ["--gpu-layers", "9"])
+ assert "--fit" in _cmd and _cmd[_cmd.index("--fit") + 1] == "on"
+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+
+ def test_an_inherited_count_above_the_block_count_stands_too(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ _cmd, env = self._auto_mode_with(
+ tmp_path, monkeypatch, env_extra = {"LLAMA_ARG_N_GPU_LAYERS": "9"}
+ )
+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+
+ def test_a_user_minus_one_still_leaves_the_fitter_in_charge(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ _cmd, env = self._auto_mode_with(tmp_path, monkeypatch, extra_args = ["-ngl", "-1"])
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ def test_a_user_fit_off_with_no_count_is_a_full_offload(self, tmp_path, monkeypatch, probe_env):
+ _cmd, env = self._auto_mode_with(tmp_path, monkeypatch, extra_args = ["--fit", "off"])
+ assert "--gpu-layers" not in _cmd and "-ngl" not in _cmd
+ assert _cmd[len(_cmd) - 1 - _cmd[::-1].index("--fit") + 1] == "off"
+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+
+ def test_a_user_fit_off_with_a_partial_count_is_not(self, tmp_path, monkeypatch, probe_env):
+ _cmd, env = self._auto_mode_with(
+ tmp_path, monkeypatch, extra_args = ["--fit", "off", "--gpu-layers", "4"]
+ )
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ def test_a_user_count_below_the_block_count_never_takes_it(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ _cmd, env = self._auto_mode_with(tmp_path, monkeypatch, extra_args = ["--gpu-layers", "4"])
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ def _with_unloaded_mtp_blocks(self, excluded_bytes):
+ backend = LlamaCppBackend()
+ backend._nextn_predict_layers = 1
+ backend._tensor_spill_layout = lambda _path, **_kw: (
+ None
+ if excluded_bytes is None
+ else types.SimpleNamespace(excluded_block_bytes = excluded_bytes)
+ )
+ return backend
+
+ def test_unloaded_mtp_blocks_are_not_priced(self, tmp_path, monkeypatch, probe_env):
+ _cmd, env = self._load(
+ tmp_path, monkeypatch, backend = self._with_unloaded_mtp_blocks(10 * GIB)
+ )[0]
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ def test_unloaded_mtp_blocks_still_leave_an_oversized_base(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ _cmd, env = self._load(
+ tmp_path, monkeypatch, backend = self._with_unloaded_mtp_blocks(4 * GIB)
+ )[0]
+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+
+ def test_the_drafterless_retry_reprices_without_the_mtp_blocks(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ launches = self._load(
+ tmp_path,
+ monkeypatch,
+ backend = self._with_unloaded_mtp_blocks(10 * GIB),
+ extra_args = ["--spec-type", "draft-mtp"],
+ returncode = 1,
+ output = "failed to create llama_context",
+ )
+ first_cmd, first_env = launches[0]
+ assert "draft-mtp" in first_cmd, first_cmd
+ assert first_env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+ retries = [(c, e) for c, e in launches if "--spec-default" in c and "draft-mtp" not in c]
+ assert retries, [c for c, _e in launches]
+ assert all("GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in e for _c, e in retries)
+
+ def _with_layout(self, **fields):
+ backend = LlamaCppBackend()
+ backend._tensor_spill_layout = lambda _path, **_kw: _priced_layout(**fields)
+ return backend
+
+ def test_cpu_pinned_embeddings_are_not_priced(self, tmp_path, monkeypatch, probe_env):
+ _cmd, env = self._load(
+ tmp_path, monkeypatch, backend = self._with_layout(token_embd_bytes = 10 * GIB)
+ )[0]
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ def test_tied_embeddings_still_reach_the_device(self, tmp_path, monkeypatch, probe_env):
+ _cmd, env = self._load(
+ tmp_path,
+ monkeypatch,
+ backend = self._with_layout(token_embd_bytes = 10 * GIB, lm_head_bytes = 0),
+ )[0]
assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+ def test_small_embeddings_leave_an_oversized_base(self, tmp_path, monkeypatch, probe_env):
+ _cmd, env = self._load(
+ tmp_path, monkeypatch, backend = self._with_layout(token_embd_bytes = 4 * GIB)
+ )[0]
+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+
+ def test_an_incomplete_layout_cannot_price_the_file(self, tmp_path, monkeypatch, probe_env):
+ _cmd, env = self._load(tmp_path, monkeypatch, backend = self._with_layout(complete = False))[0]
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ _PROJECTOR_OOM = (
+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 4096.00 MiB on "
+ "device 0: cudaMalloc failed: out of memory"
+ )
+
+ def _vision_launches(
+ self,
+ tmp_path,
+ monkeypatch,
+ server_caps = None,
+ ):
+ """30 GiB base + 4 GiB projector outgrow the 32 GiB carve-out; the base does not."""
+ launches = self._load(
+ tmp_path,
+ monkeypatch,
+ model_bytes = 30 * GIB,
+ mmproj_bytes = 4 * GIB,
+ intent_extra = {"is_vision": True, "mmproj_path": str(tmp_path / "mmproj.gguf")},
+ server_caps = server_caps,
+ returncode = 1,
+ output = self._PROJECTOR_OOM,
+ )
+ first_cmd, first_env = launches[0]
+ assert "--mmproj" in first_cmd and "--no-mmproj-offload" not in first_cmd, first_cmd
+ assert first_env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+ return launches
+
+ def test_the_cpu_projector_retry_reprices_without_its_bytes(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ launches = self._vision_launches(
+ tmp_path, monkeypatch, server_caps = {"supports_no_mmproj_offload": True}
+ )
+ cpu_pinned = [e for c, e in launches if "--no-mmproj-offload" in c]
+ assert cpu_pinned, [c for c, _e in launches]
+ assert all("GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in e for e in cpu_pinned)
+
+ def test_the_text_only_retry_reprices_without_the_projector(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ launches = self._vision_launches(tmp_path, monkeypatch)
+ text_only = [e for c, e in launches if "--mmproj" not in c]
+ assert text_only, [c for c, _e in launches]
+ assert all("GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in e for e in text_only)
+
+ def test_an_inherited_projector_outlives_the_text_only_retry(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ inherited = tmp_path / "inherited-mmproj.gguf"
+ inherited.write_bytes(b"GGUF")
+ launches = self._load(
+ tmp_path,
+ monkeypatch,
+ env_extra = {"LLAMA_ARG_MMPROJ": str(inherited)},
+ model_bytes = 30 * GIB,
+ mmproj_bytes = 4 * GIB,
+ intent_extra = {"is_vision": True, "mmproj_path": str(tmp_path / "mmproj.gguf")},
+ returncode = 1,
+ output = self._PROJECTOR_OOM,
+ )
+ first_cmd, first_env = launches[0]
+ assert "--mmproj" in first_cmd, first_cmd
+ assert first_env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+ text_only = [e for c, e in launches if "--mmproj" not in c]
+ assert text_only, [c for c, _e in launches]
+ assert all(e.get("LLAMA_ARG_MMPROJ") == str(inherited) for e in text_only)
+ assert all(e.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1" for e in text_only)
+
+ def test_a_sidecar_drafter_displaces_the_embedded_head(self, tmp_path, monkeypatch, probe_env):
+ sidecar = tmp_path / "draft.gguf"
+ sidecar.write_bytes(b"GGUF")
+ _cmd, env = self._load(
+ tmp_path,
+ monkeypatch,
+ backend = self._with_unloaded_mtp_blocks(10 * GIB),
+ extra_args = ["--spec-type", "draft-mtp", "-md", str(sidecar)],
+ )[0]
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ def test_an_unreadable_tensor_table_cannot_price_the_blocks(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ _cmd, env = self._load(tmp_path, monkeypatch, backend = self._with_unloaded_mtp_blocks(None))[
+ 0
+ ]
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
+ def test_the_disable_switch_beats_the_enable_switch(self, tmp_path, monkeypatch, probe_env):
+ _cmd, env = self._load(
+ tmp_path,
+ monkeypatch,
+ {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1", "UNSLOTH_DISABLE_UNIFIED_MEMORY": "1"},
+ )[0]
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
@pytest.mark.parametrize("value", ["0", "", "false", "FALSE", "no", "off", " 0 "])
def test_a_falsy_user_value_is_not_passed_through(
self, tmp_path, monkeypatch, probe_env, value
@@ -2954,9 +3268,20 @@ class TestTheCarveOutDecidesTheUnifiedMemoryEnv:
monkeypatch,
torch,
env_extra = None,
+ model_bytes = 40 * GIB, # outgrows every carve-out below
):
+ backend = LlamaCppBackend()
+ backend._n_layers = 8
return _run_auto_load(
- monkeypatch, tmp_path, torch, None, returncode = None, env_extra = env_extra
+ monkeypatch,
+ tmp_path,
+ torch,
+ None,
+ returncode = None,
+ env_extra = env_extra,
+ model_bytes = model_bytes,
+ backend = backend,
+ intent_kwargs = {"gpu_memory_mode": "manual", "gpu_layers": 9},
)
def test_a_small_carve_out_gets_it(self, tmp_path, monkeypatch, probe_env):
@@ -3001,6 +3326,26 @@ class TestTheCarveOutDecidesTheUnifiedMemoryEnv:
vendor = "amd",
)
+ def test_the_enable_switch_cannot_reach_a_mixed_selection(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ torch = self._apu_and_dgpu(monkeypatch, 40_000)
+ _cmd, env = _run_auto_load(
+ monkeypatch,
+ tmp_path,
+ torch,
+ None,
+ returncode = None,
+ intent_kwargs = {
+ "gpu_ids": (0, 1),
+ "gpu_memory_mode": "manual",
+ "gpu_layers": 1,
+ },
+ env_extra = {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"},
+ )[0]
+ assert _visibility(env) == {"ROCR_VISIBLE_DEVICES": "0,1", "CUDA_VISIBLE_DEVICES": "0,1"}
+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env
+
def test_a_discrete_card_in_the_launch_keeps_it_off(self, tmp_path, monkeypatch, probe_env):
torch = self._apu_and_dgpu(monkeypatch, 40_000)
_cmd, env = _run_auto_load(
@@ -3029,11 +3374,37 @@ class TestTheCarveOutDecidesTheUnifiedMemoryEnv:
["gfx1151"], # covers the APU, not the discrete gfx1100
returncode = None,
model_bytes = 400 * GIB,
+ env_extra = {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"},
)
_cmd, env = launches[0]
assert _visibility(env) == {"ROCR_VISIBLE_DEVICES": "0", "CUDA_VISIBLE_DEVICES": "0"}
assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+ def test_the_narrowed_selection_is_what_a_later_retry_reprices(
+ self, tmp_path, monkeypatch, probe_env
+ ):
+ torch = self._apu_and_dgpu(monkeypatch, 40_000)
+ backend = LlamaCppBackend()
+ backend._nextn_predict_layers = 1
+ backend._tensor_spill_layout = lambda _path, **_kw: _priced_layout()
+ launches = _run_auto_load(
+ monkeypatch,
+ tmp_path,
+ torch,
+ ["gfx1151"],
+ returncode = 1,
+ output = "failed to create llama_context",
+ model_bytes = 400 * GIB,
+ env_extra = {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"},
+ intent_kwargs = {"extra_args": ["--spec-type", "draft-mtp"]},
+ backend = backend,
+ )
+ first_cmd, first_env = launches[0]
+ assert "draft-mtp" in first_cmd and first_env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"
+ retries = [(c, e) for c, e in launches if "--spec-default" in c and "draft-mtp" not in c]
+ assert retries, [c for c, _e in launches]
+ assert all(e.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1" for _c, e in retries)
+
def test_the_opt_out_survives_that_narrowing(self, tmp_path, monkeypatch, probe_env):
torch = self._apu_and_dgpu(monkeypatch, 40_000)
launches = _run_auto_load(
diff --git a/studio/backend/tests/test_offload_planner.py b/studio/backend/tests/test_offload_planner.py
index 1338da59333..965eb6d40d9 100644
--- a/studio/backend/tests/test_offload_planner.py
+++ b/studio/backend/tests/test_offload_planner.py
@@ -22,7 +22,9 @@ from core.inference.offload_layout import (
BlockLayout,
ModelLayout,
_layout_from_reader,
+ _layout_from_readers,
layout_from_gguf,
+ split_shard_paths,
spill_pattern_for,
)
from core.inference.offload_cost_model import HostProfile
@@ -1490,3 +1492,60 @@ def test_a_wrong_length_vector_is_ignored_rather_than_trusted():
kv_layer_weights = [1, 2, 3],
)
assert got is not None and "sliding-window" in got
+
+
+def test_every_shard_handed_over_prices_the_whole_split():
+ """Two readers summing to one layout, token_embd in its bucket whichever shard holds it."""
+ first = _StubReader(
+ _shard_fields(**{"split.count": 2, "split.no": 0}),
+ _shard_tensors(range(32)) + [_StubTensor("output.weight", GIB)],
+ )
+ second = _StubReader(
+ _shard_fields(**{"split.count": 2, "split.no": 1}),
+ _shard_tensors(range(32, 64)) + [_StubTensor("token_embd.weight", 3 * GIB)],
+ )
+ layout = _layout_from_readers([first, second])
+ assert layout.complete
+ assert len(layout.blocks) == 64
+ assert layout.token_embd_bytes == 3 * GIB
+ assert layout.lm_head_bytes == GIB
+
+
+def test_a_partial_shard_set_still_abstains():
+ readers = [
+ _StubReader(_shard_fields(**{"split.count": 3}), _shard_tensors(range(32))),
+ _StubReader(_shard_fields(**{"split.count": 3}), _shard_tensors(range(32, 64))),
+ ]
+ assert not _layout_from_readers(readers).complete
+
+
+def test_split_shard_paths_follow_llama_cpp_naming():
+ paths = split_shard_paths("/m/model-Q4-00002-of-00003.gguf")
+ assert paths == [
+ "/m/model-Q4-00001-of-00003.gguf",
+ "/m/model-Q4-00002-of-00003.gguf",
+ "/m/model-Q4-00003-of-00003.gguf",
+ ]
+ assert split_shard_paths("/m/model-Q4.gguf") is None
+
+
+def test_layout_from_gguf_reads_the_sibling_shards_only_when_asked(tmp_path, monkeypatch):
+ import gguf
+
+ shards = [tmp_path / f"m-{i:05d}-of-00002.gguf" for i in (1, 2)]
+ for shard in shards:
+ shard.write_bytes(b"GGUF")
+ by_path = {
+ str(shards[0]): _StubReader(_shard_fields(**{"split.count": 2}), _shard_tensors(range(32))),
+ str(shards[1]): _StubReader(
+ _shard_fields(**{"split.count": 2}), _shard_tensors(range(32, 64))
+ ),
+ }
+ monkeypatch.setattr(gguf, "GGUFReader", lambda path: by_path[str(path)])
+
+ assert not layout_from_gguf(str(shards[0])).complete
+ whole = layout_from_gguf(str(shards[0]), all_shards = True)
+ assert whole.complete and len(whole.blocks) == 64
+
+ shards[1].unlink()
+ assert not layout_from_gguf(str(shards[0]), all_shards = True).complete