Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: take GGML_CUDA_ENABLE_UNIFIED_MEMORY only when the weights outgrow the APU carve-out (#10351)
Daniel Han9a21df4172f3
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex eb15d8d172f..f1fb53faabe 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -4788,6 +4788,27 @@ def _paravirtual_draft_ngl_flag(server_caps: Mapping[str, object]) -> Optional[sreturn None+def _argv_keeps_projector_on_gpu(+ args: Sequence[str], env: Optional[Mapping[str, str]] = None+) -> bool:+ """Whether *args* still load a projector onto the device. arg.cpp applies+ ``LLAMA_ARG_MMPROJ(_URL)`` before argv, so the env alone still loads one."""+ tokens = [str(a) for a in args]+ env_map = env or {}++ def _flag(token: str) -> str:+ return token.split("=", 1)[0].replace("_", "-").lower()++ from_argv = any(_flag(a) in ("--mmproj", "-mm") for a in tokens)+ from_env = any(+ str(env_map.get(name) or "").strip()+ for name in ("LLAMA_ARG_MMPROJ", "LLAMA_ARG_MMPROJ_URL")+ )+ if not from_argv and not from_env:+ return False+ return _resolved_mmproj_offload(tokens, env_map) is not False++def _paravirtual_mmproj_pinnable(server_caps: Mapping[str, object]) -> bool:"""True when --no-mmproj-offload can be emitted, including on an unanswered probe."""return bool(@@ -8715,17 +8736,13 @@ class LlamaCppBackend:return None@staticmethod- def _unified_memory_would_help(gpu_indices = None) -> bool:- """Whether managed allocation is the larger pool for the selected APUs.-- On HIP it draws host RAM instead of the selected APUs' carve-outs, so the- decision is a direct comparison of the two pools. Available host RAM against- the carve-out's TOTAL is deliberate: a carve-out another process is holding- is not credited back to the host side, because the two pools fail- differently. Over-asking the carve-out returns hipErrorOutOfMemory and the- load fails cleanly, while over-asking host RAM is the OOM kill this gate- exists to stop, so both halves of the asymmetry lean toward the pool a miss- is recoverable in. Missing data and mixed-device selections fail closed.+ def _unified_memory_would_help(gpu_indices = None, need_bytes = None) -> bool:+ """Whether managed allocation is worth taking for the selected APUs.++ Host RAM must be the larger pool, against the carve-out's TOTAL (over-asking+ the carve-out fails cleanly, over-asking host RAM is an OOM kill), AND the+ weights must not fit that carve-out: managed pages fault in k_set_rows on+ Linux ROCm gfx1151 (HF Qwen3.8-Flash-Next-GGUF discussion 30, #10330)."""try:pool_mib = LlamaCppBackend._rocm_selected_pool_mib(gpu_indices)@@ -8734,7 +8751,11 @@ class LlamaCppBackend:host_mib = LlamaCppBackend._available_system_memory_mib()if not host_mib:return False- return int(host_mib) > int(pool_mib)+ if int(host_mib) <= int(pool_mib):+ return False+ if need_bytes is None:+ return False+ return int(need_bytes) > int(pool_mib) * 1024 * 1024except Exception:return False@@ -8877,6 +8898,29 @@ class LlamaCppBackend:except Exception:return False+ @staticmethod+ def _unified_memory_opted_in(env = None) -> bool:+ """True when UNSLOTH_ENABLE_UNIFIED_MEMORY=1 asks for managed allocation even+ though the weights fit. Exact "1"; the opt-out wins over it."""+ try:+ source = os.environ if env is None else env+ return str(source.get("UNSLOTH_ENABLE_UNIFIED_MEMORY", "")).strip() == "1"+ except Exception:+ return False++ @staticmethod+ def _unified_memory_for_launch(+ gpu_indices,+ need_bytes,+ *,+ opted_in = False,+ ) -> bool:+ """The launch-time decision behind GGML_CUDA_ENABLE_UNIFIED_MEMORY. It is+ process-wide and hurts discrete cards, so a mixed selection is always False."""+ if opted_in and LlamaCppBackend._rocm_selected_pool_mib(gpu_indices) is not None:+ return True+ return LlamaCppBackend._unified_memory_would_help(gpu_indices, need_bytes = need_bytes)+# Datacenter / professional NVIDIA parts that benefit from the llama.cpp# FP32-accum / P2P tunings. Whole-word (\b) so short markers don't match# workstation parts as substrings: "a100" must not fire on "RTX A1000".@@ -10320,6 +10364,42 @@ class LlamaCppBackend:n_layers = self.n_layersreturn requested == -1 or (bool(n_layers) and requested > n_layers)+ def _launch_forces_full_offload(+ self,+ argv: Iterable[str],+ env: Optional[Mapping[str, str]] = None,+ ) -> bool:+ """Whether the child puts every layer on a GPU whatever the fitter says. The+ fitter cannot lower a count the user fixed (common/fit.cpp "n_gpu_layers+ already set by user", downgraded to a warning), so a count above the block+ count stands under ``--fit on``, as does no count at all with the fitter off+ (llama.cpp's default is ``-1``)."""+ args = [str(a) for a in argv or ()]+ if self._argv_offloads_every_layer(args, env):+ return True+ if _device_selection_is_cpu(args, env):+ return False+ if _args_place_tensors_on_cpu(args) or _env_places_tensors_on_cpu(env):+ return False+ try:+ requested = parse_gpu_layers_override(args)+ except ValueError:+ return False+ if requested is None and env and _env_fixes_gpu_layers(env):+ raw = str(env.get("LLAMA_ARG_N_GPU_LAYERS", "")).strip().lower()+ if raw == "all":+ return True+ try:+ requested = int(raw)+ except ValueError:+ return False+ if requested is None:+ return not fit_is_effectively_on(args, env)+ n_layers = self.n_layers+ if not n_layers:+ return False+ return requested > n_layers+@staticmethoddef _rows_the_child_can_reach(detected_gpus, pinned_ids) -> list:"""``detected_gpus`` narrowed to the cards THIS launch pinned the child to.@@ -19042,6 +19122,8 @@ class LlamaCppBackend:total_by_idx: dict[int, int] = {}_gpu_mem: list[tuple[int, int, int]] = []model_size = None # set in the fit try; used by the APU RAM guard+ _mtp_will_engage = False+ _separate_draft_launches = False # a sidecar displaces an embedded head# "none" once the fit proves the load needs no demand paging, else None# for llama.cpp's own default. Bound before the try like the verdict# flags above: the except path falls through to the launch, which reads it.@@ -22817,6 +22899,60 @@ class LlamaCppBackend:# preserve inherited values._unified_env_applied = False_unified_opt_out = self._unified_memory_opted_out(env)+ _unified_opt_in = self._unified_memory_opted_in(env)+ # The arch gate and arch-crash retry re-select without rebinding gpu_indices.+ _unified_gpu_indices = gpu_indices++ def _unified_need_now(argv = None, mtp_engages = None):+ """Bytes a forced full offload puts on the device, or None."""+ run_argv = cmd if argv is None else argv+ # The trailing blocks load only when the EMBEDDED head drafts: a+ # sidecar wins over it (llama.cpp has_dft()), wherever it lands.+ if mtp_engages is None:+ engages = _mtp_will_engage and not _separate_draft_launches+ else:+ engages = mtp_engages+ if model_size is None or not self._launch_forces_full_offload(run_argv, env):+ return None+ need = int(model_size)+ if mmproj_size and not _argv_keeps_projector_on_gpu(run_argv, env):+ need -= int(mmproj_size)+ # dev_input is CPU-pinned even at full offload, so token_embd never+ # reaches the device; a tied file duplicates the matrix onto it instead.+ layout = self._tensor_spill_layout(model_path, all_shards = True)+ if layout is None or not getattr(layout, "complete", True):+ return None+ if int(getattr(layout, "lm_head_bytes", 0) or 0):+ need -= int(getattr(layout, "token_embd_bytes", 0) or 0)+ if not engages and self._nextn_predict_layers:+ need -= int(getattr(layout, "excluded_block_bytes", 0) or 0)+ return need++ def _unified_withdraw_if_unneeded(+ run_cmd,+ *,+ mtp_engages = None,+ why = "",+ ):+ """Drop the variable THIS launch set once a respawn stops needing it."""+ nonlocal _unified_env_applied+ if not _unified_env_applied:+ return+ if self._unified_memory_for_launch(+ _unified_gpu_indices,+ _unified_need_now(argv = run_cmd, mtp_engages = mtp_engages),+ opted_in = _unified_opt_in,+ ):+ return+ env.pop("GGML_CUDA_ENABLE_UNIFIED_MEMORY", None)+ _unified_env_applied = False+ logger.info(+ "%s no longer needs managed memory; dropped "+ "GGML_CUDA_ENABLE_UNIFIED_MEMORY.",+ why or "The retry",+ )++ _unified_need = _unified_need_now()if _unified_opt_out:# ggml tests presence, so passing a user's "0" through would# ENABLE what they turned off (#8651). Only absence is off.@@ -22824,12 +22960,16 @@ class LlamaCppBackend:logger.info("Unified memory opted out: unset GGML_CUDA_ENABLE_UNIFIED_MEMORY")- elif not is_vulkan_backend and self._unified_memory_would_help(gpu_indices):+ elif not is_vulkan_backend and self._unified_memory_for_launch(+ gpu_indices, _unified_need, opted_in = _unified_opt_in+ ):_unified_env_applied = "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in envenv.setdefault("GGML_CUDA_ENABLE_UNIFIED_MEMORY", "1")logger.info(- "AMD unified-memory APU whose carve-out is smaller than host "- "RAM: set GGML_CUDA_ENABLE_UNIFIED_MEMORY=1"+ "AMD unified-memory APU: set GGML_CUDA_ENABLE_UNIFIED_MEMORY=1 (%s)",+ "UNSLOTH_ENABLE_UNIFIED_MEMORY=1"+ if _unified_opt_in+ else "the weights outgrow the carve-out and host RAM is the larger pool",)# DC NVIDIA GPUs: FP32 accum (+ P2P / launch queues for multi-GPU).@@ -22981,7 +23121,10 @@ class LlamaCppBackend:self._clear_split_placement_env(env)# The setting is process-wide, so recompute it after changing# the selected devices. Ownership protects inherited values.- _survivors_gain_unified = self._unified_memory_would_help(_survivors)+ _unified_gpu_indices = _survivors+ _survivors_gain_unified = self._unified_memory_for_launch(+ _survivors, _unified_need_now(), opted_in = _unified_opt_in+ )if _unified_env_applied and not _survivors_gain_unified:env.pop("GGML_CUDA_ENABLE_UNIFIED_MEMORY", None)_unified_env_applied = False@@ -22999,7 +23142,7 @@ class LlamaCppBackend:_unified_env_applied = Truelogger.info("Arch gate narrowed the launch onto a unified-memory "- "APU whose carve-out is smaller than host RAM; set "+ "APU the weights outgrow; set ""GGML_CUDA_ENABLE_UNIFIED_MEMORY=1.")self._emit_child_gpu_visibility(@@ -23824,7 +23967,9 @@ class LlamaCppBackend:# APU presence controls the RAM guard; relative pool sizes# determine whether managed allocations help._retry_wants_unified = self._amd_apu_wants_unified_memory(_remaining)- _retry_unified_helps = self._unified_memory_would_help(_remaining)+ _retry_unified_helps = self._unified_memory_for_launch(+ _remaining, _unified_need_now(), opted_in = _unified_opt_in+ )# Everything recorded so far priced the CRASHED selection, and# that placement is gone: the canonical #7624 shape pins the APU# whose shared-pool "free memory" outranked the dGPU, warns that@@ -23879,6 +24024,7 @@ class LlamaCppBackend:)self._kill_process()gpu_indices = _remaining+ _unified_gpu_indices = _remaining# GGML_CUDA_ENABLE_UNIFIED_MEMORY was decided for the CRASHED# set. The canonical #7624 shape crashes on the APU and retries# on the dGPU, where it is harmful, so withdraw it, but only@@ -23901,8 +24047,8 @@ class LlamaCppBackend:env["GGML_CUDA_ENABLE_UNIFIED_MEMORY"] = "1"_unified_env_applied = Truelogger.info(- "Arch-crash retry targets a unified-memory APU whose "- "carve-out is smaller than host RAM; set "+ "Arch-crash retry targets a unified-memory APU the "+ "weights outgrow; set ""GGML_CUDA_ENABLE_UNIFIED_MEMORY=1 for the respawn.")self._emit_child_gpu_visibility(@@ -24292,6 +24438,11 @@ class LlamaCppBackend:strip_split_mode = False,)fallback_cmd = cmd[:_spec_at] + ["--spec-default"] + _fb_tail+ _unified_withdraw_if_unneeded(+ fallback_cmd,+ mtp_engages = False,+ why = "The retry without speculative decoding",+ )healthy = _spawn_and_wait(fallback_cmd, label = "-retry")if healthy:self._speculative_type = "default"@@ -24375,6 +24526,9 @@ class LlamaCppBackend:"projector on CPU to preserve image input.")cmd = _cpu_projector_cmd+ _unified_withdraw_if_unneeded(+ cmd, why = "The retry with the projector on CPU"+ )healthy = _spawn_and_wait(cmd, label = "-mmproj-cpu")if healthy:self._mmproj_fallback_reason = "cpu_offload"@@ -24448,6 +24602,7 @@ class LlamaCppBackend:)self._mmproj_fallback_reason = "projector_startup_failure"cmd = self._strip_mmproj_args(_vision_gpu_cmd)+ _unified_withdraw_if_unneeded(cmd, why = "The text-only retry")# This retry bypasses _spawn_and_wait, so refresh the# launched-argv snapshot itself -- the zero-offload# classification below must not see the stripped --mmproj.@@ -26715,7 +26870,12 @@ class LlamaCppBackend:logger.info("Tensor spill: dropping the plan for the %s retry; %s", why, "using --fit on")return [*stripped, "--fit", "on"]- def _tensor_spill_layout(self, model_path: "Optional[str]") -> "Optional[ModelLayout]":+ def _tensor_spill_layout(+ self,+ model_path: "Optional[str]",+ *,+ all_shards: bool = False,+ ) -> "Optional[ModelLayout]":"""The GGUF's placement buckets, read once per path and cached.Per BLOCK, not per bucket total, so the planner can spill the minimum set@@ -26737,14 +26897,18 @@ class LlamaCppBackend:# (size, mtime_ns) stat identity _slot_launch_fingerprint uses.try:st = os.stat(model_path)- key = (model_path, st.st_size, st.st_mtime_ns)+ key = (model_path, st.st_size, st.st_mtime_ns, all_shards)except OSError:- key = (model_path, None, None)+ key = (model_path, None, None, all_shards)cached = getattr(self, "_spill_layout_cache", None)if cached is not None and cached[0] == key:return cached[1]try:- layout = layout_from_gguf(model_path)+ layout = (+ layout_from_gguf(model_path, all_shards = True)+ if all_shards+ else layout_from_gguf(model_path)+ )except Exception as e: # unreadable, truncated, or an arch we cannot bucketlogger.debug("Tensor spill: cannot read layout from %s (%s)", model_path, e)layout = Nonediff --git a/studio/backend/core/inference/offload_layout.py b/studio/backend/core/inference/offload_layout.pyindex ce885e1f3fb..bf1779a3984 100644--- a/studio/backend/core/inference/offload_layout.py+++ b/studio/backend/core/inference/offload_layout.py@@ -14,6 +14,7 @@ import loggingimport refrom dataclasses import dataclass, fieldfrom typing import Optional+import oslogger = logging.getLogger(__name__)@@ -123,33 +124,63 @@ def _field(return default-def layout_from_gguf(path: str) -> ModelLayout:+_SPLIT_SHARD_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})\.gguf$", re.IGNORECASE)+++def split_shard_paths(path: str) -> Optional[list[str]]:+ """Every shard of the split *path* belongs to, in order, or None when the name+ is not llama.cpp's ``<prefix>-NNNNN-of-MMMMM.gguf`` (llama_split_path)."""+ directory, name = os.path.split(path)+ match = _SPLIT_SHARD_RE.match(name)+ if not match:+ return None+ prefix, _index, total = match.groups()+ return [+ os.path.join(directory, f"{prefix}-{i:05d}-of-{int(total):05d}.gguf")+ for i in range(1, int(total) + 1)+ ]+++def layout_from_gguf(path: str, *, all_shards: bool = False) -> ModelLayout:"""Read ``path`` into a :class:`ModelLayout`.Returns an incomplete layout (``complete = False``) rather than raising whenanything required is missing, so a surprising GGUF makes the planner abstaininstead of failing a load that llama.cpp would have handled.+ ``all_shards`` reads every sibling shard; all of them must be present."""try:from gguf import GGUFReader- reader = GGUFReader(path)+ readers = [GGUFReader(path)]+ if all_shards and int(_field(readers[0], "split.count") or 0) > 1:+ shards = split_shard_paths(path)+ if not shards or not all(os.path.isfile(p) for p in shards):+ logger.debug("offload layout: split %s is missing a shard", path)+ return ModelLayout()+ readers = [GGUFReader(p) for p in shards]except Exception as exc:logger.debug("offload layout: cannot read %s (%s)", path, exc)return ModelLayout()try:- return _layout_from_reader(reader)+ return _layout_from_readers(readers)except Exception as exc:logger.debug("offload layout: cannot interpret %s (%s)", path, exc)return ModelLayout()def _layout_from_reader(reader) -> ModelLayout:+ return _layout_from_readers([reader])+++def _layout_from_readers(readers) -> ModelLayout:+ """One reader per shard, the first carrying the metadata."""+ reader = readers[0]# Split GGUF: llama.cpp loads every sibling shard (llama-model-loader.cpp:590-618), but GGUFReader memmaps only the# ONE path it was given. Shard 1 still carries the metadata, so the layout would look complete while undercounting# resident and spillable by most of the model -- an overstated fit, too few -ot patterns, and a startup OOM with- # --fit off. Abstain instead; the seam then reproduces --fit on exactly.- if int(_field(reader, "split.count") or 0) > 1:+ # --fit off. Abstain unless every shard was handed over; the seam then reproduces --fit on exactly.+ if (int(_field(reader, "split.count") or 0) or 1) != len(readers):return ModelLayout()arch = str(_field(reader, "general.architecture") or "")@@ -213,7 +244,7 @@ def _layout_from_reader(reader) -> ModelLayout:token_embd = 0other_resident = 0- for tensor in reader.tensors:+ for tensor in (t for r in readers for t in r.tensors):name = str(tensor.name)nbytes = int(tensor.n_bytes)match = _BLOCK_RE.match(name)diff --git a/studio/backend/tests/test_amd_apu_unified_memory.py b/studio/backend/tests/test_amd_apu_unified_memory.pyindex 65f52233ea1..265144df908 100644--- a/studio/backend/tests/test_amd_apu_unified_memory.py+++ b/studio/backend/tests/test_amd_apu_unified_memory.py@@ -420,7 +420,8 @@ _GIB = 1024**3class TestTheUnifiedMemorySwapIsOnlyTakenWhenItPays:- """Managed allocation is enabled only when host RAM is the larger pool."""+ """Managed allocation is enabled only when host RAM is the larger pool. Every arm+ prices weights that outgrow the carve-out, so only the pools decide."""def _host(self, monkeypatch, specs, ram_mib):monkeypatch.setitem(sys.modules, "torch", _fake_torch_sized(specs))@@ -430,23 +431,23 @@ class TestTheUnifiedMemorySwapIsOnlyTakenWhenItPays:def test_a_small_carve_out_still_gets_it(self, monkeypatch):self._host(monkeypatch, [("gfx1151", 16 * _GIB)], 117_000)- assert LlamaCppBackend._unified_memory_would_help() is True+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is Truedef test_a_carve_out_larger_than_host_ram_does_not(self, monkeypatch):self._host(monkeypatch, [("gfx1151", 64 * _GIB)], 58_880)- assert LlamaCppBackend._unified_memory_would_help() is False+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is Falsedef test_an_exact_tie_does_not(self, monkeypatch):self._host(monkeypatch, [("gfx1151", 32 * _GIB)], 32 * 1024)- assert LlamaCppBackend._unified_memory_would_help() is False+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is Falsedef test_a_discrete_card_is_never_offered_it(self, monkeypatch):self._host(monkeypatch, [("gfx1100", 16 * _GIB)], 117_000)- assert LlamaCppBackend._unified_memory_would_help() is False+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is Falsedef test_unreadable_host_ram_fails_closed(self, monkeypatch):self._host(monkeypatch, [("gfx1151", 16 * _GIB)], None)- assert LlamaCppBackend._unified_memory_would_help() is False+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is Falsedef test_an_unreported_pool_size_fails_closed(self, monkeypatch):monkeypatch.setitem(sys.modules, "torch", _fake_torch("6.2.0", ["gfx1151"]))@@ -454,41 +455,149 @@ class TestTheUnifiedMemorySwapIsOnlyTakenWhenItPays:LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 117_000))assert LlamaCppBackend._rocm_selected_pool_mib() is None- assert LlamaCppBackend._unified_memory_would_help() is False+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is Falsedef test_a_missing_torch_fails_closed(self, monkeypatch):monkeypatch.setitem(sys.modules, "torch", None)assert LlamaCppBackend._rocm_selected_pool_mib() is None- assert LlamaCppBackend._unified_memory_would_help() is False+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is Falsedef test_two_apus_are_weighed_together(self, monkeypatch):self._host(monkeypatch, [("gfx1151", 8 * _GIB), ("gfx1150", 24 * _GIB)], 40_000)assert LlamaCppBackend._rocm_selected_pool_mib() == 32 * 1024- assert LlamaCppBackend._unified_memory_would_help() is True+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is Trueself._host(monkeypatch, [("gfx1151", 8 * _GIB), ("gfx1150", 24 * _GIB)], 30_000)- assert LlamaCppBackend._unified_memory_would_help() is False+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is Falsedef test_the_answer_is_scoped_to_the_selected_gpu(self, monkeypatch):self._host(monkeypatch, [("gfx1151", 8 * _GIB), ("gfx1150", 64 * _GIB)], 40_000)- assert LlamaCppBackend._unified_memory_would_help([0]) is True- assert LlamaCppBackend._unified_memory_would_help([1]) is False+ assert LlamaCppBackend._unified_memory_would_help([0], need_bytes = 100 * _GIB) is True+ assert LlamaCppBackend._unified_memory_would_help([1], need_bytes = 100 * _GIB) is Falsedef test_a_discrete_card_in_the_selection_fails_closed(self, monkeypatch):self._host(monkeypatch, [("gfx1151", 8 * _GIB), ("gfx1100", 64 * _GIB)], 40_000)assert LlamaCppBackend._rocm_selected_pool_mib() is None- assert LlamaCppBackend._unified_memory_would_help() is False+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is Falsedef test_the_same_mixed_host_pinned_to_the_apu_still_gains(self, monkeypatch):self._host(monkeypatch, [("gfx1151", 8 * _GIB), ("gfx1100", 64 * _GIB)], 40_000)assert LlamaCppBackend._rocm_selected_pool_mib([0]) == 8 * 1024- assert LlamaCppBackend._unified_memory_would_help([0]) is True+ assert LlamaCppBackend._unified_memory_would_help([0], need_bytes = 100 * _GIB) is Truedef test_an_id_this_host_does_not_enumerate_fails_closed(self, monkeypatch):self._host(monkeypatch, [("gfx1151", 8 * _GIB)], 117_000)assert LlamaCppBackend._rocm_selected_pool_mib([0, 3]) is None- assert LlamaCppBackend._unified_memory_would_help([0, 3]) is False+ assert LlamaCppBackend._unified_memory_would_help([0, 3], need_bytes = 100 * _GIB) is Falsedef test_an_empty_selection_fails_closed(self, monkeypatch):self._host(monkeypatch, [("gfx1151", 8 * _GIB)], 117_000)assert LlamaCppBackend._rocm_selected_pool_mib([]) is None- assert LlamaCppBackend._unified_memory_would_help([]) is False+ assert LlamaCppBackend._unified_memory_would_help([], need_bytes = 100 * _GIB) is False+++class TestManagedMemoryIsTakenOnlyWhenTheWeightsOutgrowTheCarveOut:+ """The second condition: managed pages fault in k_set_rows on Linux ROCm gfx1151+ (HF Qwen3.8-Flash-Next-GGUF discussion 30, #10330), so a fitting load never takes them."""++ def _strix_halo(+ self,+ monkeypatch,+ carve_out = 64 * _GIB,+ ram_mib = 117_000,+ ):+ monkeypatch.setitem(sys.modules, "torch", _fake_torch_sized([("gfx1151", carve_out)]))+ monkeypatch.setattr(+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: ram_mib)+ )++ def test_weights_that_fit_the_carve_out_do_not_get_it(self, monkeypatch):+ self._strix_halo(monkeypatch)+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 45 * _GIB) is False++ def test_weights_that_outgrow_it_do(self, monkeypatch):+ self._strix_halo(monkeypatch)+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 100 * _GIB) is True++ def test_weights_exactly_the_carve_out_do_not(self, monkeypatch):+ self._strix_halo(monkeypatch)+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 64 * _GIB) is False++ def test_one_byte_over_does(self, monkeypatch):+ """Bytes against bytes: flooring the need to MiB hid a sub-MiB overrun."""+ self._strix_halo(monkeypatch)+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 64 * _GIB + 1) is True++ def test_unpriced_weights_fail_closed(self, monkeypatch):+ self._strix_halo(monkeypatch)+ assert LlamaCppBackend._unified_memory_would_help() is False+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = None) is False++ def test_the_pool_comparison_still_comes_first(self, monkeypatch):+ self._strix_halo(monkeypatch, carve_out = 96 * _GIB, ram_mib = 30_000)+ assert LlamaCppBackend._unified_memory_would_help(need_bytes = 120 * _GIB) is False+++class TestTheEnableSwitch:+ """UNSLOTH_ENABLE_UNIFIED_MEMORY=1 takes managed allocation even when weights fit."""++ @pytest.mark.parametrize(+ "env, expected",+ [+ ({"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"}, True),+ ({"UNSLOTH_ENABLE_UNIFIED_MEMORY": " 1 "}, True),+ ({"UNSLOTH_ENABLE_UNIFIED_MEMORY": "0"}, False),+ ({"UNSLOTH_ENABLE_UNIFIED_MEMORY": "yes"}, False),+ ({}, False),+ ],+ )+ def test_opt_in_decisions(self, env, expected):+ assert LlamaCppBackend._unified_memory_opted_in(env) is expected++ def test_none_reads_the_process_env(self, monkeypatch):+ monkeypatch.delenv("UNSLOTH_ENABLE_UNIFIED_MEMORY", raising = False)+ assert LlamaCppBackend._unified_memory_opted_in() is False+ monkeypatch.setenv("UNSLOTH_ENABLE_UNIFIED_MEMORY", "1")+ assert LlamaCppBackend._unified_memory_opted_in() is True++ def test_a_hostile_env_fails_closed(self):+ class _Exploding:+ def get(self, *_a, **_k):+ raise RuntimeError("no")++ assert LlamaCppBackend._unified_memory_opted_in(_Exploding()) is False++ def _strix_halo(self, monkeypatch):+ monkeypatch.setitem(sys.modules, "torch", _fake_torch_sized([("gfx1151", 64 * _GIB)]))+ monkeypatch.setattr(+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 117_000)+ )++ def test_the_switch_takes_it_for_weights_that_fit(self, monkeypatch):+ self._strix_halo(monkeypatch)+ assert LlamaCppBackend._unified_memory_for_launch([0], 45 * _GIB) is False+ assert LlamaCppBackend._unified_memory_for_launch([0], 45 * _GIB, opted_in = True) is True++ def test_the_switch_never_reaches_a_discrete_card(self, monkeypatch):+ monkeypatch.setitem(sys.modules, "torch", _fake_torch_sized([("gfx1100", 16 * _GIB)]))+ monkeypatch.setattr(+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 117_000)+ )+ assert LlamaCppBackend._unified_memory_for_launch([0], 45 * _GIB, opted_in = True) is False++ def test_the_switch_refuses_a_mixed_selection(self, monkeypatch):+ monkeypatch.setitem(+ sys.modules,+ "torch",+ _fake_torch_sized([("gfx1151", 64 * _GIB), ("gfx1100", 16 * _GIB)]),+ )+ monkeypatch.setattr(+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: 117_000)+ )+ assert LlamaCppBackend._unified_memory_for_launch([0, 1], 45 * _GIB, opted_in = True) is False+ assert LlamaCppBackend._unified_memory_for_launch(None, 45 * _GIB, opted_in = True) is False+ assert LlamaCppBackend._unified_memory_for_launch([0], 45 * _GIB, opted_in = True) is True++ def test_without_the_switch_the_priced_gate_decides(self, monkeypatch):+ self._strix_halo(monkeypatch)+ assert LlamaCppBackend._unified_memory_for_launch([0], 100 * _GIB) is True+ assert LlamaCppBackend._unified_memory_for_launch([0], None) is Falsediff --git a/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py b/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.pyindex ef3d6ad3703..159647d278b 100644--- a/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py+++ b/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py@@ -770,6 +770,15 @@ class TestArchStringRobustness:) == frozenset({"gfx1030"})+def _priced_layout(**fields):+ """A readable layout: untied embeddings, nothing CPU-pinned, no MTP blocks."""+ from core.inference.offload_layout import ModelLayout++ values = {"complete": True, "lm_head_bytes": 1}+ values.update(fields)+ return ModelLayout(**values)++def _run_auto_load(monkeypatch,tmp_path,@@ -785,6 +794,8 @@ def _run_auto_load(apu_ram_stub = None,host_offload_stub = None,backend = None,+ mmproj_bytes = 0,+ server_caps = None,):"""Drive a real automatic (no explicit GPU pick) llama-server load with the real``_get_gpu_memory`` behind it, and return the spawned (cmd, env) list.@@ -817,8 +828,17 @@ def _run_auto_load(# model_bytes drives the placement decision: a model no GPU can hold makes# _select_gpus return (None, True), so `--fit on` owns placement.backend._get_gguf_size_bytes = lambda _path: model_bytes- backend._mmproj_vram_bytes = lambda _path: 0- backend._resolve_launch_mmproj_path = lambda **_kwargs: None+ if mmproj_bytes:+ mmproj = tmp_path / "mmproj.gguf"+ mmproj.write_bytes(b"GGUF")+ backend._mmproj_vram_bytes = lambda _path: mmproj_bytes+ backend._resolve_launch_mmproj_path = lambda **_kwargs: str(mmproj)+ else:+ backend._mmproj_vram_bytes = lambda _path: 0+ backend._resolve_launch_mmproj_path = lambda **_kwargs: None+ # A header-only GGUF prices nothing, so hand the load a readable layout.+ if "_tensor_spill_layout" not in vars(backend):+ backend._tensor_spill_layout = lambda _path, **_kw: _priced_layout()# Off by default: the APU RAM preflight is not what most of these cells are# about. A test that IS about it passes its own recording stub.backend._apu_ram_shortfall_message = apu_ram_stub or (lambda *_args, **_kwargs: None)@@ -826,7 +846,7 @@ def _run_auto_load(backend._host_offload_shortfall_message = host_offload_stub or (lambda *_args, **_kwargs: None)backend._find_llama_server_binary = lambda include_denied = False: binarybackend._fit_off_retry_eligible = lambda *_args, **_kwargs: False- backend.probe_server_capabilities = lambda _binary: {"found": True}+ backend.probe_server_capabilities = lambda _binary: {"found": True, **(server_caps or {})}backend._record_server_pid = lambda _pid: Nonebackend._clear_server_pid = lambda: None# env_extra seeds the child env the way an inherited / user-set variable@@ -1144,7 +1164,7 @@ class TestArchCrashRetryEnv:)return _fake_torch([- _device("gfx1151", free_mib = 40000, is_integrated = 1),+ _device("gfx1151", free_mib = 40000, total_bytes = 8 * GIB, is_integrated = 1),_device("gfx1030", free_mib = 12049),],vendor = "amd",@@ -1164,6 +1184,7 @@ class TestArchCrashRetryEnv:None, # no marker: the proactive gate fails open, so the crash path runsreturncode = 1,output = "ROCm error: device kernel image is invalid",+ model_bytes = 10 * GIB, # outgrows the APU's 8 GiB carve-out)# The APU is pinned first, so the crashed spawn carries the env and the# respawns are masked onto the discrete card. The unrelated --fit off retry@@ -1715,6 +1736,7 @@ class TestGatedNarrowingDropsUnifiedMemory:["gfx1151"], # covers the APU, not the gfx1200 dGPUreturncode = None,model_bytes = 400 * 1024**3,+ env_extra = {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"}, # the fitter owns the layers)_cmd, env = launches[0]assert _visibility(env) == {"ROCR_VISIBLE_DEVICES": "0", "CUDA_VISIBLE_DEVICES": "0"}@@ -1911,7 +1933,7 @@ class TestArchCrashRetryOntoAnApu:return _fake_torch([_device("gfx1030", free_mib = 40000),- _device("gfx1151", free_mib = 12000, is_integrated = 1),+ _device("gfx1151", free_mib = 12000, total_bytes = 8 * GIB, is_integrated = 1),],vendor = "amd",)@@ -1925,6 +1947,7 @@ class TestArchCrashRetryOntoAnApu:None, # no marker: the proactive gate fails open, so the crash path runsreturncode = 1,output = "ROCm error: device kernel image is invalid",+ model_bytes = 10 * GIB, # outgrows the APU's 8 GiB carve-out)assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in launches[0][1]_retry = [env for _c, env in launches if env.get("ROCR_VISIBLE_DEVICES") == "1"]@@ -1957,21 +1980,312 @@ class TestUnifiedMemoryOptOut:tmp_path,monkeypatch,env_extra = None,+ model_bytes = 40 * GIB, # outgrows the fake 32 GiB carve-out+ backend = None,+ extra_args = None,+ returncode = None,+ output = "",+ intent_extra = None,+ mmproj_bytes = 0,+ server_caps = None,):+ # Manual mode above the block count: the only launch that needs managed pages.+ backend = backend or LlamaCppBackend()+ backend._n_layers = 8+ intent_kwargs = {"gpu_memory_mode": "manual", "gpu_layers": 9}+ if extra_args:+ intent_kwargs["extra_args"] = list(extra_args)+ intent_kwargs.update(intent_extra or {})return _run_auto_load(monkeypatch,tmp_path,self._strix_halo(monkeypatch),None,- returncode = None,+ returncode = returncode,+ output = output,env_extra = env_extra,+ model_bytes = model_bytes,+ backend = backend,+ intent_kwargs = intent_kwargs,+ mmproj_bytes = mmproj_bytes,+ server_caps = server_caps,)- def test_the_apu_still_gets_it_by_default(self, tmp_path, monkeypatch, probe_env):+ def test_a_forced_full_offload_that_outgrows_the_carve_out_gets_it(+ self, tmp_path, monkeypatch, probe_env+ ):"""Baseline: #5301 added the variable for exactly this hardware."""_cmd, env = self._load(tmp_path, monkeypatch)[0]+ assert "--gpu-layers" in _cmd and _cmd[_cmd.index("--gpu-layers") + 1] == "9"+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"++ def test_weights_that_fit_the_carve_out_keep_it_unset(self, tmp_path, monkeypatch, probe_env):+ """Managed pages fault Qwen3.8-Flash-Next on Linux gfx1151 (#10330)."""+ _cmd, env = self._load(tmp_path, monkeypatch, model_bytes = 20 * GIB)[0]+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ def test_the_enable_switch_takes_it_for_a_fitting_model(self, tmp_path, monkeypatch, probe_env):+ _cmd, env = self._load(+ tmp_path, monkeypatch, {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"}, model_bytes = 20 * GIB+ )[0]+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"++ def test_a_manual_partial_placement_never_takes_it(self, tmp_path, monkeypatch, probe_env):+ launches = _run_auto_load(+ monkeypatch,+ tmp_path,+ self._strix_halo(monkeypatch),+ None,+ returncode = None,+ model_bytes = 400 * GIB,+ intent_kwargs = {"gpu_memory_mode": "manual", "gpu_layers": 1},+ )+ _cmd, env = launches[0]+ assert "--gpu-layers" in _cmd and _cmd[_cmd.index("--gpu-layers") + 1] == "1"+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ def test_the_fitter_owning_the_layer_count_never_takes_it(+ self, tmp_path, monkeypatch, probe_env+ ):+ launches = _run_auto_load(+ monkeypatch,+ tmp_path,+ self._strix_halo(monkeypatch),+ None,+ returncode = None,+ model_bytes = 400 * GIB,+ )+ _cmd, env = launches[0]+ assert "--fit" in _cmd and _cmd[_cmd.index("--fit") + 1] == "on"+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ def _auto_mode_with(+ self,+ tmp_path,+ monkeypatch,+ *,+ extra_args = None,+ env_extra = None,+ ):+ backend = LlamaCppBackend()+ backend._n_layers = 8+ return _run_auto_load(+ monkeypatch,+ tmp_path,+ self._strix_halo(monkeypatch),+ None,+ returncode = None,+ model_bytes = 40 * GIB,+ backend = backend,+ intent_kwargs = {"extra_args": list(extra_args or [])},+ env_extra = env_extra,+ )[0]++ def test_a_user_count_above_the_block_count_stands_under_the_fitter(+ self, tmp_path, monkeypatch, probe_env+ ):+ _cmd, env = self._auto_mode_with(tmp_path, monkeypatch, extra_args = ["--gpu-layers", "9"])+ assert "--fit" in _cmd and _cmd[_cmd.index("--fit") + 1] == "on"+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"++ def test_an_inherited_count_above_the_block_count_stands_too(+ self, tmp_path, monkeypatch, probe_env+ ):+ _cmd, env = self._auto_mode_with(+ tmp_path, monkeypatch, env_extra = {"LLAMA_ARG_N_GPU_LAYERS": "9"}+ )+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"++ def test_a_user_minus_one_still_leaves_the_fitter_in_charge(+ self, tmp_path, monkeypatch, probe_env+ ):+ _cmd, env = self._auto_mode_with(tmp_path, monkeypatch, extra_args = ["-ngl", "-1"])+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ def test_a_user_fit_off_with_no_count_is_a_full_offload(self, tmp_path, monkeypatch, probe_env):+ _cmd, env = self._auto_mode_with(tmp_path, monkeypatch, extra_args = ["--fit", "off"])+ assert "--gpu-layers" not in _cmd and "-ngl" not in _cmd+ assert _cmd[len(_cmd) - 1 - _cmd[::-1].index("--fit") + 1] == "off"+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"++ def test_a_user_fit_off_with_a_partial_count_is_not(self, tmp_path, monkeypatch, probe_env):+ _cmd, env = self._auto_mode_with(+ tmp_path, monkeypatch, extra_args = ["--fit", "off", "--gpu-layers", "4"]+ )+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ def test_a_user_count_below_the_block_count_never_takes_it(+ self, tmp_path, monkeypatch, probe_env+ ):+ _cmd, env = self._auto_mode_with(tmp_path, monkeypatch, extra_args = ["--gpu-layers", "4"])+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ def _with_unloaded_mtp_blocks(self, excluded_bytes):+ backend = LlamaCppBackend()+ backend._nextn_predict_layers = 1+ backend._tensor_spill_layout = lambda _path, **_kw: (+ None+ if excluded_bytes is None+ else types.SimpleNamespace(excluded_block_bytes = excluded_bytes)+ )+ return backend++ def test_unloaded_mtp_blocks_are_not_priced(self, tmp_path, monkeypatch, probe_env):+ _cmd, env = self._load(+ tmp_path, monkeypatch, backend = self._with_unloaded_mtp_blocks(10 * GIB)+ )[0]+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ def test_unloaded_mtp_blocks_still_leave_an_oversized_base(+ self, tmp_path, monkeypatch, probe_env+ ):+ _cmd, env = self._load(+ tmp_path, monkeypatch, backend = self._with_unloaded_mtp_blocks(4 * GIB)+ )[0]+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"++ def test_the_drafterless_retry_reprices_without_the_mtp_blocks(+ self, tmp_path, monkeypatch, probe_env+ ):+ launches = self._load(+ tmp_path,+ monkeypatch,+ backend = self._with_unloaded_mtp_blocks(10 * GIB),+ extra_args = ["--spec-type", "draft-mtp"],+ returncode = 1,+ output = "failed to create llama_context",+ )+ first_cmd, first_env = launches[0]+ assert "draft-mtp" in first_cmd, first_cmd+ assert first_env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"+ retries = [(c, e) for c, e in launches if "--spec-default" in c and "draft-mtp" not in c]+ assert retries, [c for c, _e in launches]+ assert all("GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in e for _c, e in retries)++ def _with_layout(self, **fields):+ backend = LlamaCppBackend()+ backend._tensor_spill_layout = lambda _path, **_kw: _priced_layout(**fields)+ return backend++ def test_cpu_pinned_embeddings_are_not_priced(self, tmp_path, monkeypatch, probe_env):+ _cmd, env = self._load(+ tmp_path, monkeypatch, backend = self._with_layout(token_embd_bytes = 10 * GIB)+ )[0]+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ def test_tied_embeddings_still_reach_the_device(self, tmp_path, monkeypatch, probe_env):+ _cmd, env = self._load(+ tmp_path,+ monkeypatch,+ backend = self._with_layout(token_embd_bytes = 10 * GIB, lm_head_bytes = 0),+ )[0]assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"+ def test_small_embeddings_leave_an_oversized_base(self, tmp_path, monkeypatch, probe_env):+ _cmd, env = self._load(+ tmp_path, monkeypatch, backend = self._with_layout(token_embd_bytes = 4 * GIB)+ )[0]+ assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"++ def test_an_incomplete_layout_cannot_price_the_file(self, tmp_path, monkeypatch, probe_env):+ _cmd, env = self._load(tmp_path, monkeypatch, backend = self._with_layout(complete = False))[0]+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ _PROJECTOR_OOM = (+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 4096.00 MiB on "+ "device 0: cudaMalloc failed: out of memory"+ )++ def _vision_launches(+ self,+ tmp_path,+ monkeypatch,+ server_caps = None,+ ):+ """30 GiB base + 4 GiB projector outgrow the 32 GiB carve-out; the base does not."""+ launches = self._load(+ tmp_path,+ monkeypatch,+ model_bytes = 30 * GIB,+ mmproj_bytes = 4 * GIB,+ intent_extra = {"is_vision": True, "mmproj_path": str(tmp_path / "mmproj.gguf")},+ server_caps = server_caps,+ returncode = 1,+ output = self._PROJECTOR_OOM,+ )+ first_cmd, first_env = launches[0]+ assert "--mmproj" in first_cmd and "--no-mmproj-offload" not in first_cmd, first_cmd+ assert first_env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"+ return launches++ def test_the_cpu_projector_retry_reprices_without_its_bytes(+ self, tmp_path, monkeypatch, probe_env+ ):+ launches = self._vision_launches(+ tmp_path, monkeypatch, server_caps = {"supports_no_mmproj_offload": True}+ )+ cpu_pinned = [e for c, e in launches if "--no-mmproj-offload" in c]+ assert cpu_pinned, [c for c, _e in launches]+ assert all("GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in e for e in cpu_pinned)++ def test_the_text_only_retry_reprices_without_the_projector(+ self, tmp_path, monkeypatch, probe_env+ ):+ launches = self._vision_launches(tmp_path, monkeypatch)+ text_only = [e for c, e in launches if "--mmproj" not in c]+ assert text_only, [c for c, _e in launches]+ assert all("GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in e for e in text_only)++ def test_an_inherited_projector_outlives_the_text_only_retry(+ self, tmp_path, monkeypatch, probe_env+ ):+ inherited = tmp_path / "inherited-mmproj.gguf"+ inherited.write_bytes(b"GGUF")+ launches = self._load(+ tmp_path,+ monkeypatch,+ env_extra = {"LLAMA_ARG_MMPROJ": str(inherited)},+ model_bytes = 30 * GIB,+ mmproj_bytes = 4 * GIB,+ intent_extra = {"is_vision": True, "mmproj_path": str(tmp_path / "mmproj.gguf")},+ returncode = 1,+ output = self._PROJECTOR_OOM,+ )+ first_cmd, first_env = launches[0]+ assert "--mmproj" in first_cmd, first_cmd+ assert first_env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"+ text_only = [e for c, e in launches if "--mmproj" not in c]+ assert text_only, [c for c, _e in launches]+ assert all(e.get("LLAMA_ARG_MMPROJ") == str(inherited) for e in text_only)+ assert all(e.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1" for e in text_only)++ def test_a_sidecar_drafter_displaces_the_embedded_head(self, tmp_path, monkeypatch, probe_env):+ sidecar = tmp_path / "draft.gguf"+ sidecar.write_bytes(b"GGUF")+ _cmd, env = self._load(+ tmp_path,+ monkeypatch,+ backend = self._with_unloaded_mtp_blocks(10 * GIB),+ extra_args = ["--spec-type", "draft-mtp", "-md", str(sidecar)],+ )[0]+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ def test_an_unreadable_tensor_table_cannot_price_the_blocks(+ self, tmp_path, monkeypatch, probe_env+ ):+ _cmd, env = self._load(tmp_path, monkeypatch, backend = self._with_unloaded_mtp_blocks(None))[+ 0+ ]+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env++ def test_the_disable_switch_beats_the_enable_switch(self, tmp_path, monkeypatch, probe_env):+ _cmd, env = self._load(+ tmp_path,+ monkeypatch,+ {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1", "UNSLOTH_DISABLE_UNIFIED_MEMORY": "1"},+ )[0]+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env+@pytest.mark.parametrize("value", ["0", "", "false", "FALSE", "no", "off", " 0 "])def test_a_falsy_user_value_is_not_passed_through(self, tmp_path, monkeypatch, probe_env, value@@ -2954,9 +3268,20 @@ class TestTheCarveOutDecidesTheUnifiedMemoryEnv:monkeypatch,torch,env_extra = None,+ model_bytes = 40 * GIB, # outgrows every carve-out below):+ backend = LlamaCppBackend()+ backend._n_layers = 8return _run_auto_load(- monkeypatch, tmp_path, torch, None, returncode = None, env_extra = env_extra+ monkeypatch,+ tmp_path,+ torch,+ None,+ returncode = None,+ env_extra = env_extra,+ model_bytes = model_bytes,+ backend = backend,+ intent_kwargs = {"gpu_memory_mode": "manual", "gpu_layers": 9},)def test_a_small_carve_out_gets_it(self, tmp_path, monkeypatch, probe_env):@@ -3001,6 +3326,26 @@ class TestTheCarveOutDecidesTheUnifiedMemoryEnv:vendor = "amd",)+ def test_the_enable_switch_cannot_reach_a_mixed_selection(+ self, tmp_path, monkeypatch, probe_env+ ):+ torch = self._apu_and_dgpu(monkeypatch, 40_000)+ _cmd, env = _run_auto_load(+ monkeypatch,+ tmp_path,+ torch,+ None,+ returncode = None,+ intent_kwargs = {+ "gpu_ids": (0, 1),+ "gpu_memory_mode": "manual",+ "gpu_layers": 1,+ },+ env_extra = {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"},+ )[0]+ assert _visibility(env) == {"ROCR_VISIBLE_DEVICES": "0,1", "CUDA_VISIBLE_DEVICES": "0,1"}+ assert "GGML_CUDA_ENABLE_UNIFIED_MEMORY" not in env+def test_a_discrete_card_in_the_launch_keeps_it_off(self, tmp_path, monkeypatch, probe_env):torch = self._apu_and_dgpu(monkeypatch, 40_000)_cmd, env = _run_auto_load(@@ -3029,11 +3374,37 @@ class TestTheCarveOutDecidesTheUnifiedMemoryEnv:["gfx1151"], # covers the APU, not the discrete gfx1100returncode = None,model_bytes = 400 * GIB,+ env_extra = {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"},)_cmd, env = launches[0]assert _visibility(env) == {"ROCR_VISIBLE_DEVICES": "0", "CUDA_VISIBLE_DEVICES": "0"}assert env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"+ def test_the_narrowed_selection_is_what_a_later_retry_reprices(+ self, tmp_path, monkeypatch, probe_env+ ):+ torch = self._apu_and_dgpu(monkeypatch, 40_000)+ backend = LlamaCppBackend()+ backend._nextn_predict_layers = 1+ backend._tensor_spill_layout = lambda _path, **_kw: _priced_layout()+ launches = _run_auto_load(+ monkeypatch,+ tmp_path,+ torch,+ ["gfx1151"],+ returncode = 1,+ output = "failed to create llama_context",+ model_bytes = 400 * GIB,+ env_extra = {"UNSLOTH_ENABLE_UNIFIED_MEMORY": "1"},+ intent_kwargs = {"extra_args": ["--spec-type", "draft-mtp"]},+ backend = backend,+ )+ first_cmd, first_env = launches[0]+ assert "draft-mtp" in first_cmd and first_env.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1"+ retries = [(c, e) for c, e in launches if "--spec-default" in c and "draft-mtp" not in c]+ assert retries, [c for c, _e in launches]+ assert all(e.get("GGML_CUDA_ENABLE_UNIFIED_MEMORY") == "1" for _c, e in retries)+def test_the_opt_out_survives_that_narrowing(self, tmp_path, monkeypatch, probe_env):torch = self._apu_and_dgpu(monkeypatch, 40_000)launches = _run_auto_load(diff --git a/studio/backend/tests/test_offload_planner.py b/studio/backend/tests/test_offload_planner.pyindex 1338da59333..965eb6d40d9 100644--- a/studio/backend/tests/test_offload_planner.py+++ b/studio/backend/tests/test_offload_planner.py@@ -22,7 +22,9 @@ from core.inference.offload_layout import (BlockLayout,ModelLayout,_layout_from_reader,+ _layout_from_readers,layout_from_gguf,+ split_shard_paths,spill_pattern_for,)from core.inference.offload_cost_model import HostProfile@@ -1490,3 +1492,60 @@ def test_a_wrong_length_vector_is_ignored_rather_than_trusted():kv_layer_weights = [1, 2, 3],)assert got is not None and "sliding-window" in got+++def test_every_shard_handed_over_prices_the_whole_split():+ """Two readers summing to one layout, token_embd in its bucket whichever shard holds it."""+ first = _StubReader(+ _shard_fields(**{"split.count": 2, "split.no": 0}),+ _shard_tensors(range(32)) + [_StubTensor("output.weight", GIB)],+ )+ second = _StubReader(+ _shard_fields(**{"split.count": 2, "split.no": 1}),+ _shard_tensors(range(32, 64)) + [_StubTensor("token_embd.weight", 3 * GIB)],+ )+ layout = _layout_from_readers([first, second])+ assert layout.complete+ assert len(layout.blocks) == 64+ assert layout.token_embd_bytes == 3 * GIB+ assert layout.lm_head_bytes == GIB+++def test_a_partial_shard_set_still_abstains():+ readers = [+ _StubReader(_shard_fields(**{"split.count": 3}), _shard_tensors(range(32))),+ _StubReader(_shard_fields(**{"split.count": 3}), _shard_tensors(range(32, 64))),+ ]+ assert not _layout_from_readers(readers).complete+++def test_split_shard_paths_follow_llama_cpp_naming():+ paths = split_shard_paths("/m/model-Q4-00002-of-00003.gguf")+ assert paths == [+ "/m/model-Q4-00001-of-00003.gguf",+ "/m/model-Q4-00002-of-00003.gguf",+ "/m/model-Q4-00003-of-00003.gguf",+ ]+ assert split_shard_paths("/m/model-Q4.gguf") is None+++def test_layout_from_gguf_reads_the_sibling_shards_only_when_asked(tmp_path, monkeypatch):+ import gguf++ shards = [tmp_path / f"m-{i:05d}-of-00002.gguf" for i in (1, 2)]+ for shard in shards:+ shard.write_bytes(b"GGUF")+ by_path = {+ str(shards[0]): _StubReader(_shard_fields(**{"split.count": 2}), _shard_tensors(range(32))),+ str(shards[1]): _StubReader(+ _shard_fields(**{"split.count": 2}), _shard_tensors(range(32, 64))+ ),+ }+ monkeypatch.setattr(gguf, "GGUFReader", lambda path: by_path[str(path)])++ assert not layout_from_gguf(str(shards[0])).complete+ whole = layout_from_gguf(str(shards[0]), all_shards = True)+ assert whole.complete and len(whole.blocks) == 64++ shards[1].unlink()+ assert not layout_from_gguf(str(shards[0]), all_shards = True).complete