Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Unsloth Studio (Vulkan): fill the discrete GPU before a shared-memory iGPU in a mixed pin (#12650)
Leo Borcherding3f11b2eb51af
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex dad0ef61894..e13200090ae 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -8821,6 +8821,8 @@ class LlamaCppBackend:# The "--load-mode none" tokens the FIT emitted, so paths that replace the# placement can take them back out. Empty for a mode the user asked for.self._fit_load_mode_flags: list[str] = []+ # Mixed-pin --tensor-split tokens; the --fit on retry strips them (fit.cpp aborts on a set split).+ self._mixed_split_flags: list[str] = []# The pair a launch in flight is committed to, None when none is. ONE attribute:# a separate marker and snapshot read out of step answered a save with# reload_required=false about a child already committed to the pre-save flags.@@ -10143,6 +10145,189 @@ class LlamaCppBackend:rendered = _render(v / peak for v in values)return ",".join(rendered)+ @staticmethod+ def _discrete_first_split(+ gpu_indices: List[int],+ usable_mib: dict[int, float],+ shared_gpu_ids: Iterable[int],+ layer_mib: List[float],+ per_device_mib: float = 0.0,+ main_reserve_mib: float = 0.0,+ pipeline_mib: float = 0.0,+ ) -> Optional[List[float]]:+ """Layer-count ``--tensor-split`` shares over ``gpu_indices``: discrete cards first,+ iGPUs (whose "free" is the host pool) take the overflow. llama.cpp cuts contiguous runs+ by COUNT (get_layer_buft_list). None unless the pin mixes both kinds and every run fits."""+ shared = set(shared_gpu_ids)+ discrete = [i for i in gpu_indices if i not in shared]+ igpus = [i for i in gpu_indices if i in shared]+ n = len(layer_mib)+ if not discrete or not igpus or n == 0:+ return None+ room = {+ i: usable_mib.get(i, 0.0)+ - per_device_mib+ - (main_reserve_mib if k == 0 else pipeline_mib)+ for k, i in enumerate(gpu_indices)+ }+ prefix = [0.0]+ for mib in layer_mib:+ prefix.append(prefix[-1] + mib)++ def most_layers(cap: float) -> int:+ k = 0+ while k < n and max(prefix[j + k + 1] - prefix[j] for j in range(n - k)) <= cap:+ k += 1+ return k++ counts: dict[int, float] = {}+ left = n+ for i in sorted(discrete, key = lambda d: room[d], reverse = True):+ counts[i] = min(most_layers(room[i]), left)+ left -= counts[i]+ if left == n:+ return None+ igpu_room = {i: max(0.0, room[i]) for i in igpus}+ igpu_total = sum(igpu_room.values())+ exact = {+ i: (left * igpu_room[i] / igpu_total if igpu_total > 0 else left / len(igpus))+ for i in igpus+ }+ for i in igpus:+ counts[i] = int(exact[i])+ for i in sorted(igpus, key = lambda d: exact[d] - counts[d], reverse = True)[+ : left - sum(counts[i] for i in igpus)+ ]:+ counts[i] += 1+ # The runs llama.cpp will really cut, in device order: each must fit its room.+ start = 0+ for i in gpu_indices:+ end = start + int(counts[i])+ if prefix[end] - prefix[start] > max(0.0, room[i]):+ return None+ start = end+ shares = [float(counts[i]) for i in gpu_indices]+ # Boundaries half a layer early: float rounding in upper_bound cannot add a layer.+ first = next(k for k, v in enumerate(shares) if v > 0)+ if first != len(shares) - 1:+ shares[first] -= 0.5+ shares[-1] += 0.5+ return shares++ def _mixed_pin_split(+ self,+ gpu_indices: List[int],+ spill_inputs: Optional[dict],+ shared_gpu_ids: Iterable[int],+ extra_args: Optional[Iterable[str]],+ env: Mapping[str, str],+ *,+ layer_min_gpus: int,+ tensor_parallel: bool,+ pin_owns_devices: bool = False,+ ) -> Optional[List[float]]:+ """The discrete-first split for a full-offload mixed Vulkan pin, or None to+ leave llama.cpp's own split (the pre-existing behaviour) wherever its inputs+ cannot be priced or someone else owns the placement."""+ if (+ not shared_gpu_ids+ or spill_inputs is None+ or tensor_parallel+ # A downgraded tensor request keeps every device in use.+ or layer_min_gpus > 1+ # A GPU-resident separate drafter spreads over every device.+ or spill_inputs["separate_draft_on_gpu"]+ or spill_inputs.get("env_mmproj_unsized")+ # A host-resident cache or projector draws on the pool the iGPU reports.+ or spill_inputs.get("host_mmproj_bytes")+ or not _kv_offload_from_args(extra_args, env)+ or _extra_args_have_tensor_split(extra_args, env)+ # argv only: a non-layer inherited LLAMA_ARG_SPLIT_MODE is popped before launch.+ or _extra_args_split_mode(extra_args, {}) not in (None, "layer")+ # Extras follow the generated --fit off; the fitter rejects a multi-device split.+ or any(+ str(a) in ("-fit", "--fit") or str(a).startswith(("-fit=", "--fit="))+ for a in (extra_args or ())+ )+ # RPC devices go ahead of the local ones, so the shares would land on them.+ or (_extra_args_device(extra_args, {"--rpc"}) or "").strip()+ or str(env.get("LLAMA_ARG_RPC", "")).strip()+ # Tensor overrides move bytes after the split is priced.+ or _args_place_tensors_on_cpu(extra_args)+ or _env_places_tensors_on_cpu(env)+ # A projector pinned to another device is charged to device 0 below.+ or (_extra_args_device(extra_args, {"--mmproj-device", "-mmdev"}) or "").strip()+ or str(env.get("MTMD_BACKEND_DEVICE", "")).strip()+ # A surviving device list owns the share order; a gpu_ids pin strips it.+ or (+ not pin_owns_devices+ and (+ _extra_args_main_device(extra_args) is not None+ or str(env.get("LLAMA_ARG_DEVICE", "")).strip()+ )+ )+ ):+ return None+ # Adapters may target only some layers, which a total cannot place.+ if _sidecar_adapter_paths(extra_args):+ return None+ # A larger pass-through --parallel (last-wins) grows the cache priced here.+ _np_override = _extra_args_n_parallel(extra_args, env)+ if _np_override is not None and _np_override > int(spill_inputs.get("n_parallel") or 1):+ return None+ layout = self._tensor_spill_layout(spill_inputs.get("model_path"))+ if layout is None or not layout.complete or not layout.blocks:+ return None+ n_blocks = len(layout.blocks)+ kv_total = int(spill_inputs["kv_cache_bytes"] or 0)+ kv_weights = list(spill_inputs.get("kv_layer_weights") or ())+ if len(kv_weights) != n_blocks or sum(kv_weights) <= 0:+ # MLA, hybrid recurrent and an unpriced sliding window are uneven (_kv_layer_weights).+ if kv_total and (+ self._kv_lora_rank is not None+ or self._ssm_inner_size is not None+ or self._sliding_window_pattern+ or self._effective_sliding_window()+ ):+ return None+ kv_weights = [1] * n_blocks+ kv_scale = kv_total / sum(kv_weights)+ layer_bytes = [+ blk.spillable_bytes + blk.resident_bytes + w * kv_scale+ for blk, w in zip(layout.blocks, kv_weights)+ ]+ # Trailing nextn rows count in llama.cpp's n_layer_all + 1; resident only with a draft.+ n_mtp = int(self._nextn_predict_layers or 0) if layout.has_excluded_blocks else 0+ if layout.has_excluded_blocks and n_mtp <= 0:+ return None+ mtp_row = (+ layout.excluded_block_bytes / n_mtp+ if n_mtp and spill_inputs.get("mtp_will_engage")+ else 0.0+ ) + max(kv_weights) * kv_scale # archs walking n_layer_all cache these rows too+ layer_bytes.extend([mtp_row] * n_mtp)+ # The output layer rides the last row (the last device's run).+ layer_bytes.append(layout.lm_head_bytes + layout.other_resident_bytes)+ reserve = (+ spill_inputs["compute_buffer_flat"]+ + spill_inputs["extra_gpu_bytes"]+ + int(spill_inputs.get("env_mmproj_bytes") or 0)+ )+ # As _planned_tensor_spill; a hybrid run may hold one attention layer over its share.+ per_device = spill_inputs["ctx_compute_per_device"] + spill_inputs["soft_overhead"]+ if 0 < layout.n_attention_layers < n_blocks and len(kv_weights) == n_blocks:+ per_device += kv_total / layout.n_attention_layers+ mib = 1024 * 1024+ return self._discrete_first_split(+ list(gpu_indices),+ spill_inputs["gpu_usable_mib"],+ shared_gpu_ids,+ [x / mib for x in layer_bytes],+ per_device / mib,+ reserve / mib,+ self._PIPELINE_PER_DEVICE_OVERHEAD_MIB,+ )+@staticmethoddef _auto_split_fingerprint(tensor_split: Optional[List[float]]) -> Optional[tuple[float, ...]]:"""What an auto-mode load ASKED for, as a comparable ratio.@@ -17810,6 +17995,7 @@ class LlamaCppBackend:shared: frozenset = frozenset(),integrated: frozenset = frozenset(),discrete_size_bytes: Optional[int] = None,+ shared_gpu_ids: Optional[Iterable[int]] = None,) -> tuple[Optional[list[int]], bool]:"""Pick GPU(s) for a model from estimated VRAM and free memory.@@ -17832,6 +18018,7 @@ class LlamaCppBackend:``per_device_overhead_bytes`` is the fixed layer-split cost per GPU beyondthe first; a k-GPU pin must hold ``model + (k-1) * overhead`` or it can OOMa device after -ngl -1 (no --fit fallback). Single-GPU adds none.+ ``shared_gpu_ids`` (iGPUs whose "VRAM" is host RAM) rank after every discrete card.Returns (gpu_indices, use_fit):- ([1], False) fits on 1 GPU at the headroom threshold@@ -17873,13 +18060,31 @@ class LlamaCppBackend:# Rank by usable budget (free - reserve), not raw free: a more-used large# card can have less usable room than a less-used small one.- ranked = sorted(gpus, key = lambda g: _usable(g[0], g[1]), reverse = True)+ _shared = set(shared_gpu_ids or ())+ ranked = sorted(+ gpus,+ key = lambda g: (+ g[0] not in _shared and _usable(g[0], g[1]) > overhead_mib,+ _usable(g[0], g[1]),+ ),+ reverse = True,+ )# Cap a downgraded multi-GPU request to the usable count so it doesn't pull# in a near-full card to hit min_gpus. No-op for the default min_gpus == 1.usable_count = sum(1 for idx, free_mib in ranked if _usable(idx, free_mib) > overhead_mib)min_gpus = max(1, min(min_gpus, usable_count or 1))+ # A busy card too small alone must not pull a card that fits into a shared pin.+ if (+ min_gpus <= 1+ and ranked[0][0] in shared+ and _usable(ranked[0][0], ranked[0][1]) < model_size_mib+ ):+ for idx, free_mib in ranked:+ if idx not in shared and _usable(idx, free_mib) >= model_size_mib:+ return [idx], False+# Try 1 GPU at the usable-VRAM threshold (only when one device is allowed).if min_gpus <= 1 and _usable(ranked[0][0], ranked[0][1]) >= model_size_mib:for idx, free_mib in ranked:@@ -17920,6 +18125,7 @@ class LlamaCppBackend:shared: frozenset = frozenset(),integrated: frozenset = frozenset(),discrete_size_bytes: Optional[int] = None,+ shared_gpu_ids: Optional[Iterable[int]] = None,) -> tuple[Optional[list[int]], bool]:"""``_select_gpus``, re-checked at the multi-device context-compute rate.@@ -17958,6 +18164,7 @@ class LlamaCppBackend:per_device_overhead_bytes = per_device_overhead_bytes,min_gpus = min_gpus,shared = shared,+ shared_gpu_ids = shared_gpu_ids,)if use_fit or split_extra_bytes <= 0 or not gpu_indices or len(gpu_indices) < 2:return gpu_indices, use_fit@@ -17968,6 +18175,7 @@ class LlamaCppBackend:total_by_idx = total_by_idx,per_device_overhead_bytes = per_device_overhead_bytes + split_extra_bytes,min_gpus = min_gpus,+ shared_gpu_ids = shared_gpu_ids,)if not retry_fit and retry_indices and len(retry_indices) >= 2:return retry_indices, retry_fit@@ -17979,6 +18187,7 @@ class LlamaCppBackend:per_device_overhead_bytes = per_device_overhead_bytes,min_gpus = 1,shared = shared,+ shared_gpu_ids = shared_gpu_ids,)if not single_fit and single is not None and len(single) == 1:return single, False@@ -19238,6 +19447,7 @@ class LlamaCppBackend:exact: bool = False,integrated: frozenset = frozenset(),discrete_host_only_bytes: int = 0,+ shared_gpu_ids: Optional[Iterable[int]] = None,) -> tuple[Optional[list[int]], bool, int]:"""Largest serving-slot count that fits fully on GPU, so Unsloth keeps the model onGPU (-ngl -1) instead of --fit on, which offloads layers to host and collapses decode@@ -19317,6 +19527,7 @@ class LlamaCppBackend:),integrated = integrated,discrete_size_bytes = discrete_total,+ shared_gpu_ids = shared_gpu_ids,)if not use_fit:return gpu_indices, False, slots@@ -20824,6 +21035,7 @@ class LlamaCppBackend:self._memory_policy_extras_touched = Falseself._memory_mlock_applicable = Trueself._fit_load_mode_flags = []+ self._mixed_split_flags = []self._memory_pending_launch = Noneself._n_ubatch = self._DEFAULT_N_UBATCHself._requested_n_batch = None@@ -26699,6 +26911,7 @@ class LlamaCppBackend:self._tensor_split = Noneself._auto_tensor_split = Noneself._auto_tensor_split_emitted = None+ self._mixed_split_flags = []self._requested_gpu_ids = [int(i) for i in gpu_ids] if gpu_ids else Noneself._gpu_ids = list(self._requested_gpu_ids) if self._requested_gpu_ids else None# Manual offload skips the TP planner but still emits --split-mode@@ -27195,6 +27408,15 @@ class LlamaCppBackend:sysmem_fallback = _sysmem_fallback,)+ def _gpu_rank(+ g,+ frac = _vram_frac,+ floor_mib = 0.0,+ ):+ # iGPUs after every discrete card that holds its split buffers (_select_gpus).+ usable = _gpu_usable(g, frac)+ return (g[0] not in _shared_gpu_ids and usable > floor_mib, usable)+def _pool_budget_mib(subset, frac):# Sum each GPU's own usable budget. Pooling free and total# separately would let an unknown-total GPU (MIG/vGPU/N/A)@@ -28010,6 +28232,7 @@ class LlamaCppBackend:+ _cc_bytes(_mm_floor_ctx),min_gpus = _layer_min_gpus,split_extra_bytes = _cc_split_extra(_mm_floor_ctx),+ shared_gpu_ids = _shared_gpu_ids,)if _mm_split_awareelse self._select_gpus(@@ -28019,6 +28242,7 @@ class LlamaCppBackend:total_by_idx = total_by_idx,per_device_overhead_bytes = _pipeline_overhead_bytes,min_gpus = _layer_min_gpus,+ shared_gpu_ids = _shared_gpu_ids,))[1]@@ -28120,7 +28344,11 @@ class LlamaCppBackend:def _probe_rank(drafter: bool) -> list:return sorted(gpus,- key = lambda g: _gpu_usable(g, _probe_frac(drafter)),+ key = lambda g: _gpu_rank(+ g,+ _probe_frac(drafter),+ _pipeline_overhead_bytes / (1024 * 1024),+ ),reverse = True,)@@ -28642,7 +28870,11 @@ class LlamaCppBackend:if native_ctx_for_cap > 0:ranked_for_cap = sorted(gpus,- key = lambda g: _gpu_usable(g, _vram_frac - _flat_mtp_reserve),+ key = lambda g: _gpu_rank(+ g,+ _vram_frac - _flat_mtp_reserve,+ _pipeline_overhead_bytes / (1024 * 1024),+ ),reverse = True,)best_cap = 0@@ -28760,6 +28992,7 @@ class LlamaCppBackend:shared = _shared_gpus,integrated = _tier_gpus,discrete_size_bytes = requested_total - _discrete_host_only(),+ shared_gpu_ids = _shared_gpu_ids,)# No silent shrink: effective_ctx stays == requested_ctx.# use_fit = the pin failed and --fit on will offload; say why.@@ -28824,6 +29057,7 @@ class LlamaCppBackend:+ _mtp_bytes(effective_ctx)+ _cc_bytes(effective_ctx, cache_type = "q8_0")- _discrete_host_only(),+ shared_gpu_ids = _shared_gpu_ids,)_q8_fits = not _q8_use_fit_cuda_ctx_notice = self._cuda_context_overcommit_notice(@@ -28837,8 +29071,14 @@ class LlamaCppBackend:# headroom threshold as _select_gpus (#5106). Rank by the# active pin fraction so the order matches the fit budget.pin_fraction = _pin_fraction+ # Max _reserve_at ask: a short card must not hide an iGPU that fits.+ _rank_floor_mib = (+ _pipeline_overhead_bytes + _cc_bytes(effective_ctx, 2) // 2+ ) / (1024 * 1024)ranked = sorted(- gpus, key = lambda g: _gpu_usable(g, pin_fraction), reverse = True+ gpus,+ key = lambda g: _gpu_rank(g, pin_fraction, _rank_floor_mib),+ reverse = True,)# Skips _select_gpus, so apply its cap: count only cards# whose usable VRAM clears the per-device layer overhead.@@ -28928,10 +29168,8 @@ class LlamaCppBackend:):# Keep the largest context the smallest card holds# rather than lose the subset to the 4096 drop below.- # Costs no context: subsets are prefixes of a- # usable-descending ranking and the reserve is the- # same function of context for every n > 1, so any- # later subset clears the gate no higher than this.+ # Costs no context: a longer prefix never has a larger+ # smallest card, and the reserve is one function for n > 1.capped = self._cap_ctx_to_per_device_reserve(capped, _usable_mib, _reserve_at)@@ -29026,6 +29264,7 @@ class LlamaCppBackend:shared = _shared_gpus,integrated = _tier_gpus,discrete_size_bytes = _fs_total - _discrete_host_only(),+ shared_gpu_ids = _shared_gpu_ids,)if use_fit and not explicit_ctx:# Without KV metadata, llama.cpp owns the fit. Keep the@@ -29446,6 +29685,7 @@ class LlamaCppBackend:# Picks a subset, so an explicit device pick keeps the base pick.integrated = _tier_gpus,discrete_host_only_bytes = _discrete_host_only(),+ shared_gpu_ids = _shared_gpu_ids,)if not _uf_slots:_slots_asked = n_parallel@@ -29483,6 +29723,7 @@ class LlamaCppBackend:ctx_checkpoints = _placement_ctx_checkpoints,integrated = _tier_gpus,discrete_host_only_bytes = _discrete_host_only(),+ shared_gpu_ids = _shared_gpu_ids,)return _gi if not _uf and _got == n_parallel else None@@ -30148,6 +30389,9 @@ class LlamaCppBackend:# allowance, exactly what _fit_model_size and _fit_soft_overhead# just charged, and the same abstain when it cannot be sized._spill_inputs["env_mmproj_unsized"] = _fit_env_mmproj_unsized+ _spill_inputs["host_mmproj_bytes"] = _mmproj_pinned_bytes + (+ _fit_env_mmproj_bytes if _fit_env_mmproj_on_host else 0+ )_spill_inputs["env_mmproj_bytes"] = (0if _fit_env_mmproj_on_host@@ -30876,6 +31120,32 @@ class LlamaCppBackend:# and offloads ~1 GB at --parallel 4 even though the model fits.cmd.extend(["-ngl", "-1", "--fit", "off"])fully_gpu_offloaded = True+ _mixed_split = self._mixed_pin_split(+ list(gpu_indices),+ _spill_inputs,+ _shared_gpu_ids,+ extra_args,+ env,+ layer_min_gpus = _layer_min_gpus,+ tensor_parallel = tensor_parallel,+ pin_owns_devices = _gpu_ids_own_device_flags,+ )+ if _mixed_split is not None:+ self._mixed_split_flags = [+ "--tensor-split",+ self._format_tensor_split(_mixed_split),+ ]+ cmd.extend(self._mixed_split_flags)+ if gpu_memory_mode != "manual":+ self._auto_tensor_split_emitted = self._auto_split_fingerprint(+ _mixed_split+ )+ logger.info(+ "Filling discrete GPU(s) before the shared-memory iGPU: "+ "--tensor-split %s over %s",+ cmd[-1],+ list(gpu_indices),+ )# Expose Prometheus /metrics for the engine-stats logger, only# when the binary advertises it (older/custom binaries may not).@@ -33366,6 +33636,10 @@ class LlamaCppBackend:_run = list(run_cmd)if "--fit" in _run:_run[_run.index("--fit") + 1] = "on"+ if self._mixed_split_flags:+ # common/fit.cpp aborts a multi-device fit on a set split.+ _run = _without_subsequence(_run, self._mixed_split_flags)+ self._auto_tensor_split_emitted = None# Same reasoning as the page-lock below, one step# earlier: the fit picked "none" because it had proved# the load fits, and that proof is what just failed, sodiff --git a/studio/backend/tests/test_mtp_vram_budget.py b/studio/backend/tests/test_mtp_vram_budget.pyindex b1575c77220..9c647e7fa01 100644--- a/studio/backend/tests/test_mtp_vram_budget.py+++ b/studio/backend/tests/test_mtp_vram_budget.py@@ -586,8 +586,9 @@ class TestExtraArgsMtpDetection:def test_load_model_ranks_subsets_by_active_pin_fraction(self):compact = "".join(inspect.getsource(LlamaCppBackend.load_model).split())- assert "_gpu_usable(g,pin_fraction)" in compact- assert "_gpu_usable(g,_vram_frac-_flat_mtp_reserve)" in compact+ assert "usable=_gpu_usable(g,frac)" in compact+ assert "_gpu_rank(g,pin_fraction," in compact+ assert "_gpu_rank(g,_vram_frac-_flat_mtp_reserve," in compactassert "_vram_frac=_active_vram_fraction()" in compact@pytest.mark.parametrize(diff --git a/studio/backend/tests/test_vulkan_igpu_split.py b/studio/backend/tests/test_vulkan_igpu_split.pynew file mode 100644index 00000000000..e6efe4a6579--- /dev/null+++ b/studio/backend/tests/test_vulkan_igpu_split.py@@ -0,0 +1,426 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A Vulkan pin mixing a discrete card with a shared-memory iGPU: the card fills first.+Reported on an RX 7700 XT + Ryzen iGPU (27B: 8.3 GB on the iGPU, 1.05 t/s)."""++from __future__ import annotations++import inspect+import sys+import types as _types+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++import importlib as _importlib # noqa: E402+++def _maybe_stub(name: str, builder):+ try:+ _importlib.import_module(name)+ except ImportError:+ sys.modules[name] = builder()+++def _build_loggers_stub():+ m = _types.ModuleType("loggers")+ m.get_logger = lambda name: __import__("logging").getLogger(name)+ return m+++_maybe_stub("loggers", _build_loggers_stub)+_maybe_stub("structlog", lambda: _types.ModuleType("structlog"))++from core.inference.llama_cpp import LlamaCppBackend # noqa: E402++MIB = 1024 * 1024++# The reporter's probe: VK0 the RX 7700 XT, VK1 the iGPU after its host reserve.+DGPU, IGPU = 0, 1+GPUS = [(DGPU, 11313), (IGPU, 14352)]+SHARED = {IGPU}+++def test_a_model_either_device_holds_lands_on_the_discrete_card():+ picked, use_fit = LlamaCppBackend._select_gpus(+ 6000 * MIB, GPUS, usable_fraction = 0.9, shared_gpu_ids = SHARED+ )+ assert (picked, use_fit) == ([DGPU], False)+++def test_without_shared_ids_the_larger_shared_pool_still_wins():+ # CUDA / ROCm callers pass no shared set and keep the old ranking.+ picked, _ = LlamaCppBackend._select_gpus(6000 * MIB, GPUS, usable_fraction = 0.9)+ assert picked == [IGPU]+++def test_a_model_that_needs_both_still_pins_both():+ picked, use_fit = LlamaCppBackend._select_gpus(+ 15000 * MIB, GPUS, usable_fraction = 0.9, shared_gpu_ids = SHARED+ )+ assert (picked, use_fit) == ([DGPU, IGPU], False)+++def test_split_aware_passes_the_shared_set_through():+ picked, _ = LlamaCppBackend._select_gpus_split_aware(+ 6000 * MIB,+ GPUS,+ usable_fraction = 0.9,+ split_extra_bytes = 256 * MIB,+ shared_gpu_ids = SHARED,+ )+ assert picked == [DGPU]+++def test_a_card_too_full_for_its_split_buffers_does_not_outrank_the_igpu():+ picked, use_fit = LlamaCppBackend._select_gpus(+ 10000 * MIB,+ [(DGPU, 900), (IGPU, 14352)],+ usable_fraction = 0.9,+ per_device_overhead_bytes = 1024 * MIB,+ shared_gpu_ids = SHARED,+ )+ assert (picked, use_fit) == ([IGPU], False)+++def test_every_auto_placement_sort_ranks_discrete_first():+ src = inspect.getsource(LlamaCppBackend.load_model)+ assert "key = lambda g: _gpu_usable(" not in src+ assert src.count("key = lambda g: _gpu_rank(") == 3+ assert "_gpu_rank(g, pin_fraction, _rank_floor_mib)" in src+++def test_a_busy_card_too_small_alone_does_not_pull_in_the_igpu():+ # Another model runs on the card; the iGPU holds this one alone, so it goes there.+ picked, use_fit = LlamaCppBackend._select_gpus(+ 12000 * MIB,+ GPUS,+ usable_fraction = 0.9,+ shared = frozenset({DGPU}),+ shared_gpu_ids = SHARED,+ )+ assert (picked, use_fit) == ([IGPU], False)+++def test_the_fit_on_retry_takes_the_generated_split_back_out():+ src = inspect.getsource(LlamaCppBackend.load_model)+ retry = src[src.index("with forced --fit off; the fit estimate was optimistic") :]+ retry = retry[: retry.index("self._fit_load_mode_flags")]+ assert "_without_subsequence(_run, self._mixed_split_flags)" in retry+ assert "self._auto_tensor_split_emitted = None" in retry+++split = LlamaCppBackend._discrete_first_split+++def test_the_card_takes_the_layers_its_room_holds_and_the_igpu_the_rest():+ # 10 blocks of 1000 MiB + a 500 MiB output layer; the card keeps 300 for itself.+ shares = split(+ [DGPU, IGPU], {DGPU: 10180.0, IGPU: 12917.0}, SHARED, [1000.0] * 10 + [500.0], 300.0+ )+ # 9 layers then 2; each boundary half a layer early against float rounding.+ assert shares == [8.5, 2.5]+++def test_the_heaviest_contiguous_run_decides_not_the_average():+ # A full-attention layer among window layers: an even byte share would give 7.+ layers = [100.0] * 4 + [3000.0] + [100.0] * 4+ assert split([DGPU, IGPU], {DGPU: 3500.0, IGPU: 9000.0}, SHARED, layers) == [5.5, 3.5]+++def test_nothing_left_over_gives_the_igpu_nothing():+ assert split([DGPU, IGPU], {DGPU: 10000.0, IGPU: 12000.0}, SHARED, [1000.0] * 5 + [100.0]) == [+ 5.5,+ 0.5,+ ]+++def test_the_split_is_positional_over_the_pin_order():+ shares = split([IGPU, DGPU], {DGPU: 10000.0, IGPU: 12000.0}, SHARED, [1000.0] * 12)+ assert shares == [1.5, 10.5]+++def test_device_zero_carries_the_one_time_reserve():+ # iGPU first: the flat buffer lands there, so the card keeps its whole budget.+ shares = split(+ [IGPU, DGPU], {DGPU: 10000.0, IGPU: 12000.0}, SHARED, [1000.0] * 12, main_reserve_mib = 1500.0+ )+ assert shares == [1.5, 10.5]+ shares = split(+ [DGPU, IGPU], {DGPU: 10000.0, IGPU: 12000.0}, SHARED, [1000.0] * 12, main_reserve_mib = 1500.0+ )+ assert shares == [7.5, 4.5]+++def test_overflow_is_shared_across_igpus_by_their_own_room():+ shares = split(+ [0, 1, 2],+ {0: 4000.0, 1: 3300.0, 2: 1300.0},+ {1, 2},+ [1000.0] * 7,+ per_device_mib = 100.0,+ pipeline_mib = 200.0,+ )+ assert shares == [2.5, 3.0, 1.5]+++def test_only_a_mixed_pin_gets_a_split():+ usable = {0: 10000.0, 1: 12000.0}+ assert split([0, 1], usable, set(), [1000.0] * 15) is None+ assert split([0, 1], usable, {0, 1}, [1000.0] * 15) is None+ assert split([0, 1], usable, {1}, []) is None+ # A card that cannot hold even one layer: llama.cpp's split, as before.+ assert split([0, 1], {0: 500.0, 1: 12000.0}, {1}, [1000.0] * 15) is None+++def test_the_launch_hands_placement_back_wherever_it_cannot_price_it():+ src = inspect.getsource(LlamaCppBackend._mixed_pin_split)+ for needle in (+ "tensor_parallel",+ "layer_min_gpus > 1",+ 'spill_inputs["separate_draft_on_gpu"]',+ '"env_mmproj_unsized"',+ '"host_mmproj_bytes"',+ "_kv_offload_from_args(extra_args, env)",+ "_extra_args_have_tensor_split(extra_args, env)",+ "_extra_args_main_device(extra_args) is not None",+ "not pin_owns_devices",+ '"LLAMA_ARG_DEVICE"',+ "_sidecar_adapter_paths(extra_args)",+ '"kv_layer_weights"',+ '"compute_buffer_flat"',+ 'spill_inputs["ctx_compute_per_device"] + spill_inputs["soft_overhead"]',+ '"extra_gpu_bytes"',+ '"env_mmproj_bytes"',+ ):+ assert needle in src, needle+++def test_the_launch_records_what_it_emitted():+ src = inspect.getsource(LlamaCppBackend.load_model)+ arm = src[+ src.index("_mixed_split = self._mixed_pin_split(") : src.index(+ "# Expose Prometheus /metrics"+ )+ ]+ assert "self._mixed_split_flags = [" in arm+ assert "pin_owns_devices = _gpu_ids_own_device_flags" in arm+ assert "self._auto_tensor_split_emitted = self._auto_split_fingerprint(" in arm+++def test_an_igpu_run_that_overflows_its_room_leaves_the_split_off():+ # The pooled fit passes (10 of 10), but the 9 MiB layer would land on the 1 MiB iGPU.+ assert split([DGPU, IGPU], {DGPU: 9.0, IGPU: 1.0}, SHARED, [1.0, 9.0]) is None+++def test_few_rows_over_many_igpus_never_go_negative():+ shares = split(+ [0, 1, 2, 3, 4],+ {0: 3.0, 1: 30.0, 2: 30.0, 3: 30.0, 4: 10.0},+ {1, 2, 3, 4},+ [1.0] * 5,+ )+ assert shares is not None and min(shares) >= 0 and sum(shares) == 5+ assert shares[0] == 2.5 # three rows on the card+++def test_trailing_mtp_rows_keep_their_place_in_the_split():+ src = inspect.getsource(LlamaCppBackend._mixed_pin_split)+ assert "self._nextn_predict_layers" in src+ assert "layer_bytes.extend([mtp_row] * n_mtp)" in src+++def test_a_row_or_tensor_split_mode_leaves_the_split_to_llama_cpp():+ class Priced(Exception):+ pass++ backend = LlamaCppBackend.__new__(LlamaCppBackend)++ def _priced(*_a, **_k):+ raise Priced++ backend._tensor_spill_layout = _priced+ inputs = {"separate_draft_on_gpu": False, "model_path": "m.gguf"}++ def call(args, env):+ return backend._mixed_pin_split(+ [DGPU, IGPU],+ inputs,+ SHARED,+ args,+ env,+ layer_min_gpus = 1,+ tensor_parallel = False,+ pin_owns_devices = True,+ )++ for args in (+ ["-sm", "row"],+ ["--split-mode=tensor"],+ ["--rpc", "10.0.0.2:50052"],+ ["--fit", "on"],+ ["--fit=on"],+ ):+ assert call(args, {}) is None, args+ assert call([], {"LLAMA_ARG_RPC": "10.0.0.2:50052"}) is None+ # The launch pops an inherited non-layer mode, so the child runs layer split.+ with pytest.raises(Priced):+ call([], {"LLAMA_ARG_SPLIT_MODE": "row"})+++def test_an_unpriced_uneven_kv_cache_leaves_the_split_to_llama_cpp():+ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ block = _types.SimpleNamespace(spillable_bytes = 1, resident_bytes = 1)+ layout = _types.SimpleNamespace(+ complete = True,+ blocks = [block] * 4,+ has_excluded_blocks = False,+ lm_head_bytes = 0,+ other_resident_bytes = 0,+ )+ backend._tensor_spill_layout = lambda _path: layout+ # No compute_buffer_flat: a load that passes the KV gate stops on the KeyError.+ inputs = {+ "separate_draft_on_gpu": False,+ "model_path": "m.gguf",+ "kv_cache_bytes": 4 * MIB,+ "kv_layer_weights": [],+ }++ def call():+ return backend._mixed_pin_split(+ [DGPU, IGPU],+ inputs,+ SHARED,+ [],+ {},+ layer_min_gpus = 1,+ tensor_parallel = False,+ pin_owns_devices = True,+ )++ for mla, ssm, pattern in ((512, None, None), (None, 1024, None), (None, None, [1, 0, 1, 0])):+ backend._kv_lora_rank, backend._ssm_inner_size = mla, ssm+ backend._sliding_window_pattern = pattern+ assert call() is None, (mla, ssm, pattern)+ # A declared window whose pattern did not resolve is still uneven.+ backend._kv_lora_rank = backend._ssm_inner_size = backend._sliding_window_pattern = None+ backend._sliding_window, backend._context_length = 4096, 1048576+ assert call() is None+ # A plain cache is uniform, so it is priced.+ backend._sliding_window = None+ with pytest.raises(KeyError):+ call()+++def _priced_backend(layout):+ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._tensor_spill_layout = lambda _path: layout+ backend._kv_lora_rank = backend._ssm_inner_size = backend._sliding_window_pattern = None+ return backend+++def _mixed(+ backend,+ inputs,+ args = (),+ env = None,+):+ return backend._mixed_pin_split(+ [DGPU, IGPU],+ inputs,+ SHARED,+ list(args),+ env or {},+ layer_min_gpus = 1,+ tensor_parallel = False,+ pin_owns_devices = True,+ )+++class _Priced(Exception):+ pass+++def _pricing_backend():+ backend = LlamaCppBackend.__new__(LlamaCppBackend)++ def _priced(*_a, **_k):+ raise _Priced++ backend._tensor_spill_layout = _priced+ return backend+++def test_a_projector_pinned_to_another_device_leaves_the_split_to_llama_cpp():+ backend = _pricing_backend()+ inputs = {"separate_draft_on_gpu": False, "model_path": "m.gguf"}+ assert _mixed(backend, inputs, ["--mmproj-device", "Vulkan1"]) is None+ assert _mixed(backend, inputs, ["-mmdev=Vulkan1"]) is None+ assert _mixed(backend, inputs, env = {"MTMD_BACKEND_DEVICE": "Vulkan1"}) is None+ with pytest.raises(_Priced):+ _mixed(backend, inputs)+++def test_a_larger_pass_through_parallel_leaves_the_split_to_llama_cpp():+ backend = _pricing_backend()+ inputs = {"separate_draft_on_gpu": False, "model_path": "m.gguf", "n_parallel": 1}+ assert _mixed(backend, inputs, ["--parallel", "4"]) is None+ assert _mixed(backend, inputs, ["-np=2"]) is None+ # Fewer slots than priced leaves the plan over-reserved, the safe direction.+ with pytest.raises(_Priced):+ _mixed(backend, {**inputs, "n_parallel": 4}, ["--parallel", "2"])+++def test_trailing_mtp_rows_carry_a_layer_of_cache():+ block = _types.SimpleNamespace(spillable_bytes = 0, resident_bytes = 0)+ layout = _types.SimpleNamespace(+ complete = True,+ blocks = [block] * 4,+ has_excluded_blocks = True,+ excluded_block_bytes = 0,+ n_attention_layers = 4,+ lm_head_bytes = 0,+ other_resident_bytes = 0,+ )+ backend = _priced_backend(layout)+ backend._nextn_predict_layers = 1+ seen = {}++ def _record(gpu_indices, usable, shared, layer_mib, *_a):+ seen["rows"] = layer_mib+ return None++ backend._discrete_first_split = _record+ _mixed(+ backend,+ {+ "separate_draft_on_gpu": False,+ "model_path": "m.gguf",+ "kv_cache_bytes": 8 * MIB,+ "kv_layer_weights": [],+ "mtp_will_engage": False,+ "compute_buffer_flat": 0,+ "extra_gpu_bytes": 0,+ "ctx_compute_per_device": 0,+ "soft_overhead": 0,+ "gpu_usable_mib": {DGPU: 100.0, IGPU: 100.0},+ },+ )+ # 4 blocks + 1 nextn row + output: the nextn row holds one block's 2 MiB of cache.+ assert seen["rows"] == [2.0, 2.0, 2.0, 2.0, 2.0, 0.0]+++def test_a_tensor_override_leaves_the_split_to_llama_cpp():+ backend = _pricing_backend()+ inputs = {"separate_draft_on_gpu": False, "model_path": "m.gguf"}+ assert _mixed(backend, inputs, ["-ot", "blk\\.1[2-5]\\.ffn_.*=Vulkan0"]) is None+ assert _mixed(backend, inputs, ["--n-cpu-moe", "4"]) is None+ assert _mixed(backend, inputs, env = {"LLAMA_ARG_OVERRIDE_TENSOR": "ffn=Vulkan0"}) is None+ with pytest.raises(_Priced):+ _mixed(backend, inputs, ["--n-cpu-moe", "0"])