Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: prefer discrete GPUs over unified-memory APUs in automatic GGUF placement; clamp Vulkan free to total (#14372)

Daniel Han4279fe4ae248
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index cfebc9b733f..8ef8450597e 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -15714,12 +15714,16 @@ class LlamaCppBackend:
if len(parts) not in (4, 5, 6):
continue
try:
+ free_b, total_b = int(parts[1]), int(parts[3])
+ # Intel's Windows driver can report budget > heap size (ggml-org/llama.cpp#29277).
+ if 0 < total_b < free_b:
+ free_b = total_b
rows.append(
{
"index": int(parts[0]),
- "free_mib": int(parts[1]) // (1024 * 1024),
+ "free_mib": free_b // (1024 * 1024),
"is_igpu": parts[2] == "1",
- "total_mib": int(parts[3]) // (1024 * 1024),
+ "total_mib": total_b // (1024 * 1024),
"name": parts[4].strip() if len(parts) >= 5 else "",
# Absent on an older staged probe, so it reads unknown and declines.
"type_known": len(parts) >= 6 and parts[5].strip() == "1",
@@ -17713,6 +17717,19 @@ class LlamaCppBackend:
for name in ("LLAMA_ARG_SPLIT_MODE", "LLAMA_ARG_TENSOR_SPLIT"):
env.pop(name, None)
+ @staticmethod
+ def _discrete_tier(
+ gpus: list,
+ integrated: frozenset,
+ shared: frozenset = frozenset(),
+ ) -> Optional[list]:
+ """The discrete-first tier (#7624): unshared discrete cards in ``gpus`` order, or
+ None when ``gpus`` holds no integrated GPU (no tier; the base pick applies). Shared
+ dGPUs are left to the full-pool pick, whose shared preference spans every card."""
+ if not integrated or all(g[0] not in integrated for g in gpus):
+ return None
+ return [g for g in gpus if g[0] not in integrated and g[0] not in shared]
+
@staticmethod
def _select_gpus(
model_size_bytes: int,
@@ -17722,12 +17739,18 @@ class LlamaCppBackend:
per_device_overhead_bytes: int = 0,
min_gpus: int = 1,
shared: frozenset = frozenset(),
+ integrated: frozenset = frozenset(),
+ discrete_size_bytes: Optional[int] = None,
) -> tuple[Optional[list[int]], bool]:
"""Pick GPU(s) for a model from estimated VRAM and free memory.
``shared``: GPUs another loaded model runs on. A model that fits one card alone
takes a card outside it when one fits, so two models don't share compute.
+ ``integrated``: unified-memory GPUs; discrete GPUs win when they alone hold the model.
+ ``discrete_size_bytes`` prices that discrete attempt (default ``model_size_bytes``):
+ bytes that live in host RAM only on unified memory are not charged to a dGPU.
+
``min_gpus`` (default 1, capped at ``len(gpus)``) keeps a downgraded
tensor/multi-GPU request spread instead of collapsing to one card.
@@ -17750,6 +17773,19 @@ class LlamaCppBackend:
return None, True
min_gpus = max(1, min(min_gpus, len(gpus)))
+ # Discrete first (#7624): an APU's "free" is system RAM. Falls through if it can't meet min_gpus.
+ discrete = LlamaCppBackend._discrete_tier(gpus, integrated, shared)
+ if discrete is not None and min_gpus <= len(discrete):
+ picked, use_fit = LlamaCppBackend._select_gpus(
+ model_size_bytes if discrete_size_bytes is None else discrete_size_bytes,
+ discrete,
+ usable_fraction = usable_fraction,
+ total_by_idx = total_by_idx,
+ per_device_overhead_bytes = per_device_overhead_bytes,
+ min_gpus = min_gpus,
+ )
+ if not use_fit and picked and len(picked) >= min_gpus:
+ return picked, False
model_size_mib = model_size_bytes / (1024 * 1024)
if usable_fraction is None:
usable_fraction = _active_vram_fraction()
@@ -17813,6 +17849,8 @@ class LlamaCppBackend:
min_gpus: int = 1,
split_extra_bytes: int = 0,
shared: frozenset = frozenset(),
+ integrated: frozenset = frozenset(),
+ discrete_size_bytes: Optional[int] = None,
) -> tuple[Optional[list[int]], bool]:
"""``_select_gpus``, re-checked at the multi-device context-compute rate.
@@ -17824,7 +17862,25 @@ class LlamaCppBackend:
count-independent. The bigger overhead can cut ``_select_gpus``'s usable-card
count to one, and a lone card is not a split, so a retry landing below two
devices is re-priced without the delta. Still no fit returns (None, True) ->
- --fit on, i.e. CPU offload rather than a pin that OOMs at launch."""
+ --fit on, i.e. CPU offload rather than a pin that OOMs at launch.
+
+ ``integrated``: as in ``_select_gpus``, but the discrete tier must pass the whole
+ split-aware check (priced at ``discrete_size_bytes`` when given); otherwise the
+ full pool is priced exactly as without it."""
+ discrete = LlamaCppBackend._discrete_tier(gpus, integrated, shared)
+ if discrete is not None:
+ if max(1, min(min_gpus, len(gpus))) <= len(discrete):
+ picked, needs_fit = LlamaCppBackend._select_gpus_split_aware(
+ model_size_bytes if discrete_size_bytes is None else discrete_size_bytes,
+ discrete,
+ usable_fraction = usable_fraction,
+ total_by_idx = total_by_idx,
+ per_device_overhead_bytes = per_device_overhead_bytes,
+ min_gpus = min_gpus,
+ split_extra_bytes = split_extra_bytes,
+ )
+ if not needs_fit and picked and len(picked) >= max(1, min(min_gpus, len(gpus))):
+ return picked, False
gpu_indices, use_fit = LlamaCppBackend._select_gpus(
model_size_bytes,
gpus,
@@ -19111,6 +19167,8 @@ class LlamaCppBackend:
ctx_checkpoints: int = 0,
include_requested: bool = False,
exact: bool = False,
+ integrated: frozenset = frozenset(),
+ discrete_host_only_bytes: int = 0,
) -> tuple[Optional[list[int]], bool, int]:
"""Largest serving-slot count that fits fully on GPU, so Unsloth keeps the model on
GPU (-ngl -1) instead of --fit on, which offloads layers to host and collapses decode
@@ -19126,6 +19184,9 @@ class LlamaCppBackend:
``n_parallel`` itself and would re-price every lower count just to discard it.
``ctx_checkpoints`` mirrors --ctx-checkpoints: N host-RAM snapshots per slot,
passed only when the GPUs share that pool (unified memory), else 0.
+ ``integrated``: the discrete-tier attempt is priced without the checkpoints and
+ without ``discrete_host_only_bytes`` (e.g. a CPU-pinned projector on a shared pool);
+ ``gpus`` already narrowed to discrete cards of a mixed pool are priced that way outright.
``ubatch_for_slots`` re-derives the micro-batch per candidate: the emitted
--batch-size is raised to max(slots, 2), and llama.cpp caps the micro-batch
against it, so a batch below the requested slot count shrinks as the candidates
@@ -19146,21 +19207,35 @@ class LlamaCppBackend:
)
if cb <= 0:
cb = self._TENSOR_PARALLEL_BUFFER_RESERVE_MIB * 1024 * 1024
- total = (
- base_footprint_bytes
- + cb
- + (mtp_bytes_for_slots(slots, _ub) if mtp_bytes_for_slots else 0)
- + self._estimate_kv_cache_bytes(
+
+ def _kv(ckpts: int) -> int:
+ return self._estimate_kv_cache_bytes(
effective_ctx,
cache_type_kv,
n_parallel = slots,
swa_full = swa_full,
kv_unified = kv_unified,
n_ubatch = _ub,
- ctx_checkpoints = ctx_checkpoints,
+ ctx_checkpoints = ckpts,
flash_attn = flash_attn,
)
+
+ kv = _kv(ctx_checkpoints)
+ total = (
+ base_footprint_bytes
+ + cb
+ + (mtp_bytes_for_slots(slots, _ub) if mtp_bytes_for_slots else 0)
+ + kv
)
+ # Checkpoints and host-only bytes sit in host RAM, which a dGPU does not draw on.
+ discrete_total = None
+ if integrated and (ctx_checkpoints or discrete_host_only_bytes):
+ discrete_total = (
+ total - discrete_host_only_bytes - (kv - _kv(0) if ctx_checkpoints else 0)
+ )
+ # Devices already narrowed to dGPUs of a mixed pool (the slot re-fit).
+ if all(g[0] not in integrated for g in gpus):
+ total = discrete_total
gpu_indices, use_fit = self._select_gpus_split_aware(
total,
gpus,
@@ -19171,6 +19246,8 @@ class LlamaCppBackend:
split_extra_bytes = (
split_extra_for_slots(slots) if split_extra_for_slots else split_extra_bytes
),
+ integrated = integrated,
+ discrete_size_bytes = discrete_total,
)
if not use_fit:
return gpu_indices, False, slots
@@ -26708,6 +26785,7 @@ class LlamaCppBackend:
_moe_cache_candidate = False
_moe_cache_ram_ok = False
_shared_gpus = frozenset()
+ _integrated_gpus = frozenset()
# Sized inputs for the tensor-spill planner, None when the fit never
# priced them. Bound before the try like _placement_verdict_partial:
# the except arm restores use_fit=True without rebinding
@@ -26905,6 +26983,22 @@ class LlamaCppBackend:
# GPU-aware speculative defaults; the list feeds the
# CPU-fallback check.
_detected_gpus = list(gpus)
+ # Auto placement only; Vulkan auto already dropped its iGPUs above.
+ if not gpu_ids and not is_vulkan_backend and len(gpus) > 1:
+ _unified = self._rocm_unified_memory_gpu_ids()
+ # CUDA ids are ordinals; nvidia-smi rows are PCI indices. Join only
+ # where they agree, as _widen_integrated_cuda_rows does.
+ if LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is False or (
+ (os.environ.get("CUDA_DEVICE_ORDER") or "").strip().upper()
+ == "PCI_BUS_ID"
+ and not self._visibility_mask_is_unmappable()
+ ):
+ _unified |= set(
+ LlamaCppBackend._INTEGRATED_CUDA_IDS.get(
+ LlamaCppBackend._integrated_cuda_mask_key(), ()
+ )
+ )
+ _integrated_gpus = frozenset(idx for idx, _free in gpus if idx in _unified)
# Vulkan reports total 0 only for integrated GPUs. Their
# free "VRAM" is the same host pool the RAM guard prices.
_shared_gpu_ids = (
@@ -26935,6 +27029,23 @@ class LlamaCppBackend:
)
else 0
)
+
+ def _subset_ckpts(subset) -> int:
+ # A discrete-tier subset holds no unified-memory GPU, so no checkpoints.
+ if _integrated_gpus and _integrated_gpus.isdisjoint(i for i, _f in subset):
+ return 0
+ return _placement_ctx_checkpoints
+
+ def _tiered_subsets(ranked: list, floor: int) -> list:
+ # (count, prefix) in _select_gpus order: unshared discrete first.
+ tier = LlamaCppBackend._discrete_tier(
+ ranked, _integrated_gpus, _shared_gpus
+ )
+ tiers = (
+ (tier, ranked) if tier is not None and floor <= len(tier) else (ranked,)
+ )
+ return [(n, t[:n]) for t in tiers for n in range(floor, len(t) + 1)]
+
# The --fit fallback is llama.cpp's own fitter, which knows nothing
# about this budget: it keeps its own margin and packs the rest on,
# so the slider never reached the path that runs when the fit is
@@ -27776,27 +27887,50 @@ class LlamaCppBackend:
# whether _mm_need carries a context-compute term, so the two
# cannot drift apart.
_mm_split_aware = self._can_estimate_kv()
- _, _mm_needs_fit = (
- self._select_gpus_split_aware(
- _mm_need,
- _mm_budgeted_gpus,
- usable_fraction = _mm_frac,
- total_by_idx = total_by_idx,
- per_device_overhead_bytes = _pipeline_overhead_bytes
- + _cc_bytes(_mm_floor_ctx),
- min_gpus = _layer_min_gpus,
- split_extra_bytes = _cc_split_extra(_mm_floor_ctx),
- )
- if _mm_split_aware
- else self._select_gpus(
- _mm_need,
- _mm_budgeted_gpus,
- usable_fraction = _mm_frac,
- total_by_idx = total_by_idx,
- per_device_overhead_bytes = _pipeline_overhead_bytes,
- min_gpus = _layer_min_gpus,
- )
+
+ def _mm_probe_needs_fit(need: int, devices: list) -> bool:
+ return (
+ self._select_gpus_split_aware(
+ need,
+ devices,
+ usable_fraction = _mm_frac,
+ total_by_idx = total_by_idx,
+ per_device_overhead_bytes = _pipeline_overhead_bytes
+ + _cc_bytes(_mm_floor_ctx),
+ min_gpus = _layer_min_gpus,
+ split_extra_bytes = _cc_split_extra(_mm_floor_ctx),
+ )
+ if _mm_split_aware
+ else self._select_gpus(
+ need,
+ devices,
+ usable_fraction = _mm_frac,
+ total_by_idx = total_by_idx,
+ per_device_overhead_bytes = _pipeline_overhead_bytes,
+ min_gpus = _layer_min_gpus,
+ )
+ )[1]
+
+ _mm_needs_fit = _mm_probe_needs_fit(_mm_need, _mm_budgeted_gpus)
+ # An APU reporting a total is budgeted above, but its "VRAM" is host
+ # RAM, so moving the projector frees nothing there. Also pin when that
+ # alone lets the discrete cards hold the model (#7624).
+ _mm_discrete = LlamaCppBackend._discrete_tier(
+ _mm_budgeted_gpus, _integrated_gpus, _shared_gpus
)
+ if (
+ not _mm_needs_fit
+ and _mm_discrete
+ and _layer_min_gpus <= len(_mm_discrete)
+ ):
+ _mm_needs_fit = _mm_probe_needs_fit(
+ _mm_need, _mm_discrete
+ ) and not _mm_probe_needs_fit(
+ _mm_need
+ - mmproj_size
+ - int(mmproj_size * (self._MMPROJ_VRAM_SAFETY - 1.0)),
+ _mm_discrete,
+ )
if _mm_needs_fit:
if tensor_parallel:
# Held, not dropped. Applying it here would price a
@@ -27914,15 +28048,16 @@ class LlamaCppBackend:
_both_fit_somewhere = False
_probe_ctx = 0
_probe_need = _probe_have = 0.0
- # (GPU count, target-only ctx, budget with drafter) per drafter-ranked subset.
- _target_placements: list[tuple[int, int, float]] = []
+ # (GPU count, target-only ctx, budget with drafter, checkpoints) per
+ # drafter-ranked subset; a discrete-tier subset pays no checkpoints.
+ _target_placements: list[tuple[int, int, float, int]] = []
_drafter_order = _probe_orders[-1]
for _probe_ranked in _probe_orders:
if _both_fit_somewhere:
break
_probe_min_gpus = _probe_floor(_probe_ranked, False)
- for _n in range(_probe_min_gpus, len(_probe_ranked) + 1):
- _subset = _probe_ranked[:_n]
+ # The placement loop's subsets in its order: discrete first.
+ for _n, _subset in _tiered_subsets(_probe_ranked, _probe_min_gpus):
_cc_n = lambda c, _k = _n: _cc_bytes(c, _k)
_base_wo = _probe_base(False, _n)
_budget_wo = _pool_budget_mib(_subset, _probe_frac(False))
@@ -27947,7 +28082,7 @@ class LlamaCppBackend:
budget_frac = 1.0,
pooled = True,
total_mib = None,
- ctx_checkpoints = _placement_ctx_checkpoints,
+ ctx_checkpoints = _subset_ckpts(_subset),
)
)
if _ctx_wo <= 0:
@@ -28001,7 +28136,9 @@ class LlamaCppBackend:
_budget_w,
)
if _probe_ranked is _drafter_order:
- _target_placements.append((_n, _ctx_wo, _budget_w))
+ _target_placements.append(
+ (_n, _ctx_wo, _budget_w, _subset_ckpts(_subset))
+ )
if _foot_w <= _budget_w:
_both_fit_somewhere = True
break
@@ -28030,7 +28167,7 @@ class LlamaCppBackend:
budget_frac = 1.0,
pooled = True,
total_mib = None,
- ctx_checkpoints = _placement_ctx_checkpoints,
+ ctx_checkpoints = _subset_ckpts(_subset),
)
)
if (
@@ -28089,7 +28226,9 @@ class LlamaCppBackend:
+ _mtp_bytes(ctx, s, ub)
) / (1024 * 1024)
- def _ctx_with_drafter(n: int, ctx: int, budget: float, s: int) -> int:
+ def _ctx_with_drafter(
+ n: int, ctx: int, budget: float, s: int, ckpts: int
+ ) -> int:
if explicit_ctx:
return ctx if _mtp_foot_mib(n, ctx, s) <= budget else 0
ub = _ubatch_for_slots(s)
@@ -28113,7 +28252,7 @@ class LlamaCppBackend:
budget_frac = 1.0,
pooled = True,
total_mib = None,
- ctx_checkpoints = _placement_ctx_checkpoints,
+ ctx_checkpoints = ckpts,
)
return (
fitted
@@ -28126,8 +28265,8 @@ class LlamaCppBackend:
_keep = 1 - (
_MTP_SLOT_CTX_TRIM if _s == 1 and not explicit_ctx else 0
)
- for _n, _ctx_c, _budget_c in _target_placements:
- _c = _ctx_with_drafter(_n, _ctx_c, _budget_c, _s)
+ for _n, _ctx_c, _budget_c, _ckpts_c in _target_placements:
+ _c = _ctx_with_drafter(_n, _ctx_c, _budget_c, _s, _ckpts_c)
if _c <= 0:
continue
if _c >= _ctx_c * _keep:
@@ -28245,6 +28384,22 @@ class LlamaCppBackend:
def _subset_model_size(n_gpus: int) -> int:
return model_size_fit + max(0, n_gpus - 1) * _pipeline_overhead_bytes
+ def _discrete_host_only(subset = None) -> int:
+ # The shared-pool projector charge a discrete tier drops: only when
+ # every discrete total is known, and none for a subset holding an
+ # iGPU. Read by name so a later rebind of _shared_pool_mmproj carries.
+ if subset is not None and not _integrated_gpus.isdisjoint(
+ i for i, _f in subset
+ ):
+ return 0
+ if _integrated_gpus and all(
+ (total_by_idx or {}).get(i, 0) > 0
+ for i, _f in gpus
+ if i not in _integrated_gpus
+ ):
+ return _shared_pool_mmproj
+ return 0
+
# Unified-memory budget (0 off Apple Silicon) for the no-GPU Metal cap below.
_apple_budget_mib = self._apple_metal_memory_budget_bytes() // (1024 * 1024)
@@ -28457,6 +28612,8 @@ class LlamaCppBackend:
min_gpus = _layer_min_gpus,
split_extra_bytes = _cc_split_extra(effective_ctx),
shared = _shared_gpus,
+ integrated = _integrated_gpus,
+ discrete_size_bytes = requested_total - _discrete_host_only(),
)
# No silent shrink: effective_ctx stays == requested_ctx.
# use_fit = the pin failed and --fit on will offload; say why.
@@ -28567,10 +28724,16 @@ class LlamaCppBackend:
if _unshared is not None:
ranked.remove(_unshared)
ranked.insert(0, _unshared)
- for n_gpus in range(_auto_min_gpus, len(ranked) + 1):
- subset = ranked[:n_gpus]
+ _subsets = _tiered_subsets(ranked, _auto_min_gpus)
+
+ def _subset_ms(n_gpus: int, subset) -> int:
+ # A CPU-pinned projector is charged for a shared pool; a
+ # discrete-tier subset draws on none.
+ return _subset_model_size(n_gpus) - _discrete_host_only(subset)
+
+ for n_gpus, subset in _subsets:
pool_budget = _pool_budget_mib(subset, pin_fraction)
- _ms = _subset_model_size(n_gpus)
+ _ms = _subset_ms(n_gpus, subset)
# Compute buffer is replicated per device in a layer
# split, so scale the context term by the subset size.
_cc_sub = lambda c, n = n_gpus: _cc_bytes(c, n)
@@ -28590,7 +28753,7 @@ class LlamaCppBackend:
budget_frac = 1.0,
pooled = True,
total_mib = None,
- ctx_checkpoints = _placement_ctx_checkpoints,
+ ctx_checkpoints = _subset_ckpts(subset),
)
kv = _kv_bytes(capped)
footprint_mib = (
@@ -28635,16 +28798,15 @@ class LlamaCppBackend:
use_fit = False
break
else:
- # No discrete-GPU subset holds the model at a fitted
- # context. Prefer a useful chat context before
+ # No GPU subset (discrete or full pool) holds the model at a
+ # fitted context. Prefer a useful chat context before
# handing placement to --fit on and host offload.
effective_ctx = min(_AUTO_OFFLOAD_CTX, effective_ctx)
if effective_ctx > 0:
- for n_gpus in range(_auto_min_gpus, len(ranked) + 1):
- subset = ranked[:n_gpus]
+ for n_gpus, subset in _subsets:
kv = _kv_bytes(effective_ctx)
footprint_mib = (
- _subset_model_size(n_gpus)
+ _subset_ms(n_gpus, subset)
+ kv
+ _mtp_bytes(effective_ctx)
+ _cc_bytes(effective_ctx, n_gpus)
@@ -28683,6 +28845,8 @@ class LlamaCppBackend:
per_device_overhead_bytes = _pipeline_overhead_bytes,
min_gpus = _layer_min_gpus,
shared = _shared_gpus,
+ integrated = _integrated_gpus,
+ discrete_size_bytes = _fs_total - _discrete_host_only(),
)
if use_fit and not explicit_ctx:
# Without KV metadata, llama.cpp owns the fit. Keep the
@@ -29100,6 +29264,8 @@ class LlamaCppBackend:
ubatch_for_slots = _ubatch_for_slots,
mtp_bytes_for_slots = lambda s, ub: _mtp_bytes(_reduce_ctx, s, ub),
ctx_checkpoints = _placement_ctx_checkpoints,
+ integrated = _integrated_gpus,
+ discrete_host_only_bytes = _discrete_host_only(),
)
if not _uf_slots:
_slots_asked = n_parallel
@@ -29135,6 +29301,8 @@ class LlamaCppBackend:
include_requested = True,
exact = True, # only n_parallel is accepted below
ctx_checkpoints = _placement_ctx_checkpoints,
+ integrated = _integrated_gpus,
+ discrete_host_only_bytes = _discrete_host_only(),
)
return _gi if not _uf and _got == n_parallel else None
diff --git a/studio/backend/tests/test_ctx_checkpoint_host_budget.py b/studio/backend/tests/test_ctx_checkpoint_host_budget.py
index ac4c7aa29b4..a1abf1df8d9 100644
--- a/studio/backend/tests/test_ctx_checkpoint_host_budget.py
+++ b/studio/backend/tests/test_ctx_checkpoint_host_budget.py
@@ -478,7 +478,9 @@ class TestCheckpointsNeverReachAVramFigure:
assert "0 if self._rollback_state_bytes(1) > 0 else _requested_ctx_checkpoints" in source
assert source.count("ctx_checkpoints = _fit_ctx_checkpoints,") == 1
# +1: the drafter probe's one-slot fit (#8872)
- assert source.count("ctx_checkpoints = _placement_ctx_checkpoints,") == 7
+ # Per subset (incl. the drafter's slot search): a discrete-tier subset pays none.
+ assert source.count("ctx_checkpoints = _placement_ctx_checkpoints,") == 3
+ assert source.count("ctx_checkpoints = _subset_ckpts(") == 3
assert source.count("_kv_bytes(effective_ctx, _effective_ctx_checkpoints)") == 1
def test_the_fit_is_unmoved_by_the_new_term(self):
diff --git a/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py b/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py
index 44aef0457b0..bd6c1d3d1d3 100644
--- a/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py
+++ b/studio/backend/tests/test_gpu_arch_gate_os_matrix_7624.py
@@ -2559,6 +2559,14 @@ class TestArchCrashRetryReplacesTheCrashedSelectionsWarning:
):
# 30000MiB free holds the 20GB model outright, so the respawn spills nothing.
torch = self._apu_then_dgpu(monkeypatch, dgpu_free_mib = 30000)
+ # Auto now pins a dGPU that holds the model before the APU; drop that tier so
+ # the crashed-APU-first selection whose verdict is under test still happens.
+ _select = LlamaCppBackend._select_gpus
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_select_gpus",
+ staticmethod(lambda *a, integrated = frozenset(), **k: _select(*a, **k)),
+ )
calls: list = []
capture: dict = {}
launches = _run_auto_load(
@@ -2704,10 +2712,10 @@ class TestAnInstallFromBeforeThisPr:
assert len(launches) == 1
_cmd, env = launches[0]
assert env.get("HIP_VISIBLE_DEVICES") != "-1", "an old install was gated onto the CPU"
- # Pre-PR placement, reproduced: the shared-pool iGPU still outranks the dGPU
- # (#7669) until the install is refreshed. CUDA_VISIBLE_DEVICES is 0 because
- # ROCr re-indexes the visible agents from zero; the physical pick is ROCR's.
- assert _visibility(env) == {"ROCR_VISIBLE_DEVICES": "1", "CUDA_VISIBLE_DEVICES": "0"}
+ # Not gated, and auto now prefers the dGPU that holds the model over the
+ # shared-pool iGPU (#7624). CUDA_VISIBLE_DEVICES is 0 because ROCr re-indexes
+ # the visible agents from zero; the physical pick is ROCR's.
+ assert _visibility(env) == {"ROCR_VISIBLE_DEVICES": "0", "CUDA_VISIBLE_DEVICES": "0"}
class TestArchForcedCpuFlagLifecycle:
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index 9d91a24262f..a9d42a35b99 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -5887,3 +5887,528 @@ def test_the_header_scan_refuses_what_it_cannot_parse(tmp_path):
with pytest.raises(ValueError):
LlamaCppBackend._gguf_scan_tensor_bytes(str(bad))
assert LlamaCppBackend()._gguf_tensor_scan(str(bad)) is None
+
+
+# Discrete before integrated (#7624): a unified-memory APU's "free" is system RAM.
+_APU = (0, 60_000, 0)
+_DGPU = (1, 16_000, 16_384)
+
+
+def _mixed_host_launch(
+ tmp_path,
+ monkeypatch,
+ memory,
+ *,
+ gguf_gb,
+ path,
+ rocm = (),
+ cuda = (),
+ setup = None,
+ **kw,
+):
+ monkeypatch.setattr(
+ LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: set(rocm))
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_INTEGRATED_CUDA_IDS",
+ {LlamaCppBackend._integrated_cuda_mask_key(): set(cuda)},
+ )
+ backend, gguf = _backend(tmp_path, vulkan = False, memory = memory)
+ backend._get_gguf_size_bytes = lambda _path: int(gguf_gb * 1024**3)
+ if path != "filesize":
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * 64 * 1024
+ backend._compute_buffer_ctx_bytes = lambda *a, **k: 0
+ backend._estimate_compute_buffer_bytes = lambda **k: 1
+
+ def read_metadata(_path):
+ backend._context_length = 131_072
+
+ backend._read_gguf_metadata = read_metadata
+ kw["n_ctx"] = 0 if path == "auto_ctx" else 8192
+ if setup is not None:
+ setup(backend)
+ return _launch(backend, gguf, **kw)["env"].get("CUDA_VISIBLE_DEVICES")
+
+
[email protected]("path", ["filesize", "auto_ctx", "explicit_ctx"])
+ "memory, rocm, cuda, expected",
+ [
+ ([_APU, _DGPU], {0}, (), "1"),
+ ([(0, 60_000, 65_536), _DGPU], {0}, (), "1"),
+ ([(0, 16_000, 16_384), (1, 60_000, 0)], {1}, (), "0"),
+ ([_APU, _DGPU], (), {0}, "1"),
+ ],
+ ids = ["rocm_apu_total0", "rocm_apu_total", "apu_second", "cuda_integrated"],
+)
+def test_auto_placement_prefers_a_discrete_gpu_that_holds_the_model(
+ tmp_path, monkeypatch, memory, rocm, cuda, expected, path
+):
+ got = _mixed_host_launch(
+ tmp_path, monkeypatch, memory, gguf_gb = 8, path = path, rocm = rocm, cuda = cuda
+ )
+ assert got == expected
+
+
[email protected]("path", ["filesize", "auto_ctx", "explicit_ctx"])
+ "memory, rocm, gguf_gb, gpu_ids, expected",
+ [
+ # Homogeneous hosts: most usable VRAM wins, as before.
+ ([(0, 10_000, 16_384), (1, 16_000, 16_384)], (), 8, None, "1"),
+ ([_APU, (1, 30_000, 0)], {0, 1}, 8, None, "0"),
+ ([_APU, _DGPU], {0}, 20, None, "0"),
+ # User pick / mask leaving only the APU.
+ ([_APU, _DGPU], {0}, 8, [0], "0"),
+ ([_APU, _DGPU], {0}, 8, [0, 1], "0"),
+ ([_APU], {0}, 8, None, "0"),
+ ],
+ ids = ["discrete_only", "integrated_only", "only_apu_fits", "pick_apu", "pick_both", "mask_apu"],
+)
+def test_discrete_first_leaves_other_placements_alone(
+ tmp_path, monkeypatch, memory, rocm, gguf_gb, gpu_ids, expected, path
+):
+ kw = {"gpu_ids": gpu_ids} if gpu_ids is not None else {}
+ got = _mixed_host_launch(
+ tmp_path, monkeypatch, memory, gguf_gb = gguf_gb, path = path, rocm = rocm, **kw
+ )
+ assert got == expected
+
+
+def test_select_gpus_discrete_tier_keeps_a_spread_request_spread():
+ gib = 1024**3
+ gpus = [(0, 60_000), (1, 16_000)]
+ total = {0: 0, 1: 16_384}
+ # min_gpus=2 but one discrete GPU: both count.
+ assert LlamaCppBackend._select_gpus(
+ 8 * gib, gpus, total_by_idx = total, min_gpus = 2, integrated = frozenset({0})
+ ) == ([0, 1], False)
+ assert LlamaCppBackend._select_gpus(
+ 8 * gib, gpus, total_by_idx = total, integrated = frozenset({0})
+ ) == ([1], False)
+ assert LlamaCppBackend._select_gpus(8 * gib, gpus, total_by_idx = total) == ([0], False)
+
+
+def _mixed_drafter_launch(tmp_path, monkeypatch, *, speculative_type):
+ """14 GiB target and a sidecar drafter; the dGPU holds the target alone at native context."""
+ gb, mib = 1024**3, 1024 * 1024
+ monkeypatch.setattr(LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {0}))
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_INTEGRATED_CUDA_IDS",
+ {LlamaCppBackend._integrated_cuda_mask_key(): set()},
+ )
+ backend, gguf = _backend(tmp_path, vulkan = False, memory = [(0, 60_000, 0), (1, 24_576, 24_576)])
+ sidecar = tmp_path / "dspark-model-Q8_0.gguf"
+ sidecar.write_bytes(b"draft")
+ backend._get_gguf_size_bytes = lambda path: (
+ int(0.5 * gb) if str(path) == str(sidecar) else 14 * gb
+ )
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * mib // 4
+ backend._compute_buffer_ctx_bytes = lambda *a, **k: 0
+ backend._estimate_compute_buffer_bytes = lambda **k: 1
+ backend._mtp_draft_kv_bytes = lambda *a, **k: 0
+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, *a, **k: int(n_ctx) * mib // 4
+
+ def read_metadata(_path):
+ backend._context_length = 32_768
+
+ backend._read_gguf_metadata = read_metadata
+ backend.probe_server_capabilities = lambda _binary = None: {
+ "supports_dspark": True,
+ "spec_draft_n_max_flag": "--spec-draft-n-max",
+ }
+ kw = {"dspark_draft_path": str(sidecar)} if speculative_type == "auto" else {}
+ result = _launch(backend, gguf, n_ctx = 0, speculative_type = speculative_type, **kw)
+ cmd = result["cmd"]
+ return result["env"].get("CUDA_VISIBLE_DEVICES"), int(cmd[cmd.index("-c") + 1]), cmd
+
+
+def test_drafter_probe_prices_the_discrete_tier_the_placement_takes(tmp_path, monkeypatch):
+ # The APU holds target + drafter; the dGPU holds the target only at full context. The
+ # probe must answer for the dGPU placement, not keep a drafter that costs it context.
+ _dev, ctx_off, _cmd = _mixed_drafter_launch(tmp_path, monkeypatch, speculative_type = "off")
+ dev, ctx, cmd = _mixed_drafter_launch(tmp_path, monkeypatch, speculative_type = "auto")
+ assert dev == "1"
+ assert ctx == ctx_off == 32_768
+ assert "--model-draft" not in cmd
+
+
+def test_cuda_integrated_ids_join_pci_rows_only_under_pci_bus_order(tmp_path, monkeypatch):
+ # nvidia-smi rows are PCI indices, CUDA ids are ordinals: with FASTEST_FIRST they can
+ # name different cards, so the tier stands down and base ranking applies.
+ monkeypatch.setattr(LlamaCppBackend, "_GPU_IDS_ARE_PCI_INDICES", True)
+ monkeypatch.delenv("CUDA_DEVICE_ORDER", raising = False)
+ memory = [_APU, _DGPU]
+ assert (
+ _mixed_host_launch(tmp_path, monkeypatch, memory, gguf_gb = 8, path = "filesize", cuda = {0})
+ == "0"
+ )
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ assert (
+ _mixed_host_launch(tmp_path, monkeypatch, memory, gguf_gb = 8, path = "filesize", cuda = {0})
+ == "1"
+ )
+
+
+def test_split_aware_discrete_tier_falls_back_to_the_base_pick():
+ # Two dGPUs pool the model at the single-device rate but not as a split; the APU
+ # alone holds it. Same answer as without the tier, not a mixed APU+dGPU split.
+ mib = 1024 * 1024
+ gpus = [(0, 12_000), (1, 12_000), (2, 21_000)]
+ kw = dict(usable_fraction = 1.0, split_extra_bytes = 2_000 * mib)
+ base = LlamaCppBackend._select_gpus_split_aware(20_480 * mib, gpus, **kw)
+ assert base == ([2], False)
+ assert (
+ LlamaCppBackend._select_gpus_split_aware(
+ 20_480 * mib, gpus, integrated = frozenset({2}), **kw
+ )
+ == base
+ )
+ # A discrete split that holds it at the split rate still wins.
+ assert LlamaCppBackend._select_gpus_split_aware(
+ 16_000 * mib, gpus, integrated = frozenset({2}), **kw
+ ) == ([0, 1], False)
+
+
+def test_discrete_tier_subset_is_not_charged_host_checkpoints(tmp_path, monkeypatch):
+ # Checkpoints live in host RAM: only a unified-memory GPU's "VRAM" pays for them.
+ seen = []
+ real_fit = LlamaCppBackend._fit_context_to_vram
+
+ def spy(
+ self,
+ requested,
+ available_mib,
+ *a,
+ ctx_checkpoints = 0,
+ **k,
+ ):
+ seen.append((available_mib, ctx_checkpoints))
+ return real_fit(self, requested, available_mib, *a, ctx_checkpoints = ctx_checkpoints, **k)
+
+ def setup(backend):
+ backend._rollback_state_bytes = lambda *a, **k: 0
+ backend._amd_apu_wants_unified_memory = lambda ids = None, *a, **k: 0 in (ids or [0])
+ backend.probe_server_capabilities = lambda _binary = None: {
+ "ctx_checkpoints_flag": "--ctx-checkpoints"
+ }
+
+ monkeypatch.setattr(LlamaCppBackend, "_fit_context_to_vram", spy)
+ got = _mixed_host_launch(
+ tmp_path,
+ monkeypatch,
+ [_APU, _DGPU],
+ gguf_gb = 8,
+ path = "auto_ctx",
+ rocm = {0},
+ setup = setup,
+ ctx_checkpoints = 32,
+ )
+ assert got == "1"
+ # The APU candidate makes the pool pay; the dGPU-only subset does not.
+ assert seen and seen[-1][1] == 0 and seen[-1][0] < 16_384
+
+
+def _host_projector_launch(tmp_path, monkeypatch, memory, rocm, gguf_gb):
+ """Explicit 8K vision load whose 3 GiB projector the user keeps on the CPU."""
+ gb = 1024**3
+
+ def setup(backend):
+ mmproj = tmp_path / "mmproj.gguf"
+ mmproj.write_bytes(b"GGUF")
+ backend._mmproj_vram_bytes = lambda _path: 3 * gb
+ backend._resolve_launch_mmproj_path = lambda **_kw: str(mmproj)
+
+ return _mixed_host_launch(
+ tmp_path,
+ monkeypatch,
+ memory,
+ gguf_gb = gguf_gb,
+ path = "explicit_ctx",
+ rocm = rocm,
+ setup = setup,
+ is_vision = True,
+ extra_args = ["--no-mmproj-offload"],
+ )
+
+
+def test_explicit_ctx_discrete_tier_is_not_charged_a_host_projector(tmp_path, monkeypatch):
+ # The projector sits in host RAM: the APU pool pays for it, the dGPU does not.
+ assert _host_projector_launch(tmp_path, monkeypatch, [_APU, _DGPU], {0}, 13) == "1"
+
+
+def test_explicit_ctx_host_projector_control_without_integrated_gpu(tmp_path, monkeypatch):
+ # No unified-memory GPU: the larger dGPU wins exactly as before the discrete tier.
+ memory = [(0, 60_000, 65_536), _DGPU]
+ assert _host_projector_launch(tmp_path, monkeypatch, memory, (), 13) == "0"
+
+
+def test_slot_search_discrete_tier_is_not_charged_host_checkpoints():
+ # Two slots fit the dGPU; 16 checkpoints only fit the APU's host-RAM pool.
+ mib = 1024 * 1024
+ backend = LlamaCppBackend()
+ backend._estimate_compute_buffer_bytes = lambda **k: 1
+ backend._estimate_kv_cache_bytes = lambda ctx, *a, n_parallel = 1, ctx_checkpoints = 0, **k: (
+ n_parallel * 1_000 * mib + ctx_checkpoints * 500 * mib
+ )
+ gpus = [(0, 60_000), (1, 16_000)]
+ total = {0: 0, 1: 16_384}
+ kw = dict(include_requested = True, ctx_checkpoints = 16)
+ args = (2, 8192, gpus, total, 10_000 * mib, None, 1.0, 0, 1)
+ assert backend._slots_that_fit_on_gpu(*args, integrated = frozenset({0}), **kw) == (
+ [1],
+ False,
+ 2,
+ )
+ # Control: no integrated GPU prices every candidate with the checkpoints, as before.
+ assert backend._slots_that_fit_on_gpu(*args, **kw) == ([0], False, 2)
+
+
[email protected]("path", ["filesize", "explicit_ctx"])
+def test_host_projector_is_not_charged_to_the_discrete_tier(tmp_path, monkeypatch, path):
+ # Both the file-size and explicit paths drop the host-RAM projector for the dGPU.
+ gb = 1024**3
+ # The file-size path carries a larger flat overhead, so a smaller model.
+ gguf_gb = 9.5 if path == "filesize" else 13
+
+ def setup(backend):
+ mmproj = tmp_path / "mmproj.gguf"
+ mmproj.write_bytes(b"GGUF")
+ backend._mmproj_vram_bytes = lambda _path: 3 * gb
+ backend._resolve_launch_mmproj_path = lambda **_kw: str(mmproj)
+
+ def launch(memory, rocm):
+ return _mixed_host_launch(
+ tmp_path,
+ monkeypatch,
+ memory,
+ gguf_gb = gguf_gb,
+ path = path,
+ rocm = rocm,
+ setup = setup,
+ is_vision = True,
+ extra_args = ["--no-mmproj-offload"],
+ )
+
+ assert launch([_APU, _DGPU], {0}) == "1"
+ # Control: no integrated GPU, the larger card wins as before.
+ assert launch([(0, 60_000, 65_536), _DGPU], ()) == "0"
+
+
+def test_slot_refit_on_narrowed_dgpus_drops_host_only_terms():
+ # The re-fit runs on the dGPU the reduction chose: no checkpoints, no host projector.
+ mib = 1024 * 1024
+ backend = LlamaCppBackend()
+ backend._estimate_compute_buffer_bytes = lambda **k: 1
+ backend._estimate_kv_cache_bytes = lambda ctx, *a, n_parallel = 1, ctx_checkpoints = 0, **k: (
+ n_parallel * 1_000 * mib + ctx_checkpoints * 500 * mib
+ )
+ dgpu = [(1, 16_000)]
+ total = {0: 0, 1: 16_384}
+ kw = dict(include_requested = True, exact = True)
+ args = (2, 8192, dgpu, total, 10_000 * mib, None, 1.0, 0, 1)
+ integ = frozenset({0})
+ assert backend._slots_that_fit_on_gpu(*args, ctx_checkpoints = 16, integrated = integ, **kw) == (
+ [1],
+ False,
+ 2,
+ )
+ assert backend._slots_that_fit_on_gpu(
+ 2,
+ 8192,
+ dgpu,
+ total,
+ 14_500 * mib,
+ None,
+ 1.0,
+ 0,
+ 1,
+ integrated = integ,
+ discrete_host_only_bytes = 2_000 * mib,
+ **kw,
+ ) == ([1], False, 2)
+ # Controls: no integrated GPU prices every term, as before.
+ assert backend._slots_that_fit_on_gpu(*args, ctx_checkpoints = 16, **kw) == (None, True, 2)
+ assert backend._slots_that_fit_on_gpu(
+ 2,
+ 8192,
+ dgpu,
+ total,
+ 14_500 * mib,
+ None,
+ 1.0,
+ 0,
+ 1,
+ discrete_host_only_bytes = 2_000 * mib,
+ **kw,
+ ) == (None, True, 2)
+
+
+def test_drafter_slot_search_keeps_the_discrete_placement_checkpoint_free(tmp_path, monkeypatch):
+ # The recorded dGPU placement is re-priced at fewer slots without host checkpoints.
+ gb, mib = 1024**3, 1024 * 1024
+ monkeypatch.setattr(LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {0}))
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_INTEGRATED_CUDA_IDS",
+ {LlamaCppBackend._integrated_cuda_mask_key(): set()},
+ )
+ # The APU is too small to hold the drafter's reserve beside the target.
+ backend, gguf = _backend(tmp_path, vulkan = False, memory = [(0, 2_000, 0), (1, 24_576, 24_576)])
+ backend._get_gguf_size_bytes = lambda _path: 14 * gb
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda ctx, *a, n_parallel = 1, ctx_checkpoints = 0, **k: (
+ int(ctx) * mib // 16 * n_parallel + ctx_checkpoints * 64 * mib
+ )
+ backend._compute_buffer_ctx_bytes = lambda *a, **k: 0
+ backend._estimate_compute_buffer_bytes = lambda **k: 1
+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, *a, _np = None, n_parallel = None, **k: int(
+ 3 * gb * int(_np or n_parallel or 1)
+ )
+ backend._rollback_state_bytes = lambda *a, **k: 0
+ backend._amd_apu_wants_unified_memory = lambda ids = None, *a, **k: 0 in (ids or [0])
+
+ def read_metadata(_path):
+ backend._nextn_predict_layers = 1
+ backend._context_length = 32_768
+
+ backend._read_gguf_metadata = read_metadata
+ backend.probe_server_capabilities = lambda _binary = None: {
+ "mtp_token": "draft-mtp",
+ "supports_ngram_mod": True,
+ "spec_draft_n_max_flag": "--spec-draft-n-max",
+ "supports_kv_unified": True,
+ "ctx_checkpoints_flag": "--ctx-checkpoints",
+ }
+ seen = []
+ real_fit = LlamaCppBackend._fit_context_to_vram
+
+ def spy(
+ self,
+ *a,
+ n_parallel = 1,
+ mtp_engaged = False,
+ ctx_checkpoints = 0,
+ **k,
+ ):
+ seen.append((n_parallel, mtp_engaged, ctx_checkpoints))
+ return real_fit(
+ self,
+ *a,
+ n_parallel = n_parallel,
+ mtp_engaged = mtp_engaged,
+ ctx_checkpoints = ctx_checkpoints,
+ **k,
+ )
+
+ monkeypatch.setattr(LlamaCppBackend, "_fit_context_to_vram", spy)
+ _launch_default_slots(backend, gguf, ctx_checkpoints = 32)
+ reduced = [c for n, m, c in seen if m and n < 4]
+ assert reduced, seen
+ assert any(c for _n, _m, c in seen), "the APU pool should still pay checkpoints"
+ assert reduced[0] == 0, seen
+
+
+def _shared_host_launch(tmp_path, monkeypatch, memory, rocm, path):
+ def setup(backend):
+ backend._other_planned_vram_mib = lambda: {1: 0}
+
+ return _mixed_host_launch(
+ tmp_path, monkeypatch, memory, gguf_gb = 8, path = path, rocm = rocm, setup = setup
+ )
+
+
[email protected]("path", ["filesize", "auto_ctx", "explicit_ctx"])
+ "memory, rocm, expected",
+ [
+ # Only the shared dGPU holds it among discrete cards: the base shared preference
+ # (an unshared card that fits) applies over every card.
+ ([_APU, (1, 16_000, 16_384), (2, 4_000, 4_096)], {0}, "0"),
+ # An unshared dGPU that fits still wins over the unshared APU.
+ ([_APU, (1, 16_000, 16_384), (2, 16_000, 16_384)], {0}, "2"),
+ # Control: no integrated GPU, base pick.
+ ([(0, 60_000, 65_536), (1, 16_000, 16_384), (2, 4_000, 4_096)], (), "0"),
+ ],
+ ids = ["shared_dgpu_only", "unshared_dgpu", "no_igpu_control"],
+)
+def test_discrete_tier_respects_the_shared_card_preference(
+ tmp_path, monkeypatch, memory, rocm, expected, path
+):
+ assert _shared_host_launch(tmp_path, monkeypatch, memory, rocm, path) == expected
+
+
+def test_select_gpus_discrete_tier_skips_shared_dgpus():
+ gib = 1024**3
+ gpus = [(0, 60_000), (1, 16_000), (2, 4_000)]
+ total = {0: 0, 1: 16_384, 2: 4_096}
+ kw = dict(total_by_idx = total, shared = frozenset({1}))
+ base = LlamaCppBackend._select_gpus(8 * gib, gpus, **kw)
+ assert base == ([0], False)
+ assert LlamaCppBackend._select_gpus(8 * gib, gpus, integrated = frozenset({0}), **kw) == base
+ assert (
+ LlamaCppBackend._select_gpus_split_aware(8 * gib, gpus, integrated = frozenset({0}), **kw)
+ == base
+ )
+
+
+def test_projector_preflight_does_not_count_an_apu_as_private_vram(tmp_path, monkeypatch):
+ # An APU reporting a total offered "room" for the projector; only moving it to host
+ # lets the dGPU hold the model.
+ gb = 1024**3
+
+ def setup(backend):
+ mmproj = tmp_path / "mmproj.gguf"
+ mmproj.write_bytes(b"GGUF")
+ backend._mmproj_vram_bytes = lambda _path: 3 * gb
+ backend._resolve_launch_mmproj_path = lambda **_kw: str(mmproj)
+
+ def launch(memory, rocm):
+ return _mixed_host_launch(
+ tmp_path,
+ monkeypatch,
+ memory,
+ gguf_gb = 13,
+ path = "explicit_ctx",
+ rocm = rocm,
+ setup = setup,
+ is_vision = True,
+ )
+
+ assert launch([(0, 60_000, 65_536), _DGPU], {0}) == "1"
+ # Control: the same big card as a dGPU keeps the projector there, as before.
+ assert launch([(0, 60_000, 65_536), _DGPU], ()) == "0"
+
+
[email protected]("shared, expected_gpu, pinned", [(True, "0", False), (False, "1", True)])
+def test_projector_preflight_skips_a_shared_dgpu_like_placement(
+ tmp_path, monkeypatch, shared, expected_gpu, pinned
+):
+ # The occupied dGPU fits only with the projector on host, but placement takes the
+ # unshared APU, where pinning the projector frees nothing.
+ gb = 1024**3
+ mmproj = tmp_path / "mmproj.gguf"
+ mmproj.write_bytes(b"GGUF")
+ monkeypatch.setattr(LlamaCppBackend, "_rocm_unified_memory_gpu_ids", staticmethod(lambda: {0}))
+ backend, gguf = _backend(tmp_path, vulkan = False, memory = [(0, 60_000, 65_536), _DGPU])
+ backend._get_gguf_size_bytes = lambda _path: 13 * gb
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda ctx, *a, **k: int(ctx) * 64 * 1024
+ backend._compute_buffer_ctx_bytes = lambda *a, **k: 0
+ backend._estimate_compute_buffer_bytes = lambda **k: 1
+
+ def read_metadata(_path):
+ backend._context_length = 131_072
+
+ backend._read_gguf_metadata = read_metadata
+ backend._mmproj_vram_bytes = lambda _path: 3 * gb
+ backend._resolve_launch_mmproj_path = lambda **_kw: str(mmproj)
+ if shared:
+ backend._other_planned_vram_mib = lambda: {1: 0}
+ out = _launch(backend, gguf, n_ctx = 8192, is_vision = True)
+ assert out["env"].get("CUDA_VISIBLE_DEVICES") == expected_gpu
+ assert ("--no-mmproj-offload" in out["cmd"]) is pinned
diff --git a/studio/backend/tests/test_llama_cpp_vulkan_probe.py b/studio/backend/tests/test_llama_cpp_vulkan_probe.py
index 47f744d1ef3..6a527f48598 100644
--- a/studio/backend/tests/test_llama_cpp_vulkan_probe.py
+++ b/studio/backend/tests/test_llama_cpp_vulkan_probe.py
@@ -231,6 +231,26 @@ def test_large_discrete_gpu_is_untouched(tmp_path):
assert gpus == [(0, 47 * 1024, 48 * 1024)], gpus
+ "free_mib, total_mib, is_igpu, expected_free",
+ [
+ # Intel's Windows driver reports a heap budget above the heap size.
+ (18 * 1024, 16 * 1024, 0, 16 * 1024),
+ (40 * 1024, 32 * 1024, 1, 32 * 1024),
+ (6 * 1024, 24 * 1024, 0, 6 * 1024),
+ (24 * 1024, 24 * 1024, 0, 24 * 1024),
+ (8 * 1024, 0, 1, 8 * 1024),
+ ],
+ ids = ["dgpu_over", "igpu_over", "dgpu_under", "dgpu_equal", "total_unknown"],
+)
+def test_vulkan_free_never_exceeds_total(tmp_path, free_mib, total_mib, is_igpu, expected_free):
+ binary = _make_vulkan_install(tmp_path)
+ rows = [_row(0, free_mib * MIB, is_igpu = is_igpu, total_bytes = total_mib * MIB, name = "GPU")]
+ with _mock_probe(rows):
+ (row,) = LlamaCppBackend._run_vulkan_probe(binary)
+ assert (row["free_mib"], row["total_mib"]) == (expected_free, total_mib)
+
+
def test_inherited_visible_devices_mask_is_passed_through_to_probe(tmp_path, monkeypatch):
# The mask is NOT stripped or filtered in Python: ggml parses it in raw
# physical-device space while this probe reports the compact post-filter
diff --git a/studio/backend/tests/test_tp_vision_regression.py b/studio/backend/tests/test_tp_vision_regression.py
index a1e8d4251c6..c70478c7091 100644
--- a/studio/backend/tests/test_tp_vision_regression.py
+++ b/studio/backend/tests/test_tp_vision_regression.py
@@ -251,7 +251,7 @@ def test_vision_downgrade_preserves_multi_gpu_intent():
src = inspect.getsource(LlamaCppBackend.load_model)
assert "_layer_min_gpus = max(_layer_min_gpus, len(gpus))" in src
assert src.count("min_gpus = _layer_min_gpus") >= 2
- assert "range(_auto_min_gpus, len(ranked) + 1)" in src
+ assert "_tiered_subsets(ranked, _auto_min_gpus)" in src
auto = src.find("_auto_min_gpus = max(")
assert auto != -1 and "_layer_min_gpus" in src[auto : auto + 200]
@@ -481,7 +481,7 @@ def test_auto_context_layer_loops_capped_to_usable_gpus():
"range(max(1, _layer_min_gpus), len(ranked) + 1)" not in src
), "auto-context loops must cap _layer_min_gpus to usable GPUs, not use it raw"
assert "_auto_min_gpus" in src
- assert "range(_auto_min_gpus, len(ranked) + 1)" in src
+ assert "_tiered_subsets(ranked, _auto_min_gpus)" in src
# the eligibility threshold is the per-device layer overhead, not bare > 0
auto = src.find("_auto_min_gpus = max(")
assert auto != -1