Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: lower the VRAM budget floor to 10% for multi-GPU spreading (#13427)

Shooter3k80d23aaa0c91
diff --git a/studio/backend/core/inference/engine_adapters.py b/studio/backend/core/inference/engine_adapters.py
index 7237b110e99..e7c00cf8c36 100644
--- a/studio/backend/core/inference/engine_adapters.py
+++ b/studio/backend/core/inference/engine_adapters.py
@@ -398,6 +398,9 @@ def memory_reserve_mib(engine: str, options: dict | None) -> int:
RESERVE_SHARE = 0.06
+MANAGED_ENGINE_BUDGET_FLOOR = 0.80
+
+
def gpu_memory_fraction(
gpu_ids: list[int],
reserve_mib: int = 512,
@@ -416,7 +419,9 @@ def gpu_memory_fraction(
rows = (measure or _nvidia_memory_rows)(gpu_ids)
if len(rows) != len(gpu_ids):
raise ValueError("Could not measure every selected GPU")
- fraction = get_vram_budget_fraction()
+ # vLLM / SGLang cannot offload to the CPU, so a GGUF-only low budget must not shrink them;
+ # the per-load engine_gpu_memory_utilization still caps lower.
+ fraction = max(get_vram_budget_fraction(), MANAGED_ENGINE_BUDGET_FLOOR)
for total, free in rows:
reserve = max(reserve_mib, reserve_share * total)
if total <= 0 or free <= reserve or free > total:
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 0f07aceb310..dad0ef61894 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -6728,6 +6728,16 @@ def _extra_args_draft_offloaded_to_cpu(
return False
+def _margin_mib_for(margin: Union[float, Mapping[int, float], None], idx: int) -> float:
+ """The ``--fit-target`` margin one card keeps: a single broadcast value, or that
+ card's entry in a per-device mapping (the largest entry for a card it omits)."""
+ if isinstance(margin, Mapping):
+ if not margin:
+ return 0.0
+ return max(0.0, float(margin.get(int(idx), max(margin.values()))))
+ return max(0.0, float(margin or 0.0))
+
+
def _extra_args_device(
extra_args: Optional[Iterable[str]], flags: Collection[str]
) -> Optional[str]:
@@ -16848,7 +16858,7 @@ class LlamaCppBackend:
gpu_indices: Optional[Iterable[int]] = None,
shared_gpu_ids: Iterable[int] = (),
host_only_bytes: int = 0,
- vram_margin_mib: float = 0.0,
+ vram_margin_mib: Union[float, Mapping[int, float]] = 0.0,
avail_mib: Optional[int] = None,
headroom_mib: int = _HOST_RAM_HEADROOM_MIB,
) -> Optional[bool]:
@@ -16881,16 +16891,16 @@ class LlamaCppBackend:
per device for --fit"; common/arg.cpp, common/common.h). Under ``--fit on``
the child spills weights to host RAM rather than dip into it, so those bytes
are not credit this launch may spend. Zero on the proved path, which emits
- ``-ngl -1 --fit off`` and runs no fitter at all.
+ ``-ngl -1 --fit off`` and runs no fitter at all. A mapping gives each card the
+ margin of Studio's per-device list, keyed by physical GPU id.
"""
if not footprint_bytes or footprint_bytes <= 0:
return None
host_only = max(0, int(host_only_bytes or 0))
shared = set(shared_gpu_ids or ())
pinned = None if gpu_indices is None else {int(idx) for idx in gpu_indices}
- margin_mib = max(0.0, float(vram_margin_mib or 0.0))
free_vram_mib = sum(
- max(0, row[1] - margin_mib)
+ max(0, row[1] - _margin_mib_for(vram_margin_mib, row[0]))
for row in (detected_gpus or ())
if row[0] not in shared and (pinned is None or row[0] in pinned)
)
@@ -16923,7 +16933,7 @@ class LlamaCppBackend:
gpus: Optional[Iterable[tuple]] = None,
gpu_indices: Optional[Iterable[int]] = None,
shared_gpu_ids: Iterable[int] = (),
- fit_margin_mib: float = 0.0,
+ fit_margin_mib: Union[float, Mapping[int, float]] = 0.0,
is_vulkan_backend: bool = False,
avail_mib: Optional[int] = None,
extra_args: Optional[Iterable[str]] = None,
@@ -25691,6 +25701,10 @@ class LlamaCppBackend:
# devices are known. Signed: lowering holds more back, raising packs
# closer to the card. Zero at the default.
_fit_target_delta_mib = 0.0
+ # The same delta per physical GPU id, when the per-device list applies.
+ _fit_target_delta_by_idx: Optional[dict[int, float]] = None
+ # Studio's emitted margins per card, spelled into --fit-target at spawn.
+ _fit_target_margins_by_idx: Optional[dict[int, float]] = None
# Set only if that margin actually reached the command line, which is
# what makes the child budget-priced in a mode the planner skipped.
_fit_target_priced = False
@@ -27133,14 +27147,16 @@ class LlamaCppBackend:
# so the slider never reached the path that runs when the fit is
# tightest. Hand it the difference, both directions, measured from
# the default so an untouched slider leaves the command alone.
- # One broadcast value: --fit-target does take a per-device list, but
- # in llama.cpp's enumeration order, which the visible-device pin and
- # the ROCr/Vulkan ordinal quirks make not ours to assume, and a
- # misaligned list hands a card the wrong margin. So the direction
- # picks the card: lowering scales by the largest, so no device ends
- # up under the margin it asked for, raising by the smallest, so none
- # is taken below its own. Mixed cards gain a little less, and hold
- # back a little more, than a per-device list would.
+ # --fit-target takes a per-device list in llama.cpp's enumeration
+ # order. Where CUDA's PCI order makes that order provable, each card
+ # gets its own margin, spelled at spawn against the final mask. A
+ # single broadcast value would otherwise have to be sized by the
+ # largest card when lowering, which on mixed cards can exceed a
+ # smaller card's whole memory and leave it unused. Elsewhere (Vulkan,
+ # ROCm, UUID/MIG masks, --device/--rpc) the direction picks the card:
+ # lowering scales by the largest, so no device ends up under the
+ # margin it asked for, raising by the smallest, so none is taken
+ # below its own.
if _vram_frac != _CTX_FIT_VRAM_FRACTION and gpus:
# Free stands in for an unreported total, the same scale
# _vram_usable_mib falls back to: MIG/vGPU and the two-column
@@ -27152,6 +27168,19 @@ class LlamaCppBackend:
max(_scales) if _vram_frac < _CTX_FIT_VRAM_FRACTION else min(_scales)
)
_fit_target_delta_mib = (_CTX_FIT_VRAM_FRACTION - _vram_frac) * _scale
+ # Where llama.cpp's device order is provable, each card gets
+ # its own margin instead (_per_device_fit_target_argv). Every
+ # probed row, not just `gpus`: an unpinned child sees them all.
+ if self._fit_target_per_device_ok(
+ is_vulkan_backend = is_vulkan_backend,
+ extra_args = extra_args,
+ env = os.environ,
+ ):
+ _fit_target_delta_by_idx = {
+ int(_idx): (_CTX_FIT_VRAM_FRACTION - _vram_frac)
+ * (_total if _total and _total > 0 else _free)
+ for _idx, _free, _total in _gpu_mem
+ }
_sysmem_fallback = self._sysmem_fallback_risk(binary)
@@ -29643,6 +29672,8 @@ class LlamaCppBackend:
)
# A per-device list reads as its largest entry (the larger shortfall).
_margin_mib = fit_target_margin_in(_child_extras or (), None)
+ # Studio's own per-card margins, when it spells the list itself.
+ _margins_by_idx: Optional[dict[int, float]] = None
if _margin_mib is None:
_emits = _supports_fit_target and (
_auto_fit or _managed_mib != _LLAMA_FIT_TARGET_DEFAULT_MIB
@@ -29650,6 +29681,12 @@ class LlamaCppBackend:
_env_mib = None if _emits else fit_target_margin_in((), _child_env)
# An inherited 0 is a valid margin, not "unset".
_margin_mib = _managed_mib if _env_mib is None else _env_mib
+ if _emits:
+ _margins_by_idx = self._fit_target_margins_mib(
+ auto_fit = _auto_fit,
+ fit_target_delta_by_idx = _fit_target_delta_by_idx,
+ supports_fit_target = _supports_fit_target,
+ )
# A lazily read table stays mmap'd on the host, never on the GPU.
_lazy_host = self._lazy_read_host_bytes(
model_path,
@@ -29680,7 +29717,10 @@ class LlamaCppBackend:
)
+ _cc_bytes(effective_ctx, len(_on))
)
- _free = sum(max(0.0, free - _margin_mib) for _idx, free in _on)
+ _free = sum(
+ max(0.0, free - _margin_mib_for(_margins_by_idx or _margin_mib, _idx))
+ for _idx, free in _on
+ )
return _need - _free * 1024 * 1024
def _moe_resident_share_now() -> Optional[float]:
@@ -30039,6 +30079,20 @@ class LlamaCppBackend:
if _fitter_runs
else 0.0
)
+ # Studio's per-card list replaces the single value only when Studio
+ # owns the margin: a pass-through one wins outright (above).
+ _fit_margins_by_idx = (
+ self._fit_target_margins_mib(
+ auto_fit = (gpu_memory_mode == "manual" and gpu_layers < 0),
+ fit_target_delta_by_idx = _fit_target_delta_by_idx,
+ supports_fit_target = bool(server_caps.get("supports_fit_target")),
+ )
+ if _fitter_runs
+ and use_fit
+ and not _metal_fit_target_floor_mib
+ and fit_target_margin_in(_fit_extras, _fit_env) is None
+ else None
+ )
# A projector Unsloth never resolved but the child loads anyway:
# LLAMA_ARG_MMPROJ / _URL are read before argv and only the vision
# switch and the paravirtual pin clear them, while model_size carries
@@ -30171,7 +30225,7 @@ class LlamaCppBackend:
# A running fitter keeps --fit-target free per device, so that
# margin is not VRAM this fit may spend. Zero only when the
# fitter is provably off; see _fitter_runs above.
- fit_margin_mib = _fit_margin_mib,
+ fit_margin_mib = _fit_margins_by_idx or _fit_margin_mib,
is_vulkan_backend = is_vulkan_backend,
# Never stripped, and appended last, so they beat this
# launch's own -ngl.
@@ -30877,6 +30931,16 @@ class LlamaCppBackend:
# the capability and a budget at the default all leave the child
# unpriced, and each is decided inside that call.
_fit_target_priced = "--fit-target" in _integrity_flags
+ # Spelled per device at spawn, against the child's final mask.
+ _fit_target_margins_by_idx = (
+ self._fit_target_margins_mib(
+ auto_fit = auto_fit,
+ fit_target_delta_by_idx = _fit_target_delta_by_idx,
+ supports_fit_target = bool(server_caps.get("supports_fit_target")),
+ )
+ if _fit_target_priced and use_fit and not _metal_fit_target_floor_mib
+ else None
+ )
if _metal_fit_target_floor_mib:
# Rebuilt after the price is read: the floor is the machine's
# margin, not the budget slider's.
@@ -33083,6 +33147,9 @@ class LlamaCppBackend:
# Best-effort; never block the load on logging.
logger.warning(f"Could not open llama-server log file: {e}")
self._llama_log_path = None
+ run_cmd = self._per_device_fit_target_argv(
+ run_cmd, env, _fit_target_margins_by_idx
+ )
_last_spawn_cmd = list(run_cmd)
# Read off the argv actually spawned rather than the intent, so
# every emitter is covered by construction.
@@ -34803,6 +34870,9 @@ class LlamaCppBackend:
)
self._mmproj_fallback_reason = "projector_startup_failure"
cmd = self._strip_mmproj_args(_vision_gpu_cmd)
+ cmd = self._per_device_fit_target_argv(
+ cmd, env, _fit_target_margins_by_idx
+ )
_unified_withdraw_if_unneeded(cmd, why = "The text-only retry")
# This retry bypasses _spawn_and_wait, so refresh the
# launched-argv snapshot itself -- the zero-offload
@@ -38457,6 +38527,126 @@ class LlamaCppBackend:
base = _VRAM_FLOOR_RESERVE_MIB if auto_fit else _LLAMA_FIT_TARGET_DEFAULT_MIB
return max(_VRAM_FLOOR_RESERVE_MIB, base + fit_target_delta_mib, floor_mib)
+ @staticmethod
+ def _fit_target_margins_mib(
+ *,
+ auto_fit: bool,
+ fit_target_delta_by_idx: Optional[Mapping[int, float]],
+ supports_fit_target: bool = True,
+ ) -> Optional[dict[int, float]]:
+ """``_fit_target_margin_mib`` for each card, keyed by physical GPU id.
+
+ None when there is no per-card delta, so callers keep the single margin. One
+ formula for the priced margins and the emitted list, like the scalar.
+ """
+ if not fit_target_delta_by_idx or not supports_fit_target:
+ return None
+ return {
+ int(idx): LlamaCppBackend._fit_target_margin_mib(
+ auto_fit = auto_fit, fit_target_delta_mib = delta
+ )
+ for idx, delta in fit_target_delta_by_idx.items()
+ }
+
+ @staticmethod
+ def _fit_target_per_device_ok(
+ *,
+ is_vulkan_backend: bool,
+ extra_args: Optional[Iterable[str]],
+ env: Optional[Mapping[str, str]],
+ ) -> bool:
+ """Whether a per-card ``--fit-target`` list can be lined up with llama.cpp's devices.
+
+ The list is positional over the child's device enumeration. CUDA under
+ PCI_BUS_ID enumerates the visible physical ids in mask order (ascending when
+ unmasked), which ties a position to an nvidia-smi index. Vulkan and HIP order
+ devices their own way, a UUID or MIG mask cannot be mapped back to indices,
+ and --device / --rpc / a pass-through --fit-target change or own the list, so
+ each of those keeps the single broadcast margin.
+ """
+ if is_vulkan_backend or sys.platform == "darwin":
+ return False
+ if LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not True:
+ return False
+ if (os.environ.get("CUDA_DEVICE_ORDER") or "").strip().upper() != "PCI_BUS_ID":
+ return False
+ if LlamaCppBackend._visibility_mask_is_unmappable():
+ return False
+ # A resolvable uuid mask still reaches the child as uuids, which the spawn-time
+ # spelling cannot order, so the pricing keeps the broadcast margin with it.
+ mask = os.environ.get("CUDA_VISIBLE_DEVICES")
+ if mask is not None and not all(t.strip().isdigit() for t in mask.split(",") if t.strip()):
+ return False
+ if LlamaCppBackend._SMI_MIG_INDICES:
+ return False
+ if LlamaCppBackend._host_torch_is_rocm():
+ return False
+ env = env or {}
+ if fit_target_margin_in(extra_args or (), env) is not None:
+ return False
+ if (_extra_args_device(extra_args, {"--device", "-dev", "--rpc"}) or "").strip():
+ return False
+ # A companion device flag can make the pin add --device after planning, which
+ # the spawn-time spelling then declines; decline here so pricing agrees.
+ if any(_extra_args_device(extra_args, flags) for flags in _COMPANION_DEVICE_FLAG_GROUPS):
+ return False
+ if any(str(env.get(k, "")).strip() for k in ("LLAMA_ARG_DEVICE", "LLAMA_ARG_RPC")):
+ return False
+ return True
+
+ @staticmethod
+ def _per_device_fit_target_argv(
+ cmd: list[str], env: Mapping[str, str], margins_by_idx: Optional[Mapping[int, float]]
+ ) -> list[str]:
+ """``cmd`` with Studio's ``--fit-target`` spelled per device for the child's mask.
+
+ llama.cpp broadcasts a single value, so on mixed cards the margin sized for
+ the largest card can exceed a smaller card's whole memory and leave it
+ unused. Built from the FINAL child env at spawn time, since a retry can
+ re-pin the mask. Returns ``cmd`` unchanged when the order cannot be proven
+ (a device without a margin, a non-numeric mask, another device order) or
+ when every card would get the same value, which a single value already says.
+ """
+ if not margins_by_idx:
+ return cmd
+ at = None
+ for k in range(len(cmd) - 2, -1, -1):
+ if cmd[k] == "--fit-target":
+ at = k
+ break
+ if at is None:
+ return cmd
+ if (str(env.get("CUDA_DEVICE_ORDER") or "")).strip().upper() != "PCI_BUS_ID":
+ return cmd
+ # --device narrows the model's devices and --rpc puts remote ones first, both
+ # re-indexing the list; a companion widen can add --device after planning.
+ if (_extra_args_device(cmd, {"--device", "-dev", "--rpc"}) or "").strip() or any(
+ str(env.get(k, "")).strip() for k in ("LLAMA_ARG_DEVICE", "LLAMA_ARG_RPC")
+ ):
+ return cmd
+ mask = env.get("CUDA_VISIBLE_DEVICES")
+ if mask is None:
+ order = sorted(int(i) for i in margins_by_idx)
+ else:
+ try:
+ order = [int(tok) for tok in str(mask).split(",") if tok.strip()]
+ except ValueError:
+ return cmd
+ if not order or any(i < 0 or i not in margins_by_idx for i in order):
+ return cmd
+ values = [str(int(margins_by_idx[i])) for i in order]
+ spelled = values[0] if len(set(values)) == 1 else ",".join(values)
+ if cmd[at + 1] == spelled:
+ return cmd
+ out = list(cmd)
+ out[at + 1] = spelled
+ logger.info(
+ "Per-device --fit-target %s for GPUs %s: each card keeps its own budget margin.",
+ spelled,
+ order,
+ )
+ return out
+
@staticmethod
def _ctx_integrity_flags(
n_parallel: int,
diff --git a/studio/backend/tests/test_load_mode_fit_matrix.py b/studio/backend/tests/test_load_mode_fit_matrix.py
index f14358826c6..084550cb73e 100644
--- a/studio/backend/tests/test_load_mode_fit_matrix.py
+++ b/studio/backend/tests/test_load_mode_fit_matrix.py
@@ -393,7 +393,11 @@ def test_the_launch_charges_the_fitters_margin_only_when_fit_stays_on():
assert "fit_target_delta_mib=_fit_target_delta_mibifuse_fitelse0.0" in call
assert "supports_fit_target=use_fitand" in call
assert "fit_target_margin_in(_fit_extras,_fit_env)or0.0" in call
- assert "fit_margin_mib=_fit_margin_mib" in call
+ # Studio's per-card margins replace it only while Studio owns the margin: the
+ # fitter really runs on this launch's --fit, with no pass-through --fit-target.
+ assert "if_fitter_runsanduse_fit" in call
+ assert "fit_target_margin_in(_fit_extras,_fit_env)isNoneelseNone" in call
+ assert "fit_margin_mib=_fit_margins_by_idxor_fit_margin_mib" in call
def test_the_effective_fitter_state_reads_the_launchs_own_fit_flag():
diff --git a/studio/backend/tests/test_managed_engines.py b/studio/backend/tests/test_managed_engines.py
index 0ede9bbd0d1..080a9ebb434 100644
--- a/studio/backend/tests/test_managed_engines.py
+++ b/studio/backend/tests/test_managed_engines.py
@@ -1149,8 +1149,14 @@ def test_memory_budget_uses_selected_gpu_and_reserves_headroom(monkeypatch):
assert commands[0][commands[0].index("--id") + 1] == "1"
# vLLM TorchAO weights overrun the reservation in sampler warmup, so keep 6 GiB more free.
assert engine_adapters.gpu_memory_fraction([1], 512 + 6144) == 0.395
- monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.5)
- assert engine_adapters.gpu_memory_fraction([1]) == 0.5
+ monkeypatch.setattr(
+ engine_adapters.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout = "24576, 24000")
+ )
+ monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.85)
+ assert engine_adapters.gpu_memory_fraction([1]) == 0.85
+ # A GGUF-only low budget must not shrink an engine that cannot offload to the CPU.
+ monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.3)
+ assert engine_adapters.gpu_memory_fraction([1]) == engine_adapters.MANAGED_ENGINE_BUDGET_FLOOR
monkeypatch.setattr(
engine_adapters.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout = "24576, 256")
)
diff --git a/studio/backend/tests/test_tensor_parallel.py b/studio/backend/tests/test_tensor_parallel.py
index f1f7c5ba865..6b1c9c7b699 100644
--- a/studio/backend/tests/test_tensor_parallel.py
+++ b/studio/backend/tests/test_tensor_parallel.py
@@ -1107,7 +1107,8 @@ def test_fit_context_follows_the_user_vram_budget(monkeypatch):
monkeypatch.setattr(vb, "_cached_setting", lambda _key: 1.0)
fit_max = _fit()
- monkeypatch.setattr(vb, "_cached_setting", lambda _key: vb.VRAM_FRACTION_MIN)
+ # 0.80, not VRAM_FRACTION_MIN: at 0.10 the 8 GiB weights alone exceed the budget, which hands the load to the offload path.
+ monkeypatch.setattr(vb, "_cached_setting", lambda _key: 0.80)
fit_min = _fit()
assert fit_max > fit_default, "claiming the whole card must buy context back"
diff --git a/studio/backend/tests/test_vram_budget_settings.py b/studio/backend/tests/test_vram_budget_settings.py
index ce49358d3f3..1b00928ffcb 100644
--- a/studio/backend/tests/test_vram_budget_settings.py
+++ b/studio/backend/tests/test_vram_budget_settings.py
@@ -36,16 +36,19 @@ def _isolate(monkeypatch):
class TestCoerceFraction:
- @pytest.mark.parametrize("value", [0.80, 0.9, 0.97, 1.0, "0.85", "1.00"])
+ @pytest.mark.parametrize(
+ "value", [0.10, 0.30, 0.50, 0.80, 0.9, 0.97, 1.0, "0.25", "0.85", "1.00"]
+ )
def test_accepts_in_range(self, value):
assert vb.coerce_fraction(value) == pytest.approx(float(value))
@pytest.mark.parametrize(
"value",
[
- 0.79,
- 1.01,
0.0,
+ 0.05,
+ 0.099,
+ 1.01,
-1.0,
2.0,
"nan",
@@ -150,7 +153,7 @@ class TestState:
class TestWrite:
def test_rejects_out_of_range(self, monkeypatch):
monkeypatch.setattr(vb, "_invalidate", lambda _key: None)
- for bad in (0.5, 1.5, float("nan"), "abc"):
+ for bad in (0.0, 0.05, -0.25, 1.5, float("nan"), "abc"):
with pytest.raises(ValueError):
vb.set_vram_budget_fraction(bad)
@@ -589,7 +592,7 @@ class TestFitTarget:
assert "--fit-target" not in flags
def test_the_move_is_measured_from_the_card_that_makes_it_safe(self):
- # --fit-target is broadcast as one value: llama.cpp device order is not ours to assume.
+ # The broadcast value, kept wherever llama.cpp's device order is not provable; TestPerDeviceFitTarget covers the per-device list.
compact = _shared_setup_1()
assert "if_vram_frac!=_CTX_FIT_VRAM_FRACTIONandgpus:" in compact
assert "_fit_target_delta_mib=(_CTX_FIT_VRAM_FRACTION-_vram_frac)*_scale" in compact
@@ -598,6 +601,249 @@ class TestFitTarget:
assert "total_by_idx.get(_idx)or_freefor_idx,_freeingpus" in compact
+class TestPerDeviceFitTarget:
+ """On mixed cards a lowered budget's margin is spelled per device.
+
+ One broadcast margin is sized by the largest card when lowering, so on a 48 GB +
+ 24 GB host at 30% it asked the 24 GB card to keep about 33 GiB free: more than
+ the card has, so llama.cpp's fitter put nothing on it.
+ """
+
+ _ENV = {"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}
+ # 48 GB and 24 GB cards at a 30% budget, legacy path (1024 MiB base).
+ _MARGINS = {0: 1024 + 0.67 * 49140, 1: 1024 + 0.67 * 24576}
+ _CMD = ["llama-server", "--fit", "on", "--fit-target", "33947", "--jinja"]
+
+ def _spell(
+ self,
+ cmd = None,
+ env = None,
+ margins = None,
+ ):
+ import core.inference.llama_cpp as lc
+ return lc.LlamaCppBackend._per_device_fit_target_argv(
+ list(cmd or self._CMD),
+ self._ENV if env is None else env,
+ self._MARGINS if margins is None else margins,
+ )
+
+ def test_each_card_keeps_its_own_margin(self):
+ out = self._spell()
+ assert out[out.index("--fit-target") + 1] == "33947,17489"
+ # Only the value moves.
+ assert [t for t in out if t != "33947,17489"] == [t for t in self._CMD if t != "33947"]
+
+ def test_the_list_follows_the_child_mask_order(self):
+ # CUDA enumerates a numeric mask in the order it lists the ids.
+ out = self._spell(env = {**self._ENV, "CUDA_VISIBLE_DEVICES": "1,0"})
+ assert out[out.index("--fit-target") + 1] == "17489,33947"
+
+ def test_a_pinned_single_card_gets_its_own_value(self):
+ out = self._spell(env = {**self._ENV, "CUDA_VISIBLE_DEVICES": "1"})
+ assert out[out.index("--fit-target") + 1] == "17489"
+
+ def test_identical_cards_keep_the_single_value(self):
+ # The same meaning, and an unchanged command on homogeneous hosts.
+ margins = {0: 9378.0, 1: 9378.0}
+ assert self._spell(cmd = ["--fit-target", "9378"], margins = margins) == [
+ "--fit-target",
+ "9378",
+ ]
+
+ @pytest.mark.parametrize(
+ "env",
+ [
+ {"CUDA_DEVICE_ORDER": "FASTEST_FIRST"},
+ {},
+ {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": "GPU-1234"},
+ {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": "0,2"},
+ {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": "-1"},
+ {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "LLAMA_ARG_DEVICE": "CUDA1"},
+ {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "LLAMA_ARG_RPC": "host:50052"},
+ ],
+ ids = [
+ "fastest-first",
+ "no-order",
+ "uuid-mask",
+ "unknown-id",
+ "cpu",
+ "env-device",
+ "env-rpc",
+ ],
+ )
+ def test_an_unprovable_order_keeps_the_broadcast_value(self, env):
+ assert self._spell(env = env) == self._CMD
+
+ @pytest.mark.parametrize("flag", ["--device", "-dev", "--rpc"])
+ def test_device_flags_in_the_command_keep_the_broadcast_value(self, flag):
+ cmd = [*self._CMD, flag, "CUDA0"]
+ assert self._spell(cmd = cmd) == cmd
+
+ def test_nothing_to_spell_without_margins_or_the_flag(self):
+ assert self._spell(margins = {}) == self._CMD
+ cmd = ["llama-server", "--fit", "on"]
+ assert self._spell(cmd = cmd) == cmd
+
+ def test_a_raised_budget_gives_each_card_its_own_smaller_margin(self):
+ # Above the default the delta is negative: at 98% the larger card may pack
+ # closer than the smaller one, where broadcast held both to the smaller's.
+ import core.inference.llama_cpp as lc
+
+ margins = lc.LlamaCppBackend._fit_target_margins_mib(
+ auto_fit = False,
+ fit_target_delta_by_idx = {0: -0.01 * 49140, 1: -0.01 * 24576},
+ )
+ out = self._spell(margins = margins)
+ assert out[out.index("--fit-target") + 1] == "532,778"
+ # At 100% both reach the 512 MiB floor, one value again.
+ floored = lc.LlamaCppBackend._fit_target_margins_mib(
+ auto_fit = False,
+ fit_target_delta_by_idx = {0: -0.03 * 49140, 1: -0.03 * 24576},
+ )
+ assert floored == {0: 512.0, 1: 512.0}
+ out = self._spell(margins = floored)
+ assert out[out.index("--fit-target") + 1] == "512"
+
+ def test_per_card_margins_use_the_scalar_formula(self):
+ import core.inference.llama_cpp as lc
+
+ margins = lc.LlamaCppBackend._fit_target_margins_mib(
+ auto_fit = False, fit_target_delta_by_idx = {0: 32924.0, 1: -4096.0}
+ )
+ for idx, delta in ((0, 32924.0), (1, -4096.0)):
+ assert margins[idx] == lc.LlamaCppBackend._fit_target_margin_mib(
+ auto_fit = False, fit_target_delta_mib = delta
+ )
+ assert (
+ lc.LlamaCppBackend._fit_target_margins_mib(auto_fit = False, fit_target_delta_by_idx = None)
+ is None
+ )
+ assert (
+ lc.LlamaCppBackend._fit_target_margins_mib(
+ auto_fit = False,
+ fit_target_delta_by_idx = {0: 1.0},
+ supports_fit_target = False,
+ )
+ is None
+ )
+
+ def test_the_paging_check_credits_each_card_its_own_margin(self, monkeypatch):
+ # The load-mode fit must spend what the child is told to keep, card by card.
+ import core.inference.llama_cpp as lc
+
+ backend = lc.LlamaCppBackend()
+ rows = [(0, 47000, 49140), (1, 24000, 24576)]
+ gib = 1024**3
+ # 20 GiB of weights and 4 GiB of RAM. Broadcast credits 12.7 GiB of VRAM (a
+ # 7.3 GiB spill); per card adds the 24 GB card's 6.4 GiB (a 0.9 GiB spill).
+ kwargs = dict(avail_mib = 4 * 1024, headroom_mib = 0)
+ broadcast = backend._fits_without_paging(20 * gib, rows, vram_margin_mib = 33947.0, **kwargs)
+ per_card = backend._fits_without_paging(
+ 20 * gib, rows, vram_margin_mib = {0: 33947.0, 1: 17489.0}, **kwargs
+ )
+ assert broadcast is False
+ assert per_card is True
+
+ def _ok(
+ self,
+ monkeypatch,
+ *,
+ extra_args = None,
+ env = None,
+ **attrs,
+ ):
+ import core.inference.llama_cpp as lc
+
+ B = lc.LlamaCppBackend
+ # The eligibility check declines on macOS, so pin the host OS the case means.
+ monkeypatch.setattr(lc.sys, "platform", attrs.get("platform", "linux"))
+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", attrs.get("pci", True))
+ monkeypatch.setattr(B, "_SMI_MIG_INDICES", attrs.get("mig", frozenset()))
+ monkeypatch.setattr(
+ B,
+ "_visibility_mask_is_unmappable",
+ staticmethod(lambda: attrs.get("unmappable", False)),
+ )
+ monkeypatch.setattr(
+ B, "_host_torch_is_rocm", staticmethod(lambda: attrs.get("rocm", False))
+ )
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", attrs.get("order", "PCI_BUS_ID"))
+ if attrs.get("mask") is None:
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
+ else:
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", attrs["mask"])
+ return B._fit_target_per_device_ok(
+ is_vulkan_backend = attrs.get("vulkan", False),
+ extra_args = extra_args,
+ env = env or {},
+ )
+
+ def test_eligible_on_a_plain_cuda_host(self, monkeypatch):
+ assert self._ok(monkeypatch)
+ assert self._ok(monkeypatch, mask = "1,0")
+
+ def test_a_resolvable_uuid_mask_is_ineligible(self, monkeypatch):
+ # The child inherits the uuids, which _per_device_fit_target_argv cannot order.
+ assert not self._ok(monkeypatch, mask = "GPU-aaaa,GPU-bbbb")
+
+ @pytest.mark.parametrize(
+ "attrs",
+ [
+ {"vulkan": True},
+ {"rocm": True},
+ {"pci": None},
+ {"pci": False},
+ {"order": "FASTEST_FIRST"},
+ {"unmappable": True},
+ {"mig": frozenset({0})},
+ ],
+ ids = [
+ "vulkan",
+ "rocm",
+ "ids-unknown",
+ "torch-ordinals",
+ "fastest-first",
+ "uuid-mask",
+ "mig",
+ ],
+ )
+ def test_an_unprovable_device_order_is_ineligible(self, monkeypatch, attrs):
+ assert not self._ok(monkeypatch, **attrs)
+
+ @pytest.mark.parametrize(
+ "extra_args, env",
+ [
+ (["--fit-target", "2048"], None),
+ (None, {"LLAMA_ARG_FIT_TARGET": "2048"}),
+ (["--device", "CUDA1"], None),
+ (["--rpc", "host:50052"], None),
+ (None, {"LLAMA_ARG_DEVICE": "CUDA1"}),
+ (["--spec-draft-device", "CUDA1"], None),
+ (["--mmproj-device", "CUDA1"], None),
+ ],
+ ids = [
+ "argv-fit-target",
+ "env-fit-target",
+ "device",
+ "rpc",
+ "env-device",
+ "draft-device",
+ "mmproj-device",
+ ],
+ )
+ def test_pass_through_placement_is_ineligible(self, monkeypatch, extra_args, env):
+ assert not self._ok(monkeypatch, extra_args = extra_args, env = env)
+
+ def test_the_load_path_wires_the_per_card_margins(self):
+ compact = _shared_setup_1()
+ # Planned over every probed row, which an unpinned child sees.
+ assert "for_idx,_free,_totalin_gpu_mem" in compact
+ # Spelled at both spawn sites, against the final env.
+ assert compact.count("_per_device_fit_target_argv(") == 2
+ # Priced with the same margins it emits.
+ assert "fit_margin_mib=_fit_margins_by_idxor_fit_margin_mib" in compact
+
+
class TestManualAutoIsPriced:
"""Manual + Auto empties ``gpus``, but --fit-target still spends the budget."""
diff --git a/studio/backend/utils/vram_budget_settings.py b/studio/backend/utils/vram_budget_settings.py
index b4f5ce1a674..721cfc65039 100644
--- a/studio/backend/utils/vram_budget_settings.py
+++ b/studio/backend/utils/vram_budget_settings.py
@@ -24,7 +24,8 @@ VRAM_BUDGET_SETTING_KEY = "vram_budget_fraction"
VRAM_FRACTION_ENV_VAR = "UNSLOTH_VRAM_FRACTION"
# Mirrored in per-model-config.ts as percent for the slider, a pair test_vram_budget_settings.py pins together. The default is the historical _CTX_FIT_VRAM_FRACTION / _GPU_PIN_VRAM_FRACTION.
-VRAM_FRACTION_MIN = 0.80
+# 0.10, not 0: at 0 every GGUF load runs on the CPU. vLLM / SGLang cannot offload and keep engine_adapters.MANAGED_ENGINE_BUDGET_FLOOR.
+VRAM_FRACTION_MIN = 0.10
VRAM_FRACTION_MAX = 1.00
VRAM_FRACTION_DEFAULT = 0.97
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index 6e6c0a5658c..b5438115cfc 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -133,7 +133,7 @@ export const N_PARALLEL_MAX = 64;
// Mirrors vram_budget_settings.py VRAM_FRACTION_MIN/MAX/DEFAULT as whole percent (the slider is integer).
// Server-wide, so these only bound the control; the value lives in /api/settings/vram-budget.
-export const VRAM_BUDGET_PERCENT_MIN = 80;
+export const VRAM_BUDGET_PERCENT_MIN = 10;
export const VRAM_BUDGET_PERCENT_MAX = 100;
export const VRAM_BUDGET_PERCENT_DEFAULT = 97;
// Tenths: a whole percent is ~245 MiB of context on a 24 GB card. Mirrors
diff --git a/studio/frontend/tests/vram-budget.test.ts b/studio/frontend/tests/vram-budget.test.ts
index 01305b76124..81de6d279cb 100644
--- a/studio/frontend/tests/vram-budget.test.ts
+++ b/studio/frontend/tests/vram-budget.test.ts
@@ -56,8 +56,8 @@ test("the default fraction is exactly 0.97, not a float-drifted neighbour", () =
});
test("the bounds mirror the backend range", () => {
- // vram_budget_settings.py: VRAM_FRACTION_MIN 0.80, MAX 1.00, DEFAULT 0.97.
- assert.equal(vramPercentToFraction(VRAM_BUDGET_PERCENT_MIN), 0.8);
+ // vram_budget_settings.py: VRAM_FRACTION_MIN 0.10, MAX 1.00, DEFAULT 0.97.
+ assert.equal(vramPercentToFraction(VRAM_BUDGET_PERCENT_MIN), 0.1);
assert.equal(vramPercentToFraction(VRAM_BUDGET_PERCENT_MAX), 1);
assert.equal(VRAM_BUDGET_PERCENT_DEFAULT, 97);
});