Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: lower the VRAM budget floor to 10% for multi-GPU spreading (#13427)
Shooter3k80d23aaa0c91
diff --git a/studio/backend/core/inference/engine_adapters.py b/studio/backend/core/inference/engine_adapters.pyindex 7237b110e99..e7c00cf8c36 100644--- a/studio/backend/core/inference/engine_adapters.py+++ b/studio/backend/core/inference/engine_adapters.py@@ -398,6 +398,9 @@ def memory_reserve_mib(engine: str, options: dict | None) -> int:RESERVE_SHARE = 0.06+MANAGED_ENGINE_BUDGET_FLOOR = 0.80++def gpu_memory_fraction(gpu_ids: list[int],reserve_mib: int = 512,@@ -416,7 +419,9 @@ def gpu_memory_fraction(rows = (measure or _nvidia_memory_rows)(gpu_ids)if len(rows) != len(gpu_ids):raise ValueError("Could not measure every selected GPU")- fraction = get_vram_budget_fraction()+ # vLLM / SGLang cannot offload to the CPU, so a GGUF-only low budget must not shrink them;+ # the per-load engine_gpu_memory_utilization still caps lower.+ fraction = max(get_vram_budget_fraction(), MANAGED_ENGINE_BUDGET_FLOOR)for total, free in rows:reserve = max(reserve_mib, reserve_share * total)if total <= 0 or free <= reserve or free > total:diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 0f07aceb310..dad0ef61894 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -6728,6 +6728,16 @@ def _extra_args_draft_offloaded_to_cpu(return False+def _margin_mib_for(margin: Union[float, Mapping[int, float], None], idx: int) -> float:+ """The ``--fit-target`` margin one card keeps: a single broadcast value, or that+ card's entry in a per-device mapping (the largest entry for a card it omits)."""+ if isinstance(margin, Mapping):+ if not margin:+ return 0.0+ return max(0.0, float(margin.get(int(idx), max(margin.values()))))+ return max(0.0, float(margin or 0.0))++def _extra_args_device(extra_args: Optional[Iterable[str]], flags: Collection[str]) -> Optional[str]:@@ -16848,7 +16858,7 @@ class LlamaCppBackend:gpu_indices: Optional[Iterable[int]] = None,shared_gpu_ids: Iterable[int] = (),host_only_bytes: int = 0,- vram_margin_mib: float = 0.0,+ vram_margin_mib: Union[float, Mapping[int, float]] = 0.0,avail_mib: Optional[int] = None,headroom_mib: int = _HOST_RAM_HEADROOM_MIB,) -> Optional[bool]:@@ -16881,16 +16891,16 @@ class LlamaCppBackend:per device for --fit"; common/arg.cpp, common/common.h). Under ``--fit on``the child spills weights to host RAM rather than dip into it, so those bytesare not credit this launch may spend. Zero on the proved path, which emits- ``-ngl -1 --fit off`` and runs no fitter at all.+ ``-ngl -1 --fit off`` and runs no fitter at all. A mapping gives each card the+ margin of Studio's per-device list, keyed by physical GPU id."""if not footprint_bytes or footprint_bytes <= 0:return Nonehost_only = max(0, int(host_only_bytes or 0))shared = set(shared_gpu_ids or ())pinned = None if gpu_indices is None else {int(idx) for idx in gpu_indices}- margin_mib = max(0.0, float(vram_margin_mib or 0.0))free_vram_mib = sum(- max(0, row[1] - margin_mib)+ max(0, row[1] - _margin_mib_for(vram_margin_mib, row[0]))for row in (detected_gpus or ())if row[0] not in shared and (pinned is None or row[0] in pinned))@@ -16923,7 +16933,7 @@ class LlamaCppBackend:gpus: Optional[Iterable[tuple]] = None,gpu_indices: Optional[Iterable[int]] = None,shared_gpu_ids: Iterable[int] = (),- fit_margin_mib: float = 0.0,+ fit_margin_mib: Union[float, Mapping[int, float]] = 0.0,is_vulkan_backend: bool = False,avail_mib: Optional[int] = None,extra_args: Optional[Iterable[str]] = None,@@ -25691,6 +25701,10 @@ class LlamaCppBackend:# devices are known. Signed: lowering holds more back, raising packs# closer to the card. Zero at the default._fit_target_delta_mib = 0.0+ # The same delta per physical GPU id, when the per-device list applies.+ _fit_target_delta_by_idx: Optional[dict[int, float]] = None+ # Studio's emitted margins per card, spelled into --fit-target at spawn.+ _fit_target_margins_by_idx: Optional[dict[int, float]] = None# Set only if that margin actually reached the command line, which is# what makes the child budget-priced in a mode the planner skipped._fit_target_priced = False@@ -27133,14 +27147,16 @@ class LlamaCppBackend:# so the slider never reached the path that runs when the fit is# tightest. Hand it the difference, both directions, measured from# the default so an untouched slider leaves the command alone.- # One broadcast value: --fit-target does take a per-device list, but- # in llama.cpp's enumeration order, which the visible-device pin and- # the ROCr/Vulkan ordinal quirks make not ours to assume, and a- # misaligned list hands a card the wrong margin. So the direction- # picks the card: lowering scales by the largest, so no device ends- # up under the margin it asked for, raising by the smallest, so none- # is taken below its own. Mixed cards gain a little less, and hold- # back a little more, than a per-device list would.+ # --fit-target takes a per-device list in llama.cpp's enumeration+ # order. Where CUDA's PCI order makes that order provable, each card+ # gets its own margin, spelled at spawn against the final mask. A+ # single broadcast value would otherwise have to be sized by the+ # largest card when lowering, which on mixed cards can exceed a+ # smaller card's whole memory and leave it unused. Elsewhere (Vulkan,+ # ROCm, UUID/MIG masks, --device/--rpc) the direction picks the card:+ # lowering scales by the largest, so no device ends up under the+ # margin it asked for, raising by the smallest, so none is taken+ # below its own.if _vram_frac != _CTX_FIT_VRAM_FRACTION and gpus:# Free stands in for an unreported total, the same scale# _vram_usable_mib falls back to: MIG/vGPU and the two-column@@ -27152,6 +27168,19 @@ class LlamaCppBackend:max(_scales) if _vram_frac < _CTX_FIT_VRAM_FRACTION else min(_scales))_fit_target_delta_mib = (_CTX_FIT_VRAM_FRACTION - _vram_frac) * _scale+ # Where llama.cpp's device order is provable, each card gets+ # its own margin instead (_per_device_fit_target_argv). Every+ # probed row, not just `gpus`: an unpinned child sees them all.+ if self._fit_target_per_device_ok(+ is_vulkan_backend = is_vulkan_backend,+ extra_args = extra_args,+ env = os.environ,+ ):+ _fit_target_delta_by_idx = {+ int(_idx): (_CTX_FIT_VRAM_FRACTION - _vram_frac)+ * (_total if _total and _total > 0 else _free)+ for _idx, _free, _total in _gpu_mem+ }_sysmem_fallback = self._sysmem_fallback_risk(binary)@@ -29643,6 +29672,8 @@ class LlamaCppBackend:)# A per-device list reads as its largest entry (the larger shortfall)._margin_mib = fit_target_margin_in(_child_extras or (), None)+ # Studio's own per-card margins, when it spells the list itself.+ _margins_by_idx: Optional[dict[int, float]] = Noneif _margin_mib is None:_emits = _supports_fit_target and (_auto_fit or _managed_mib != _LLAMA_FIT_TARGET_DEFAULT_MIB@@ -29650,6 +29681,12 @@ class LlamaCppBackend:_env_mib = None if _emits else fit_target_margin_in((), _child_env)# An inherited 0 is a valid margin, not "unset"._margin_mib = _managed_mib if _env_mib is None else _env_mib+ if _emits:+ _margins_by_idx = self._fit_target_margins_mib(+ auto_fit = _auto_fit,+ fit_target_delta_by_idx = _fit_target_delta_by_idx,+ supports_fit_target = _supports_fit_target,+ )# A lazily read table stays mmap'd on the host, never on the GPU._lazy_host = self._lazy_read_host_bytes(model_path,@@ -29680,7 +29717,10 @@ class LlamaCppBackend:)+ _cc_bytes(effective_ctx, len(_on)))- _free = sum(max(0.0, free - _margin_mib) for _idx, free in _on)+ _free = sum(+ max(0.0, free - _margin_mib_for(_margins_by_idx or _margin_mib, _idx))+ for _idx, free in _on+ )return _need - _free * 1024 * 1024def _moe_resident_share_now() -> Optional[float]:@@ -30039,6 +30079,20 @@ class LlamaCppBackend:if _fitter_runselse 0.0)+ # Studio's per-card list replaces the single value only when Studio+ # owns the margin: a pass-through one wins outright (above).+ _fit_margins_by_idx = (+ self._fit_target_margins_mib(+ auto_fit = (gpu_memory_mode == "manual" and gpu_layers < 0),+ fit_target_delta_by_idx = _fit_target_delta_by_idx,+ supports_fit_target = bool(server_caps.get("supports_fit_target")),+ )+ if _fitter_runs+ and use_fit+ and not _metal_fit_target_floor_mib+ and fit_target_margin_in(_fit_extras, _fit_env) is None+ else None+ )# A projector Unsloth never resolved but the child loads anyway:# LLAMA_ARG_MMPROJ / _URL are read before argv and only the vision# switch and the paravirtual pin clear them, while model_size carries@@ -30171,7 +30225,7 @@ class LlamaCppBackend:# A running fitter keeps --fit-target free per device, so that# margin is not VRAM this fit may spend. Zero only when the# fitter is provably off; see _fitter_runs above.- fit_margin_mib = _fit_margin_mib,+ fit_margin_mib = _fit_margins_by_idx or _fit_margin_mib,is_vulkan_backend = is_vulkan_backend,# Never stripped, and appended last, so they beat this# launch's own -ngl.@@ -30877,6 +30931,16 @@ class LlamaCppBackend:# the capability and a budget at the default all leave the child# unpriced, and each is decided inside that call._fit_target_priced = "--fit-target" in _integrity_flags+ # Spelled per device at spawn, against the child's final mask.+ _fit_target_margins_by_idx = (+ self._fit_target_margins_mib(+ auto_fit = auto_fit,+ fit_target_delta_by_idx = _fit_target_delta_by_idx,+ supports_fit_target = bool(server_caps.get("supports_fit_target")),+ )+ if _fit_target_priced and use_fit and not _metal_fit_target_floor_mib+ else None+ )if _metal_fit_target_floor_mib:# Rebuilt after the price is read: the floor is the machine's# margin, not the budget slider's.@@ -33083,6 +33147,9 @@ class LlamaCppBackend:# Best-effort; never block the load on logging.logger.warning(f"Could not open llama-server log file: {e}")self._llama_log_path = None+ run_cmd = self._per_device_fit_target_argv(+ run_cmd, env, _fit_target_margins_by_idx+ )_last_spawn_cmd = list(run_cmd)# Read off the argv actually spawned rather than the intent, so# every emitter is covered by construction.@@ -34803,6 +34870,9 @@ class LlamaCppBackend:)self._mmproj_fallback_reason = "projector_startup_failure"cmd = self._strip_mmproj_args(_vision_gpu_cmd)+ cmd = self._per_device_fit_target_argv(+ cmd, env, _fit_target_margins_by_idx+ )_unified_withdraw_if_unneeded(cmd, why = "The text-only retry")# This retry bypasses _spawn_and_wait, so refresh the# launched-argv snapshot itself -- the zero-offload@@ -38457,6 +38527,126 @@ class LlamaCppBackend:base = _VRAM_FLOOR_RESERVE_MIB if auto_fit else _LLAMA_FIT_TARGET_DEFAULT_MIBreturn max(_VRAM_FLOOR_RESERVE_MIB, base + fit_target_delta_mib, floor_mib)+ @staticmethod+ def _fit_target_margins_mib(+ *,+ auto_fit: bool,+ fit_target_delta_by_idx: Optional[Mapping[int, float]],+ supports_fit_target: bool = True,+ ) -> Optional[dict[int, float]]:+ """``_fit_target_margin_mib`` for each card, keyed by physical GPU id.++ None when there is no per-card delta, so callers keep the single margin. One+ formula for the priced margins and the emitted list, like the scalar.+ """+ if not fit_target_delta_by_idx or not supports_fit_target:+ return None+ return {+ int(idx): LlamaCppBackend._fit_target_margin_mib(+ auto_fit = auto_fit, fit_target_delta_mib = delta+ )+ for idx, delta in fit_target_delta_by_idx.items()+ }++ @staticmethod+ def _fit_target_per_device_ok(+ *,+ is_vulkan_backend: bool,+ extra_args: Optional[Iterable[str]],+ env: Optional[Mapping[str, str]],+ ) -> bool:+ """Whether a per-card ``--fit-target`` list can be lined up with llama.cpp's devices.++ The list is positional over the child's device enumeration. CUDA under+ PCI_BUS_ID enumerates the visible physical ids in mask order (ascending when+ unmasked), which ties a position to an nvidia-smi index. Vulkan and HIP order+ devices their own way, a UUID or MIG mask cannot be mapped back to indices,+ and --device / --rpc / a pass-through --fit-target change or own the list, so+ each of those keeps the single broadcast margin.+ """+ if is_vulkan_backend or sys.platform == "darwin":+ return False+ if LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES is not True:+ return False+ if (os.environ.get("CUDA_DEVICE_ORDER") or "").strip().upper() != "PCI_BUS_ID":+ return False+ if LlamaCppBackend._visibility_mask_is_unmappable():+ return False+ # A resolvable uuid mask still reaches the child as uuids, which the spawn-time+ # spelling cannot order, so the pricing keeps the broadcast margin with it.+ mask = os.environ.get("CUDA_VISIBLE_DEVICES")+ if mask is not None and not all(t.strip().isdigit() for t in mask.split(",") if t.strip()):+ return False+ if LlamaCppBackend._SMI_MIG_INDICES:+ return False+ if LlamaCppBackend._host_torch_is_rocm():+ return False+ env = env or {}+ if fit_target_margin_in(extra_args or (), env) is not None:+ return False+ if (_extra_args_device(extra_args, {"--device", "-dev", "--rpc"}) or "").strip():+ return False+ # A companion device flag can make the pin add --device after planning, which+ # the spawn-time spelling then declines; decline here so pricing agrees.+ if any(_extra_args_device(extra_args, flags) for flags in _COMPANION_DEVICE_FLAG_GROUPS):+ return False+ if any(str(env.get(k, "")).strip() for k in ("LLAMA_ARG_DEVICE", "LLAMA_ARG_RPC")):+ return False+ return True++ @staticmethod+ def _per_device_fit_target_argv(+ cmd: list[str], env: Mapping[str, str], margins_by_idx: Optional[Mapping[int, float]]+ ) -> list[str]:+ """``cmd`` with Studio's ``--fit-target`` spelled per device for the child's mask.++ llama.cpp broadcasts a single value, so on mixed cards the margin sized for+ the largest card can exceed a smaller card's whole memory and leave it+ unused. Built from the FINAL child env at spawn time, since a retry can+ re-pin the mask. Returns ``cmd`` unchanged when the order cannot be proven+ (a device without a margin, a non-numeric mask, another device order) or+ when every card would get the same value, which a single value already says.+ """+ if not margins_by_idx:+ return cmd+ at = None+ for k in range(len(cmd) - 2, -1, -1):+ if cmd[k] == "--fit-target":+ at = k+ break+ if at is None:+ return cmd+ if (str(env.get("CUDA_DEVICE_ORDER") or "")).strip().upper() != "PCI_BUS_ID":+ return cmd+ # --device narrows the model's devices and --rpc puts remote ones first, both+ # re-indexing the list; a companion widen can add --device after planning.+ if (_extra_args_device(cmd, {"--device", "-dev", "--rpc"}) or "").strip() or any(+ str(env.get(k, "")).strip() for k in ("LLAMA_ARG_DEVICE", "LLAMA_ARG_RPC")+ ):+ return cmd+ mask = env.get("CUDA_VISIBLE_DEVICES")+ if mask is None:+ order = sorted(int(i) for i in margins_by_idx)+ else:+ try:+ order = [int(tok) for tok in str(mask).split(",") if tok.strip()]+ except ValueError:+ return cmd+ if not order or any(i < 0 or i not in margins_by_idx for i in order):+ return cmd+ values = [str(int(margins_by_idx[i])) for i in order]+ spelled = values[0] if len(set(values)) == 1 else ",".join(values)+ if cmd[at + 1] == spelled:+ return cmd+ out = list(cmd)+ out[at + 1] = spelled+ logger.info(+ "Per-device --fit-target %s for GPUs %s: each card keeps its own budget margin.",+ spelled,+ order,+ )+ return out+@staticmethoddef _ctx_integrity_flags(n_parallel: int,diff --git a/studio/backend/tests/test_load_mode_fit_matrix.py b/studio/backend/tests/test_load_mode_fit_matrix.pyindex f14358826c6..084550cb73e 100644--- a/studio/backend/tests/test_load_mode_fit_matrix.py+++ b/studio/backend/tests/test_load_mode_fit_matrix.py@@ -393,7 +393,11 @@ def test_the_launch_charges_the_fitters_margin_only_when_fit_stays_on():assert "fit_target_delta_mib=_fit_target_delta_mibifuse_fitelse0.0" in callassert "supports_fit_target=use_fitand" in callassert "fit_target_margin_in(_fit_extras,_fit_env)or0.0" in call- assert "fit_margin_mib=_fit_margin_mib" in call+ # Studio's per-card margins replace it only while Studio owns the margin: the+ # fitter really runs on this launch's --fit, with no pass-through --fit-target.+ assert "if_fitter_runsanduse_fit" in call+ assert "fit_target_margin_in(_fit_extras,_fit_env)isNoneelseNone" in call+ assert "fit_margin_mib=_fit_margins_by_idxor_fit_margin_mib" in calldef test_the_effective_fitter_state_reads_the_launchs_own_fit_flag():diff --git a/studio/backend/tests/test_managed_engines.py b/studio/backend/tests/test_managed_engines.pyindex 0ede9bbd0d1..080a9ebb434 100644--- a/studio/backend/tests/test_managed_engines.py+++ b/studio/backend/tests/test_managed_engines.py@@ -1149,8 +1149,14 @@ def test_memory_budget_uses_selected_gpu_and_reserves_headroom(monkeypatch):assert commands[0][commands[0].index("--id") + 1] == "1"# vLLM TorchAO weights overrun the reservation in sampler warmup, so keep 6 GiB more free.assert engine_adapters.gpu_memory_fraction([1], 512 + 6144) == 0.395- monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.5)- assert engine_adapters.gpu_memory_fraction([1]) == 0.5+ monkeypatch.setattr(+ engine_adapters.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout = "24576, 24000")+ )+ monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.85)+ assert engine_adapters.gpu_memory_fraction([1]) == 0.85+ # A GGUF-only low budget must not shrink an engine that cannot offload to the CPU.+ monkeypatch.setattr(vram_budget_settings, "get_vram_budget_fraction", lambda: 0.3)+ assert engine_adapters.gpu_memory_fraction([1]) == engine_adapters.MANAGED_ENGINE_BUDGET_FLOORmonkeypatch.setattr(engine_adapters.subprocess, "run", lambda *a, **kw: SimpleNamespace(stdout = "24576, 256"))diff --git a/studio/backend/tests/test_tensor_parallel.py b/studio/backend/tests/test_tensor_parallel.pyindex f1f7c5ba865..6b1c9c7b699 100644--- a/studio/backend/tests/test_tensor_parallel.py+++ b/studio/backend/tests/test_tensor_parallel.py@@ -1107,7 +1107,8 @@ def test_fit_context_follows_the_user_vram_budget(monkeypatch):monkeypatch.setattr(vb, "_cached_setting", lambda _key: 1.0)fit_max = _fit()- monkeypatch.setattr(vb, "_cached_setting", lambda _key: vb.VRAM_FRACTION_MIN)+ # 0.80, not VRAM_FRACTION_MIN: at 0.10 the 8 GiB weights alone exceed the budget, which hands the load to the offload path.+ monkeypatch.setattr(vb, "_cached_setting", lambda _key: 0.80)fit_min = _fit()assert fit_max > fit_default, "claiming the whole card must buy context back"diff --git a/studio/backend/tests/test_vram_budget_settings.py b/studio/backend/tests/test_vram_budget_settings.pyindex ce49358d3f3..1b00928ffcb 100644--- a/studio/backend/tests/test_vram_budget_settings.py+++ b/studio/backend/tests/test_vram_budget_settings.py@@ -36,16 +36,19 @@ def _isolate(monkeypatch):class TestCoerceFraction:- @pytest.mark.parametrize("value", [0.80, 0.9, 0.97, 1.0, "0.85", "1.00"])+ @pytest.mark.parametrize(+ "value", [0.10, 0.30, 0.50, 0.80, 0.9, 0.97, 1.0, "0.25", "0.85", "1.00"]+ )def test_accepts_in_range(self, value):assert vb.coerce_fraction(value) == pytest.approx(float(value))@pytest.mark.parametrize("value",[- 0.79,- 1.01,0.0,+ 0.05,+ 0.099,+ 1.01,-1.0,2.0,"nan",@@ -150,7 +153,7 @@ class TestState:class TestWrite:def test_rejects_out_of_range(self, monkeypatch):monkeypatch.setattr(vb, "_invalidate", lambda _key: None)- for bad in (0.5, 1.5, float("nan"), "abc"):+ for bad in (0.0, 0.05, -0.25, 1.5, float("nan"), "abc"):with pytest.raises(ValueError):vb.set_vram_budget_fraction(bad)@@ -589,7 +592,7 @@ class TestFitTarget:assert "--fit-target" not in flagsdef test_the_move_is_measured_from_the_card_that_makes_it_safe(self):- # --fit-target is broadcast as one value: llama.cpp device order is not ours to assume.+ # The broadcast value, kept wherever llama.cpp's device order is not provable; TestPerDeviceFitTarget covers the per-device list.compact = _shared_setup_1()assert "if_vram_frac!=_CTX_FIT_VRAM_FRACTIONandgpus:" in compactassert "_fit_target_delta_mib=(_CTX_FIT_VRAM_FRACTION-_vram_frac)*_scale" in compact@@ -598,6 +601,249 @@ class TestFitTarget:assert "total_by_idx.get(_idx)or_freefor_idx,_freeingpus" in compact+class TestPerDeviceFitTarget:+ """On mixed cards a lowered budget's margin is spelled per device.++ One broadcast margin is sized by the largest card when lowering, so on a 48 GB ++ 24 GB host at 30% it asked the 24 GB card to keep about 33 GiB free: more than+ the card has, so llama.cpp's fitter put nothing on it.+ """++ _ENV = {"CUDA_DEVICE_ORDER": "PCI_BUS_ID"}+ # 48 GB and 24 GB cards at a 30% budget, legacy path (1024 MiB base).+ _MARGINS = {0: 1024 + 0.67 * 49140, 1: 1024 + 0.67 * 24576}+ _CMD = ["llama-server", "--fit", "on", "--fit-target", "33947", "--jinja"]++ def _spell(+ self,+ cmd = None,+ env = None,+ margins = None,+ ):+ import core.inference.llama_cpp as lc+ return lc.LlamaCppBackend._per_device_fit_target_argv(+ list(cmd or self._CMD),+ self._ENV if env is None else env,+ self._MARGINS if margins is None else margins,+ )++ def test_each_card_keeps_its_own_margin(self):+ out = self._spell()+ assert out[out.index("--fit-target") + 1] == "33947,17489"+ # Only the value moves.+ assert [t for t in out if t != "33947,17489"] == [t for t in self._CMD if t != "33947"]++ def test_the_list_follows_the_child_mask_order(self):+ # CUDA enumerates a numeric mask in the order it lists the ids.+ out = self._spell(env = {**self._ENV, "CUDA_VISIBLE_DEVICES": "1,0"})+ assert out[out.index("--fit-target") + 1] == "17489,33947"++ def test_a_pinned_single_card_gets_its_own_value(self):+ out = self._spell(env = {**self._ENV, "CUDA_VISIBLE_DEVICES": "1"})+ assert out[out.index("--fit-target") + 1] == "17489"++ def test_identical_cards_keep_the_single_value(self):+ # The same meaning, and an unchanged command on homogeneous hosts.+ margins = {0: 9378.0, 1: 9378.0}+ assert self._spell(cmd = ["--fit-target", "9378"], margins = margins) == [+ "--fit-target",+ "9378",+ ]++ @pytest.mark.parametrize(+ "env",+ [+ {"CUDA_DEVICE_ORDER": "FASTEST_FIRST"},+ {},+ {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": "GPU-1234"},+ {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": "0,2"},+ {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "CUDA_VISIBLE_DEVICES": "-1"},+ {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "LLAMA_ARG_DEVICE": "CUDA1"},+ {"CUDA_DEVICE_ORDER": "PCI_BUS_ID", "LLAMA_ARG_RPC": "host:50052"},+ ],+ ids = [+ "fastest-first",+ "no-order",+ "uuid-mask",+ "unknown-id",+ "cpu",+ "env-device",+ "env-rpc",+ ],+ )+ def test_an_unprovable_order_keeps_the_broadcast_value(self, env):+ assert self._spell(env = env) == self._CMD++ @pytest.mark.parametrize("flag", ["--device", "-dev", "--rpc"])+ def test_device_flags_in_the_command_keep_the_broadcast_value(self, flag):+ cmd = [*self._CMD, flag, "CUDA0"]+ assert self._spell(cmd = cmd) == cmd++ def test_nothing_to_spell_without_margins_or_the_flag(self):+ assert self._spell(margins = {}) == self._CMD+ cmd = ["llama-server", "--fit", "on"]+ assert self._spell(cmd = cmd) == cmd++ def test_a_raised_budget_gives_each_card_its_own_smaller_margin(self):+ # Above the default the delta is negative: at 98% the larger card may pack+ # closer than the smaller one, where broadcast held both to the smaller's.+ import core.inference.llama_cpp as lc++ margins = lc.LlamaCppBackend._fit_target_margins_mib(+ auto_fit = False,+ fit_target_delta_by_idx = {0: -0.01 * 49140, 1: -0.01 * 24576},+ )+ out = self._spell(margins = margins)+ assert out[out.index("--fit-target") + 1] == "532,778"+ # At 100% both reach the 512 MiB floor, one value again.+ floored = lc.LlamaCppBackend._fit_target_margins_mib(+ auto_fit = False,+ fit_target_delta_by_idx = {0: -0.03 * 49140, 1: -0.03 * 24576},+ )+ assert floored == {0: 512.0, 1: 512.0}+ out = self._spell(margins = floored)+ assert out[out.index("--fit-target") + 1] == "512"++ def test_per_card_margins_use_the_scalar_formula(self):+ import core.inference.llama_cpp as lc++ margins = lc.LlamaCppBackend._fit_target_margins_mib(+ auto_fit = False, fit_target_delta_by_idx = {0: 32924.0, 1: -4096.0}+ )+ for idx, delta in ((0, 32924.0), (1, -4096.0)):+ assert margins[idx] == lc.LlamaCppBackend._fit_target_margin_mib(+ auto_fit = False, fit_target_delta_mib = delta+ )+ assert (+ lc.LlamaCppBackend._fit_target_margins_mib(auto_fit = False, fit_target_delta_by_idx = None)+ is None+ )+ assert (+ lc.LlamaCppBackend._fit_target_margins_mib(+ auto_fit = False,+ fit_target_delta_by_idx = {0: 1.0},+ supports_fit_target = False,+ )+ is None+ )++ def test_the_paging_check_credits_each_card_its_own_margin(self, monkeypatch):+ # The load-mode fit must spend what the child is told to keep, card by card.+ import core.inference.llama_cpp as lc++ backend = lc.LlamaCppBackend()+ rows = [(0, 47000, 49140), (1, 24000, 24576)]+ gib = 1024**3+ # 20 GiB of weights and 4 GiB of RAM. Broadcast credits 12.7 GiB of VRAM (a+ # 7.3 GiB spill); per card adds the 24 GB card's 6.4 GiB (a 0.9 GiB spill).+ kwargs = dict(avail_mib = 4 * 1024, headroom_mib = 0)+ broadcast = backend._fits_without_paging(20 * gib, rows, vram_margin_mib = 33947.0, **kwargs)+ per_card = backend._fits_without_paging(+ 20 * gib, rows, vram_margin_mib = {0: 33947.0, 1: 17489.0}, **kwargs+ )+ assert broadcast is False+ assert per_card is True++ def _ok(+ self,+ monkeypatch,+ *,+ extra_args = None,+ env = None,+ **attrs,+ ):+ import core.inference.llama_cpp as lc++ B = lc.LlamaCppBackend+ # The eligibility check declines on macOS, so pin the host OS the case means.+ monkeypatch.setattr(lc.sys, "platform", attrs.get("platform", "linux"))+ monkeypatch.setattr(B, "_GPU_IDS_ARE_PCI_INDICES", attrs.get("pci", True))+ monkeypatch.setattr(B, "_SMI_MIG_INDICES", attrs.get("mig", frozenset()))+ monkeypatch.setattr(+ B,+ "_visibility_mask_is_unmappable",+ staticmethod(lambda: attrs.get("unmappable", False)),+ )+ monkeypatch.setattr(+ B, "_host_torch_is_rocm", staticmethod(lambda: attrs.get("rocm", False))+ )+ monkeypatch.setenv("CUDA_DEVICE_ORDER", attrs.get("order", "PCI_BUS_ID"))+ if attrs.get("mask") is None:+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ else:+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", attrs["mask"])+ return B._fit_target_per_device_ok(+ is_vulkan_backend = attrs.get("vulkan", False),+ extra_args = extra_args,+ env = env or {},+ )++ def test_eligible_on_a_plain_cuda_host(self, monkeypatch):+ assert self._ok(monkeypatch)+ assert self._ok(monkeypatch, mask = "1,0")++ def test_a_resolvable_uuid_mask_is_ineligible(self, monkeypatch):+ # The child inherits the uuids, which _per_device_fit_target_argv cannot order.+ assert not self._ok(monkeypatch, mask = "GPU-aaaa,GPU-bbbb")++ @pytest.mark.parametrize(+ "attrs",+ [+ {"vulkan": True},+ {"rocm": True},+ {"pci": None},+ {"pci": False},+ {"order": "FASTEST_FIRST"},+ {"unmappable": True},+ {"mig": frozenset({0})},+ ],+ ids = [+ "vulkan",+ "rocm",+ "ids-unknown",+ "torch-ordinals",+ "fastest-first",+ "uuid-mask",+ "mig",+ ],+ )+ def test_an_unprovable_device_order_is_ineligible(self, monkeypatch, attrs):+ assert not self._ok(monkeypatch, **attrs)++ @pytest.mark.parametrize(+ "extra_args, env",+ [+ (["--fit-target", "2048"], None),+ (None, {"LLAMA_ARG_FIT_TARGET": "2048"}),+ (["--device", "CUDA1"], None),+ (["--rpc", "host:50052"], None),+ (None, {"LLAMA_ARG_DEVICE": "CUDA1"}),+ (["--spec-draft-device", "CUDA1"], None),+ (["--mmproj-device", "CUDA1"], None),+ ],+ ids = [+ "argv-fit-target",+ "env-fit-target",+ "device",+ "rpc",+ "env-device",+ "draft-device",+ "mmproj-device",+ ],+ )+ def test_pass_through_placement_is_ineligible(self, monkeypatch, extra_args, env):+ assert not self._ok(monkeypatch, extra_args = extra_args, env = env)++ def test_the_load_path_wires_the_per_card_margins(self):+ compact = _shared_setup_1()+ # Planned over every probed row, which an unpinned child sees.+ assert "for_idx,_free,_totalin_gpu_mem" in compact+ # Spelled at both spawn sites, against the final env.+ assert compact.count("_per_device_fit_target_argv(") == 2+ # Priced with the same margins it emits.+ assert "fit_margin_mib=_fit_margins_by_idxor_fit_margin_mib" in compact++class TestManualAutoIsPriced:"""Manual + Auto empties ``gpus``, but --fit-target still spends the budget."""diff --git a/studio/backend/utils/vram_budget_settings.py b/studio/backend/utils/vram_budget_settings.pyindex b4f5ce1a674..721cfc65039 100644--- a/studio/backend/utils/vram_budget_settings.py+++ b/studio/backend/utils/vram_budget_settings.py@@ -24,7 +24,8 @@ VRAM_BUDGET_SETTING_KEY = "vram_budget_fraction"VRAM_FRACTION_ENV_VAR = "UNSLOTH_VRAM_FRACTION"# Mirrored in per-model-config.ts as percent for the slider, a pair test_vram_budget_settings.py pins together. The default is the historical _CTX_FIT_VRAM_FRACTION / _GPU_PIN_VRAM_FRACTION.-VRAM_FRACTION_MIN = 0.80+# 0.10, not 0: at 0 every GGUF load runs on the CPU. vLLM / SGLang cannot offload and keep engine_adapters.MANAGED_ENGINE_BUDGET_FLOOR.+VRAM_FRACTION_MIN = 0.10VRAM_FRACTION_MAX = 1.00VRAM_FRACTION_DEFAULT = 0.97diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.tsindex 6e6c0a5658c..b5438115cfc 100644--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts@@ -133,7 +133,7 @@ export const N_PARALLEL_MAX = 64;// Mirrors vram_budget_settings.py VRAM_FRACTION_MIN/MAX/DEFAULT as whole percent (the slider is integer).// Server-wide, so these only bound the control; the value lives in /api/settings/vram-budget.-export const VRAM_BUDGET_PERCENT_MIN = 80;+export const VRAM_BUDGET_PERCENT_MIN = 10;export const VRAM_BUDGET_PERCENT_MAX = 100;export const VRAM_BUDGET_PERCENT_DEFAULT = 97;// Tenths: a whole percent is ~245 MiB of context on a 24 GB card. Mirrorsdiff --git a/studio/frontend/tests/vram-budget.test.ts b/studio/frontend/tests/vram-budget.test.tsindex 01305b76124..81de6d279cb 100644--- a/studio/frontend/tests/vram-budget.test.ts+++ b/studio/frontend/tests/vram-budget.test.ts@@ -56,8 +56,8 @@ test("the default fraction is exactly 0.97, not a float-drifted neighbour", () =});test("the bounds mirror the backend range", () => {- // vram_budget_settings.py: VRAM_FRACTION_MIN 0.80, MAX 1.00, DEFAULT 0.97.- assert.equal(vramPercentToFraction(VRAM_BUDGET_PERCENT_MIN), 0.8);+ // vram_budget_settings.py: VRAM_FRACTION_MIN 0.10, MAX 1.00, DEFAULT 0.97.+ assert.equal(vramPercentToFraction(VRAM_BUDGET_PERCENT_MIN), 0.1);assert.equal(vramPercentToFraction(VRAM_BUDGET_PERCENT_MAX), 1);assert.equal(VRAM_BUDGET_PERCENT_DEFAULT, 97);});