Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Honor user tensor_split when auto tensor-parallel planner returns an even split (#10884)

Chakshu Dhannawata1598ffb507f
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index ea86b1e9575..efdfbc35151 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -6558,6 +6558,13 @@ class LlamaCppBackend:
# Relative model share per GPU (--tensor-split), in GPU order; None =
# default (llama.cpp splits by free VRAM).
self._tensor_split: Optional[List[float]] = None
+ # Same ratio, but for auto tensor-parallel mode. The planner may emit a
+ # weighted split or fall back to a user-supplied ratio; recording it lets
+ # _runtime_matches_intent detect a changed ratio and reload.
+ self._auto_tensor_split: Optional[tuple[float, ...]] = None
+ # The actual split that was emitted to llama-server in auto mode, used for
+ # /status reporting. Manual mode stores this in _tensor_split.
+ self._auto_tensor_split_emitted: Optional[tuple[float, ...]] = None
# Tokens the tensor-spill plan added to this argv, so a retry that re-places
# the model can strip them with _without_subsequence. Empty when it abstained.
self._spill_plan_flags: List[str] = []
@@ -7363,9 +7370,16 @@ class LlamaCppBackend:
@property
def tensor_split(self) -> Optional[List[float]]:
- """Manual-mode relative model share per GPU (--tensor-split); None =
- default (split by free VRAM)."""
- return self._tensor_split
+ """Relative model share per GPU (--tensor-split) for the active load.
+
+ Manual mode returns the requested ratio; auto tensor-parallel mode
+ returns the normalized ratio that was actually emitted, or None when
+ the planner left the split to llama.cpp's default."""
+ if self._tensor_split is not None:
+ return self._tensor_split
+ if self._auto_tensor_split_emitted is not None:
+ return list(self._auto_tensor_split_emitted)
+ return None
@property
def gpu_ids(self) -> Optional[List[int]]:
@@ -7538,6 +7552,16 @@ class LlamaCppBackend:
)
):
return False
+ # Auto tensor-parallel mode can still carry a user ratio, which the
+ # planner's own split does not describe. Two otherwise identical auto
+ # requests with different ratios must not reuse the same server --
+ # and two with the SAME ratio, or with none at all, must.
+ if (
+ intent.gpu_memory_mode != "manual"
+ and self._tensor_parallel
+ and self._auto_tensor_split != self._auto_split_fingerprint(intent.tensor_split)
+ ):
+ return False
if not self.matches_gpu_ids(list(intent.gpu_ids) if intent.gpu_ids else None):
return False
@@ -7744,6 +7768,56 @@ class LlamaCppBackend:
except (TypeError, ValueError, OverflowError):
return []
+ @staticmethod
+ def _format_tensor_split(values: List[float]) -> str:
+ """``--tensor-split`` text for a sanitized ratio, in plain decimal.
+
+ Fixed point rather than ``%g``, which renders a perfectly legal
+ ``1000000,1`` as ``1e+06,1``. ``std::stof`` reads that, but the argv a
+ user is asked to paste into a bug report should say what they asked
+ for, and a whole-number ratio should come back out as the ``3,1`` they
+ typed.
+
+ A ratio of very small numbers would round to ``0,0``, which llama.cpp
+ normalizes by dividing by a zero total. Only that case is rescaled, by
+ the largest weight -- free, because llama.cpp normalizes the list
+ anyway, so ``3,1`` and ``75,25`` are already the same instruction.
+ """
+
+ def _render(items):
+ return [f"{x:.6f}".rstrip("0").rstrip(".") or "0" for x in items]
+
+ rendered = _render(values)
+ if values and all(part == "0" for part in rendered):
+ peak = max(values)
+ if peak > 0:
+ rendered = _render(v / peak for v in values)
+ return ",".join(rendered)
+
+ @staticmethod
+ def _auto_split_fingerprint(tensor_split: Optional[List[float]]) -> Optional[tuple[float, ...]]:
+ """What an auto-mode load ASKED for, as a comparable ratio.
+
+ Requested against requested, the same rule the tuning group in
+ ``_runtime_matches_intent`` uses, and it has to be: what auto mode
+ EMITS is not in the caller's units. The planner's own split is a list
+ of per-device MiB budgets, and a user ratio the budget check declined
+ is emitted as nothing at all -- so comparing either against the
+ caller's ``tensor_split`` mismatches on every repeat of a request that
+ has not changed, and reloads a multi-gigabyte model forever.
+
+ Normalized because llama.cpp normalizes: ``-ts 3,1`` and ``-ts 75,25``
+ are the same instruction, so they are the same intent here. A ratio
+ that sanitizes to nothing is None, which is what an absent ratio is.
+ """
+ if not tensor_split:
+ return None
+ values = LlamaCppBackend._sanitize_tensor_split(tensor_split)
+ total = sum(values)
+ if not values or total <= 0:
+ return None
+ return tuple(round(v / total, 6) for v in values)
+
@property
def layer_preserves_tensor_intent(self) -> bool:
"""True when a downgraded tensor request kept this layer load multi-GPU."""
@@ -16134,6 +16208,8 @@ class LlamaCppBackend:
self._diffusion_requested_ngl = _diffusion_manual_ngl(gpu_memory_mode, gpu_layers)
self._n_cpu_moe = 0
self._tensor_split = None
+ self._auto_tensor_split = None
+ self._auto_tensor_split_emitted = None
self._spill_plan_flags = []
# Diffusion is never tensor-parallel; clear any state left by a prior TP
# chat load (load_model phase 1 only kills the process, it doesn't run
@@ -18928,6 +19004,115 @@ class LlamaCppBackend:
tensor_split = adj
return effective_ctx, max_available_ctx, gpu_indices, tensor_split
+ def _tensor_split_fits_budget(
+ self,
+ split: list[float],
+ gpus: list[tuple[int, int]],
+ gpu_indices: list[int],
+ model_size: int,
+ effective_ctx: int,
+ cache_type_kv: Optional[str] = None,
+ n_parallel: int = 1,
+ n_ubatch: Optional[int] = None,
+ kv_unified: bool = True,
+ flash_attn: bool = True,
+ swa_full: bool = False,
+ mtp_engaged: bool = False,
+ mtp_overhead_fn: Optional[Callable[[int], int]] = None,
+ mtp_flat_reserve_bytes: int = 0,
+ total_by_idx: Optional[dict[int, int]] = None,
+ vram_fraction: Optional[float] = None,
+ soft_overhead_bytes: int = 0,
+ scratch_cache_type_kv: Optional[str] = None,
+ ) -> bool:
+ """Check that a caller-supplied tensor split fits each selected GPU.
+
+ The auto planner returns ``None`` when an even share fits, but a user
+ ratio may still overshoot one card. Validate the ratio against the same
+ per-device budget the planner uses (usable VRAM minus the replicated
+ compute-graph reserve and context-linear buffer).
+
+ "The same budget" is the whole claim, so the operands are the planner's,
+ including the two that are easy to leave out. ``soft_overhead_bytes`` is
+ the CUDA context plus the vision and MTP reserves the planner subtracts
+ from its own KV budget; omitting it makes this check OPTIMISTIC, and a
+ tensor load has no ``--fit`` valve to spill into when the card it
+ over-filled runs out. ``scratch_cache_type_kv`` is the lighter of an
+ asymmetric K/V pair, which is what the context-linear dequant scratch is
+ actually sized from; pricing it from the heavier axis misreads a
+ ``-ctk q4_0 -ctv f16`` load by several gigabytes per device.
+ """
+ if not split or sum(split) <= 0 or len(split) != len(gpu_indices):
+ return False
+
+ _tp_frac = vram_fraction if vram_fraction is not None else _active_vram_fraction()
+
+ def _usable(idx: int, free_mib: int) -> float:
+ t = total_by_idx.get(idx, 0) if total_by_idx else 0
+ return max(0.0, _vram_usable_mib(free_mib, t, _tp_frac))
+
+ free_by_idx = {idx: free for idx, free in gpus}
+ # Fail closed on a device the survey does not cover. ``gpus`` is the set
+ # the tensor-parallel reserve filter admitted and ``gpu_indices`` is what
+ # the load ended up pinned to; they can disagree, and indexing straight
+ # into the first with the second raised KeyError out of load_model. A
+ # budget that cannot be priced is not a budget that passed.
+ if any(idx not in free_by_idx for idx in gpu_indices):
+ return False
+ usable_by_idx = {idx: _usable(idx, free_by_idx[idx]) for idx in gpu_indices}
+
+ _reserve_bytes = self._estimate_compute_buffer_bytes(
+ n_ubatch = n_ubatch, n_parallel = n_parallel, per_device_tensor = True
+ )
+ reserve_mib = (
+ _reserve_bytes // (1024 * 1024)
+ if _reserve_bytes > 0
+ else self._TENSOR_PARALLEL_BUFFER_RESERVE_MIB
+ )
+
+ flat_mtp = max(0, mtp_flat_reserve_bytes)
+ if mtp_engaged and mtp_overhead_fn is None:
+ flat_mtp = max(flat_mtp, 2 * 1024**3)
+
+ kv_bytes = (
+ self._estimate_kv_cache_bytes(
+ effective_ctx,
+ cache_type_kv,
+ n_parallel = n_parallel,
+ swa_full = swa_full,
+ kv_unified = kv_unified,
+ n_ubatch = n_ubatch,
+ flash_attn = flash_attn,
+ )
+ if self._can_estimate_kv() and effective_ctx > 0
+ else 0
+ )
+ mtp_bytes = (
+ mtp_overhead_fn(effective_ctx) if mtp_overhead_fn is not None else 0
+ ) + flat_mtp
+ # REPLICATED, not split: every device allocates the whole
+ # context-linear buffer whatever its weight, which is why the planner
+ # subtracts one per-device slice from each card BEFORE weighting rather
+ # than distributing the aggregate. Weighting the aggregate is the same
+ # arithmetic only for an even ratio; away from even it charges a
+ # low-weight card less than the buffer it must allocate and a high-weight
+ # card more, so it both under-refuses and over-refuses. Mirror the
+ # planner: flat per device, alongside the compute-graph reserve.
+ cc_per_device_bytes = self._compute_buffer_ctx_bytes(
+ effective_ctx, n_ubatch, scratch_cache_type_kv or cache_type_kv
+ )
+ total_bytes = model_size + kv_bytes + mtp_bytes + max(0, soft_overhead_bytes)
+
+ total_weight = sum(split)
+ for i, idx in enumerate(gpu_indices):
+ alloc_bytes = total_bytes * split[i] / total_weight
+ capacity_bytes = (
+ (usable_by_idx[idx] - reserve_mib) * 1024 * 1024 - cc_per_device_bytes
+ )
+ if alloc_bytes > capacity_bytes:
+ return False
+ return True
+
@staticmethod
def _with_mmproj_offload_disabled(
cmd: Sequence[str], env: Optional[Mapping[str, str]] = None
@@ -21216,6 +21401,11 @@ class LlamaCppBackend:
# branch so an earlier launch's recorded drop cannot excuse this one.
self._arch_gate_dropped_tensor_split = None
self._arch_gate_dropped_tensor_parallel = False
+ # Same reason, same place: the auto ratio is recorded only on the
+ # auto tensor branch, so without this a manual load would carry a
+ # previous auto load's ratio forward into the next comparison.
+ self._auto_tensor_split = None
+ self._auto_tensor_split_emitted = None
if gpu_memory_mode == "manual" and gpu_layers >= 0:
self._gpu_layers = gpu_layers
self._n_cpu_moe = n_cpu_moe
@@ -21224,6 +21414,8 @@ class LlamaCppBackend:
self._gpu_layers = -1
self._n_cpu_moe = 0
self._tensor_split = None
+ self._auto_tensor_split = None
+ self._auto_tensor_split_emitted = None
self._requested_gpu_ids = sorted(gpu_ids) if gpu_ids else None
self._gpu_ids = list(self._requested_gpu_ids) if self._requested_gpu_ids else None
# Manual offload skips the TP planner but still emits --split-mode
@@ -21384,6 +21576,16 @@ class LlamaCppBackend:
# kv_cache_bytes, so reading it from the spill arm would be an
# UnboundLocalError on the path that reaches it most often.
_spill_inputs: Optional[dict] = None
+ # Whether the tensor-parallel planner ran to completion. Bound before
+ # the try for the same reason as the two above, and read AFTER it: the
+ # auto ``--tensor-split`` fallback needs the planner's own working set
+ # (``tp_gpus``, ``gpu_indices``, the MTP terms), and every one of those
+ # is bound inside the try. The except arm restores ``tp_tensor_split =
+ # None``, which is precisely the state the fallback fires on, so
+ # without this it would read locals a throw left unbound -- or, worse,
+ # a ``gpu_indices`` the arm rebuilt from a WIDER device set than the
+ # planner's reserve filter admitted, and index a card that is not in it.
+ _tp_planned = False
total_by_idx: dict[int, int] = {}
_gpu_mem: list[tuple[int, int, int]] = []
model_size = None # set in the fit try; used by the APU RAM guard
@@ -22022,7 +22224,8 @@ class LlamaCppBackend:
tp_tensor_split: Optional[list[int]] = None
# Whether the tensor-parallel branch below really planned this load.
# Not readable from tp_tensor_split, which stays None on a tensor
- # split llama.cpp is left to size itself.
+ # split llama.cpp is left to size itself. Re-set here, not only
+ # above the try, so a retry through this scope starts clean.
_tp_planned = False
explicit_ctx = requested_ctx > 0
# Nothing speculative on the GPU: the drafter that launches is the
@@ -24276,18 +24479,75 @@ class LlamaCppBackend:
# See llama.cpp --split-mode.
if tensor_parallel:
cmd.extend(["--split-mode", "tensor"])
+ _emitted_tensor_split: Optional[str] = None
if tp_tensor_split and len(tp_tensor_split) > 1:
- cmd.extend(
- [
- "--tensor-split",
- ",".join(str(int(x)) for x in tp_tensor_split),
- ]
- )
+ _emitted_tensor_split = ",".join(str(int(x)) for x in tp_tensor_split)
+ elif (
+ gpu_memory_mode != "manual" and tensor_split and _tp_planned and gpu_indices
+ ):
+ # Auto tensor planning may decide an even split is safe and
+ # return None, but a user who set a per-GPU ratio still expects
+ # it to reach llama-server. Fall back to the ratio only when
+ # the planner emitted nothing and the ratio actually fits the
+ # per-GPU budget the planner used.
+ #
+ # Gated on _tp_planned, not on tp_tensor_split alone: the fit
+ # block's except arm ALSO leaves tp_tensor_split None, with
+ # gpu_indices either None or rebuilt from every device rather
+ # than the ones the reserve filter admitted. That arm is a
+ # designed degradation -- it launches under `--fit on` -- and it
+ # must not be turned into a KeyError by pricing a ratio against
+ # a survey that was never taken.
+ _split_gpus = self._effective_gpu_count(gpu_indices)
+ _sanitized_split = self._sanitize_tensor_split(tensor_split)
+ if (
+ len(_sanitized_split) == _split_gpus
+ and sum(_sanitized_split) > 0
+ and self._tensor_split_fits_budget(
+ _sanitized_split,
+ tp_gpus,
+ gpu_indices,
+ model_size,
+ effective_ctx,
+ cache_type_kv = cache_type_kv,
+ n_parallel = n_parallel,
+ n_ubatch = _effective_ubatch,
+ kv_unified = planned_kv_unified,
+ flash_attn = planned_flash_attn,
+ swa_full = swa_full,
+ mtp_engaged = _mtp_reserves_gpu,
+ mtp_overhead_fn = mtp_overhead_fn,
+ mtp_flat_reserve_bytes = (
+ 2 * 1024**3 if (_mtp_reserves_gpu and _mtp_kv_unsized) else 0
+ ),
+ total_by_idx = total_by_idx,
+ vram_fraction = _vram_frac,
+ # The planner's own operands, so "the same budget"
+ # in the docstring is true. Both are bound by the
+ # time _tp_planned is set.
+ soft_overhead_bytes = _soft_overhead,
+ scratch_cache_type_kv = _scratch_cache_type_kv,
+ )
+ ):
+ _emitted_tensor_split = self._format_tensor_split(_sanitized_split)
+ _emitted_values: Optional[list[float]] = None
+ if _emitted_tensor_split is not None:
+ cmd.extend(["--tensor-split", _emitted_tensor_split])
+ _emitted_values = [float(x) for x in _emitted_tensor_split.split(",")]
self._tensor_parallel = True
self._layer_preserves_tensor_intent = False
+ # Record the ratio this auto load was ASKED for, so a later
+ # request with a different ratio is not incorrectly reused.
+ # Not the emitted list: see _auto_split_fingerprint for why
+ # recording that instead reloads on every identical repeat.
+ if gpu_memory_mode != "manual":
+ self._auto_tensor_split = self._auto_split_fingerprint(tensor_split)
+ self._auto_tensor_split_emitted = self._auto_split_fingerprint(
+ _emitted_values
+ )
logger.info(
"Tensor parallelism: --split-mode tensor, --tensor-split %s",
- tp_tensor_split,
+ _emitted_tensor_split,
)
else:
self._tensor_parallel = False
@@ -26896,6 +27156,12 @@ class LlamaCppBackend:
# is non-None only in manual mode, which leaves gpu_indices
# None, and this arm needs it truthy.
self._tensor_split = None
+ # The auto twin, which this arm CAN reach: an auto
+ # tensor-parallel fallback emits its ratio with
+ # gpu_indices pinned. Left set it feeds the
+ # `tensor_split` property a ratio the respawned child
+ # does not have.
+ self._auto_tensor_split_emitted = None
# A tensor split needs two devices, so narrowing to one makes
# --split-mode tensor a no-op still REPORTED as active
# (tensor_parallel drives the UI and the MTP watchdog). Strip it.
@@ -28712,6 +28978,8 @@ class LlamaCppBackend:
self._gpu_layers = -1
self._n_cpu_moe = 0
self._tensor_split = None
+ self._auto_tensor_split = None
+ self._auto_tensor_split_emitted = None
self._arch_gate_forced_cpu = False
self._layer_preserves_tensor_intent = False
self._speculative_type = None
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index cf40255eb77..005d2f17598 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -315,11 +315,15 @@ class LoadRequest(BaseModel):
tensor_split: Optional[List[float]] = Field(
None,
description = (
- "Manual mode only: relative share of the model per GPU (--tensor-split), "
- "in the order of the GPUs in use, e.g. [2, 1] for 2:1. Omit it to let "
- "llama.cpp use its default, which splits by free VRAM. Any list given is "
- "passed through as-is, so send [1, 1] to force an even split. Ignored "
- "unless gpu_memory_mode is 'manual' with gpu_layers >= 0."
+ "Relative share of the model per GPU (--tensor-split), in the order of "
+ "the GPUs in use, e.g. [2, 1] for 2:1. Omit it to let llama.cpp use its "
+ "default, which splits by free VRAM. Values are relative, so [1, 1] "
+ "forces an even split and [3, 1] and [75, 25] mean the same thing. "
+ "In manual mode (gpu_layers >= 0) the list is passed through as-is. In "
+ "auto mode it applies only with tensor_parallel, and only when the "
+ "placement planner did not size the split itself; a ratio that does not "
+ "fit the planner's per-GPU budget is dropped rather than forwarded. "
+ "Ignored entirely when gpu_memory_mode is 'manual' with gpu_layers < 0."
),
)
@@ -1274,7 +1278,7 @@ class _InferenceRuntimeFields(BaseModel):
)
tensor_split: Optional[List[float]] = Field(
None,
- description = "Manual mode: relative model share per GPU (--tensor-split); None = default (split by free VRAM).",
+ description = "Relative model share per GPU (--tensor-split) used by the active load; None = default (split by free VRAM).",
)
n_layers: Optional[int] = Field(
None,
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index f129d9cb886..5986f0303e2 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -1790,6 +1790,136 @@ def test_an_explicit_tensor_split_leaves_the_shared_heap_uncredited(tmp_path, mo
)
+def test_auto_tensor_parallel_honors_user_tensor_split_when_planner_returns_none(tmp_path):
+ """When auto tensor planning decides an even split is safe, the user's
+ per-GPU ratio must still be emitted instead of being silently ignored.
+ Regression for unslothai/unsloth#10355."""
+ backend, gguf = _backend_non_vulkan(
+ tmp_path,
+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)],
+ )
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda *args, **kwargs: 0
+ backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0
+ backend._get_gguf_size_bytes = lambda _path: 1 * 1024**3
+ backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = 256
+
+ cmd = _launch(
+ backend,
+ gguf,
+ gpu_memory_mode = "auto",
+ tensor_parallel = True,
+ tensor_split = [3, 1],
+ gpu_ids = [0, 1],
+ n_ctx = 4096,
+ )["cmd"]
+
+ assert backend.tensor_parallel is True
+ assert "--split-mode" in cmd
+ assert cmd[cmd.index("--split-mode") + 1] == "tensor"
+ assert "--tensor-split" in cmd
+ assert cmd[cmd.index("--tensor-split") + 1] == "3,1"
+
+
+def test_auto_tensor_parallel_drops_user_split_when_it_exceeds_budget(tmp_path):
+ """A user ratio that overshoots a GPU's usable budget must not be forwarded
+ in auto mode just because an even split fits."""
+ backend, gguf = _backend_non_vulkan(
+ tmp_path,
+ memory = [(0, 16_000, 16_000), (1, 16_000, 16_000)],
+ )
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda *args, **kwargs: 0
+ backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0
+ backend._get_gguf_size_bytes = lambda _path: 25 * 1024**3
+ backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = 256
+
+ cmd = _launch(
+ backend,
+ gguf,
+ gpu_memory_mode = "auto",
+ tensor_parallel = True,
+ tensor_split = [3, 1],
+ gpu_ids = [0, 1],
+ n_ctx = 4096,
+ )["cmd"]
+
+ assert backend.tensor_parallel is True
+ assert cmd[cmd.index("--split-mode") + 1] == "tensor"
+ assert "--tensor-split" not in cmd
+
+
+def test_auto_tensor_parallel_records_split_for_reload_matching(tmp_path):
+ """An auto tensor-parallel load with a concrete ratio must not be reused
+ when a later request asks for a different ratio.
+
+ What is recorded is the ratio the load was ASKED for, normalized, not the
+ list that was emitted: the planner's own split is in per-device MiB and a
+ declined ratio emits nothing, so recording the emitted value would
+ mismatch every identical repeat and reload forever. See
+ ``_auto_split_fingerprint``.
+ """
+ backend, gguf = _backend_non_vulkan(
+ tmp_path,
+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)],
+ )
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda *args, **kwargs: 0
+ backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0
+ backend._get_gguf_size_bytes = lambda _path: 1 * 1024**3
+ backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = 256
+
+ _launch(
+ backend,
+ gguf,
+ gpu_memory_mode = "auto",
+ tensor_parallel = True,
+ tensor_split = [3, 1],
+ gpu_ids = [0, 1],
+ n_ctx = 4096,
+ )
+
+ assert backend._auto_tensor_split == (0.75, 0.25)
+ # /status should also surface the normalized ratio that was emitted.
+ assert backend.tensor_split == [0.75, 0.25]
+
+ # The same ratio written differently is the same instruction to llama.cpp,
+ # so it reuses; a different one reloads.
+ def _intent(split):
+ return GgufLoadIntent(
+ gguf_path = str(gguf),
+ model_identifier = "test",
+ gpu_memory_mode = "auto",
+ tensor_parallel = True,
+ tensor_split = split,
+ gpu_ids = [0, 1],
+ n_ctx = 4096,
+ )
+
+ assert backend.adopt_load_intent_if_matched(_intent([3, 1])) is True
+ assert backend.adopt_load_intent_if_matched(_intent([6, 2])) is True
+ assert backend.adopt_load_intent_if_matched(_intent([1, 3])) is False
+
+ intent_same = GgufLoadIntent(
+ model_identifier = "test",
+ gpu_memory_mode = "auto",
+ tensor_parallel = True,
+ tensor_split = (3, 1),
+ gpu_ids = (0, 1),
+ n_ctx = 4096,
+ )
+ intent_changed = GgufLoadIntent(
+ model_identifier = "test",
+ gpu_memory_mode = "auto",
+ tensor_parallel = True,
+ tensor_split = (1, 3),
+ gpu_ids = (0, 1),
+ n_ctx = 4096,
+ )
+ assert backend._runtime_matches_intent(intent_same, None) is True
+ assert backend._runtime_matches_intent(intent_changed, None) is False
+
+
def _mixed_vulkan(tmp_path, monkeypatch, memory):
"""A 30 GiB GGUF on a host with 4 GiB of RAM left, full manual offload."""
backend, gguf = _backend(tmp_path, vulkan = True, memory = memory)
diff --git a/studio/backend/tests/test_pr10884_auto_tensor_split_sim.py b/studio/backend/tests/test_pr10884_auto_tensor_split_sim.py
new file mode 100644
index 00000000000..6a37d6266ea
--- /dev/null
+++ b/studio/backend/tests/test_pr10884_auto_tensor_split_sim.py
@@ -0,0 +1,614 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Simulation matrix for the auto tensor-parallel split fallback (PR #10884).
+
+The PR makes an AUTO-mode tensor-parallel load fall back to the user's
+``tensor_split`` when the planner decides an even share fits and returns
+``None``, and records the emitted ratio in ``_auto_tensor_split`` so a later
+request with a different ratio reloads.
+
+This file simulates the hardware the change can meet, on a CPU-only host, by
+driving the real ``LlamaCppBackend`` against faked GPU inventories: the vendor
+(CUDA / Vulkan, which is how AMD and Intel arrive), the card count, asymmetric
+cards, cards whose driver reports no total, and the paravirtualised Metal
+device macOS hands a VM. The reload-deduplication half is driven through the
+real ``adopt_load_intent_if_matched``, because that is the method a second
+``/api/inference/load`` actually reaches.
+"""
+
+from __future__ import annotations
+
+import math
+import sys
+from pathlib import Path
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+import pytest # noqa: E402
+
+from test_llama_cpp_placement import _backend, _launch # noqa: E402
+
+from core.inference.llama_cpp import GgufLoadIntent # noqa: E402
+
+GIB = 1024**3
+
+
+def _tp_backend(
+ tmp_path: Path,
+ *,
+ memory,
+ model_gib: float = 1.0,
+ vulkan: bool = False,
+ reserve_mib: int = 256,
+):
+ """A backend whose placement inputs are fully determined.
+
+ KV and the context-linear compute buffer are zeroed so the only thing the
+ planner and the new budget check weigh is the model size against the cards.
+ That is what makes each case below a statement about ONE variable.
+ """
+ backend, gguf = _backend(tmp_path, vulkan = vulkan, memory = memory)
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda *a, **k: 0
+ backend._compute_buffer_ctx_bytes = lambda *a, **k: 0
+ backend._get_gguf_size_bytes = lambda _path: int(model_gib * GIB)
+ backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = reserve_mib
+ return backend, gguf
+
+
+def _auto_tp(backend, gguf, **kwargs):
+ """An auto-mode tensor-parallel load, the shape the PR is about."""
+ params = dict(
+ gpu_memory_mode = "auto",
+ tensor_parallel = True,
+ n_ctx = 4096,
+ )
+ params.update(kwargs)
+ return _launch(backend, gguf, **params)["cmd"]
+
+
+def _flag(cmd, name):
+ """The value of ``name`` in an argv, or None. Refuses a duplicate."""
+ hits = [i for i, tok in enumerate(cmd) if tok == name]
+ assert len(hits) <= 1, f"{name} appears {len(hits)} times in {cmd}"
+ if not hits:
+ return None
+ return cmd[hits[0] + 1]
+
+
+def _split_values(cmd):
+ raw = _flag(cmd, "--tensor-split")
+ if raw is None:
+ return None
+ return [float(part) for part in raw.split(",")]
+
+
+def _normalized(values):
+ total = sum(values)
+ return tuple(round(v / total, 9) for v in values)
+
+
+# --------------------------------------------------------------------------
+# 1. The fix itself, across card counts, vendors and ratios.
+# --------------------------------------------------------------------------
+
+
[email protected]("vulkan", [False, True], ids = ["cuda", "vulkan"])
+ "memory, ratio",
+ [
+ # Two 24GB cards, the ordinary desktop pair.
+ ([(0, 24_000, 24_000), (1, 24_000, 24_000)], [3, 1]),
+ # Two T4s, which is what a Kaggle session is.
+ ([(0, 15_360, 15_360), (1, 15_360, 15_360)], [3, 1]),
+ # Reversed, so an off-by-order bug cannot pass both.
+ ([(0, 24_000, 24_000), (1, 24_000, 24_000)], [1, 3]),
+ # An even ratio, explicitly asked for rather than defaulted.
+ ([(0, 24_000, 24_000), (1, 24_000, 24_000)], [1, 1]),
+ # Three and four cards.
+ ([(i, 24_000, 24_000) for i in range(3)], [2, 1, 1]),
+ ([(i, 24_000, 24_000) for i in range(4)], [4, 3, 2, 1]),
+ # Mismatched cards, the #10355 complaint exactly: a slow small card
+ # that the user wants to carry less.
+ ([(0, 11_000, 11_000), (1, 24_000, 24_000)], [1, 3]),
+ ],
+)
+def test_the_user_ratio_reaches_llama_server_in_auto_mode(tmp_path, memory, ratio, vulkan):
+ """The regression in #10355: auto mode dropped the user's ratio."""
+ backend, gguf = _tp_backend(tmp_path, memory = memory, vulkan = vulkan)
+ cmd = _auto_tp(
+ backend,
+ gguf,
+ tensor_split = list(ratio),
+ gpu_ids = [idx for idx, *_ in memory],
+ )
+
+ assert _flag(cmd, "--split-mode") == "tensor"
+ assert _split_values(cmd) is not None, "the user ratio was dropped again"
+ assert _normalized(_split_values(cmd)) == pytest.approx(
+ _normalized([float(x) for x in ratio]), abs = 1e-6
+ )
+
+
+def test_the_emitted_token_is_plain_decimal(tmp_path):
+ """llama.cpp splits --tensor-split on commas and parses each field. A token
+ in scientific notation, or one carrying a locale decimal comma, would either
+ be read as a different number or split into extra fields."""
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ cmd = _auto_tp(backend, gguf, tensor_split = [0.75, 0.25], gpu_ids = [0, 1])
+ raw = _flag(cmd, "--tensor-split")
+ assert raw is not None
+ assert raw.count(",") == 1, f"{raw!r} would parse as more fields than there are GPUs"
+ for part in raw.split(","):
+ assert "e" not in part.lower(), f"{part!r} is scientific notation"
+ float(part)
+
+
+ "ratio",
+ [
+ [1_000_000, 1], # a ratio is relative; this is legal and means 1e6:1
+ [0.0000004, 0.0000001], # ...and so is this, which means 4:1
+ [1, 3],
+ ],
+)
+def test_an_extreme_ratio_survives_formatting(tmp_path, ratio):
+ """llama.cpp normalizes the list, so only the proportion has to survive.
+
+ Two ways it can fail to: %g renders 1000000 as "1e+06", and a fixed-point
+ rendering collapses a ratio of very small numbers to "0,0", which is a
+ division by zero once llama.cpp normalizes it.
+ """
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ cmd = _auto_tp(backend, gguf, tensor_split = list(ratio), gpu_ids = [0, 1])
+ raw = _flag(cmd, "--tensor-split")
+ if raw is None:
+ pytest.skip("the budget check declined this ratio; formatting is not reached")
+
+ assert raw.count(",") == len(ratio) - 1, f"{raw!r} parses as the wrong field count"
+ values = [float(part) for part in raw.split(",")]
+ assert sum(values) > 0, f"{raw!r} normalizes to a division by zero"
+ assert _normalized(values) == pytest.approx(
+ _normalized([float(x) for x in ratio]), abs = 1e-6
+ ), f"{raw!r} is not the proportion that was asked for"
+
+
+# --------------------------------------------------------------------------
+# 2. Ratios that must NOT be forwarded.
+# --------------------------------------------------------------------------
+
+
+ "ratio, why",
+ [
+ ([3, 1, 1], "three weights for two cards"),
+ ([3], "one weight for two cards"),
+ ([0, 0], "a total of zero"),
+ ([-1, -1], "negatives, which sanitize to zero"),
+ ([float("nan"), 1], "a NaN weight"),
+ ([float("inf"), 1], "an infinite weight"),
+ ],
+)
+def test_a_degenerate_ratio_is_dropped_not_forwarded(tmp_path, ratio, why):
+ """A bad ratio must leave llama.cpp on its own default, never reach it."""
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ cmd = _auto_tp(backend, gguf, tensor_split = list(ratio), gpu_ids = [0, 1])
+ assert _flag(cmd, "--split-mode") == "tensor", why
+ raw = _flag(cmd, "--tensor-split")
+ if raw is not None:
+ for part in raw.split(","):
+ value = float(part)
+ assert math.isfinite(value) and value >= 0.0, f"{raw!r} ({why})"
+
+
+def test_a_ratio_that_overshoots_one_card_is_dropped(tmp_path):
+ """The planner's own budget, applied to the user's ratio. A 25GB model at
+ 3:1 puts ~19GB on a 16GB card."""
+ backend, gguf = _tp_backend(
+ tmp_path,
+ memory = [(0, 16_000, 16_000), (1, 16_000, 16_000)],
+ model_gib = 25,
+ )
+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
+ assert _flag(cmd, "--split-mode") == "tensor"
+ assert _flag(cmd, "--tensor-split") is None
+
+
+def test_the_budget_check_agrees_with_the_planner_on_an_even_share(tmp_path):
+ """The check's whole claim is that it applies "the same per-device budget
+ the planner uses". Then an EVEN ratio must be accepted exactly when the
+ planner's own even-share rule says the load fits -- otherwise a user who
+ types 1,1 is told their ratio does not fit a machine the planner just
+ decided to split evenly, which is #10355 wearing a different hat.
+
+ Sized to sit in the gap: the context compute buffer is real here, and
+ charging it once (the planner's rule) accepts while charging it twice
+ rejects.
+ """
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ mib = 1024 * 1024
+ cc_per_device_mib = 1024
+ backend._compute_buffer_ctx_bytes = lambda *a, **k: cc_per_device_mib * mib
+
+ gpus = [(0, 24_000), (1, 24_000)]
+ total_by_idx = {0: 24_000, 1: 24_000}
+ # usable = 24000 - max(0.03*24000, min(512, 0.03*24000)) = 23280 MiB per card,
+ # reserve = 256, so the planner's ceiling is 23024 MiB a card.
+ model_mib = 43_452
+ even_share = (model_mib + 2 * cc_per_device_mib) / 2
+ assert 22_000 < even_share < 23_024, "the case has drifted out of the gap it tests"
+
+ assert (
+ backend._tensor_split_fits_budget(
+ [1.0, 1.0],
+ gpus,
+ [0, 1],
+ model_mib * mib,
+ 4096,
+ n_ubatch = 512,
+ total_by_idx = total_by_idx,
+ vram_fraction = 0.97,
+ )
+ is True
+ )
+
+
+def test_the_context_buffer_is_charged_flat_not_by_the_ratio(tmp_path):
+ """The context-linear compute buffer is REPLICATED on every device: each one
+ allocates the whole thing whatever weight it carries. Distributing the
+ aggregate by the ratio instead is the same arithmetic only at an even share,
+ and away from it the high-weight card is charged nearly twice the buffer --
+ so a ratio the planner's own rule accepts is refused, which for a user who
+ typed one is #10355 again. Sized to sit exactly in that gap: the planner
+ accepts 1:9 here, the aggregate form does not.
+ """
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ mib = 1024 * 1024
+ cc_per_device_mib = 6 * 1024
+ backend._compute_buffer_ctx_bytes = lambda *a, **k: cc_per_device_mib * mib
+ model_mib = 14 * 1024
+
+ # usable - reserve = 23024 MiB a card, as in the even-share cell above.
+ ceiling_mib = 23_024
+ assert model_mib / 2 + cc_per_device_mib < ceiling_mib, "the planner would not have"
+ assert 0.9 * model_mib < ceiling_mib - cc_per_device_mib, "charged flat, 1:9 fits"
+ assert 0.9 * (model_mib + 2 * cc_per_device_mib) > ceiling_mib, "charged by ratio, it does not"
+
+ assert (
+ backend._tensor_split_fits_budget(
+ [1.0, 9.0],
+ [(0, 24_000), (1, 24_000)],
+ [0, 1],
+ model_mib * mib,
+ 4096,
+ n_ubatch = 512,
+ total_by_idx = {0: 24_000, 1: 24_000},
+ vram_fraction = 0.97,
+ )
+ is True
+ )
+
+
+def test_the_budget_check_still_refuses_what_does_not_fit(tmp_path):
+ """The other side of the rule above: relaxing the double charge must not
+ turn the check into one that accepts anything."""
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ mib = 1024 * 1024
+ backend._compute_buffer_ctx_bytes = lambda *a, **k: 0
+
+ assert (
+ backend._tensor_split_fits_budget(
+ [9.0, 1.0],
+ [(0, 24_000), (1, 24_000)],
+ [0, 1],
+ 40_000 * mib,
+ 4096,
+ n_ubatch = 512,
+ total_by_idx = {0: 24_000, 1: 24_000},
+ vram_fraction = 0.97,
+ )
+ is False
+ )
+
+
+def test_the_budget_check_fails_closed_on_an_unsurveyed_card(tmp_path):
+ """gpu_indices and the tensor-parallel survey can disagree. Pricing one
+ against the other used to raise KeyError out of load_model."""
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ assert (
+ backend._tensor_split_fits_budget(
+ [1.0, 1.0],
+ [(0, 24_000)],
+ [0, 1],
+ 1024,
+ 4096,
+ total_by_idx = {0: 24_000},
+ )
+ is False
+ )
+
+
+def test_manual_mode_is_untouched_by_the_new_branch(tmp_path):
+ """Manual mode has always emitted the ratio through its own path. The new
+ elif is guarded on `gpu_memory_mode != "manual"`, so this must be
+ byte-identical to its pre-PR behaviour."""
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ cmd = _launch(
+ backend,
+ gguf,
+ gpu_memory_mode = "manual",
+ gpu_layers = 99,
+ tensor_parallel = True,
+ tensor_split = [3, 1],
+ gpu_ids = [0, 1],
+ n_ctx = 4096,
+ )["cmd"]
+ assert _flag(cmd, "--split-mode") == "tensor"
+ assert _flag(cmd, "--tensor-split") == "3,1"
+ assert backend._auto_tensor_split is None, "manual mode must not record an auto ratio"
+
+
+# --------------------------------------------------------------------------
+# 3. Hardware shapes where tensor parallelism is not available at all.
+# --------------------------------------------------------------------------
+
+
+def test_a_single_gpu_never_gets_a_split(tmp_path):
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000)])
+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0])
+ assert _flag(cmd, "--tensor-split") is None
+
+
+def test_no_gpu_at_all_does_not_raise(tmp_path):
+ """CPU-only. The new code divides by len(gpu_indices) and by sum(split);
+ neither may be reached with a zero."""
+ backend, gguf = _tp_backend(tmp_path, memory = [])
+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1])
+ assert _flag(cmd, "--tensor-split") is None
+
+
+def test_cards_whose_driver_reports_no_total(tmp_path):
+ """Some Vulkan drivers report free memory without a total. The new budget
+ check reads `total_by_idx.get(idx, 0)`, so this is the shape that exercises
+ the zero-total branch of _vram_usable_mib."""
+ backend, gguf = _tp_backend(
+ tmp_path,
+ memory = [(0, 24_000, 0), (1, 24_000, 0)],
+ vulkan = True,
+ )
+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
+ assert _flag(cmd, "--split-mode") == "tensor"
+
+
+def test_paravirtual_metal_never_launches_a_tensor_split(tmp_path, monkeypatch):
+ """macOS in a VM pins to CPU and overrides --split-mode. The PR must not
+ reintroduce a split there."""
+ import core.inference.llama_cpp as llama_cpp
+
+ monkeypatch.setattr(llama_cpp, "_metal_device_is_paravirtual", lambda: True)
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
+ assert _flag(cmd, "--tensor-split") is None
+
+
+# --------------------------------------------------------------------------
+# 3b. The placement planner's own recovery arm.
+#
+# `load_model` prices the load inside one long `try`, and its `except` arm is a
+# designed degradation: it drops the plan, sets `--fit on` and launches anyway.
+# It resets `tp_tensor_split` to None but deliberately NOT `tensor_parallel` --
+# which is exactly the state the new fallback fires in, with `gpu_indices`
+# rebuilt from a WIDER set than the `tp_gpus` the planner had filtered, or not
+# rebuilt at all. Three shapes, all of which launched cleanly before the PR.
+# --------------------------------------------------------------------------
+
+
+def _planner_raises(backend):
+ def _boom(*args, **kwargs):
+ raise RuntimeError("simulated GPU selection failure")
+
+ backend._plan_tensor_parallel = _boom
+
+
+def test_a_failed_plan_still_launches_on_vulkan(tmp_path):
+ """Vulkan rebuilds gpu_indices from every DETECTED device, so it can name a
+ card the planner's reserve filter had dropped."""
+ backend, gguf = _tp_backend(
+ tmp_path,
+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 200, 24_000)],
+ vulkan = True,
+ )
+ _planner_raises(backend)
+ cmd = _auto_tp(backend, gguf, tensor_split = [2, 1, 1])
+ assert cmd, "the load must still launch, as it did before the fallback existed"
+
+
+def test_a_failed_plan_still_launches_when_gpu_ids_were_pinned(tmp_path):
+ """The CUDA / ROCm shape: gpu_indices comes back as sorted(gpu_ids), which
+ can include a card the tensor-parallel reserve filter excluded."""
+ backend, gguf = _tp_backend(
+ tmp_path,
+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 200, 24_000)],
+ )
+ _planner_raises(backend)
+ cmd = _auto_tp(backend, gguf, tensor_split = [2, 1, 1], gpu_ids = [0, 1, 2])
+ assert cmd
+
+
+def test_a_plan_that_fails_before_the_gpu_survey_still_launches(tmp_path):
+ """The earliest throw: nothing the fallback reads has been bound yet."""
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+
+ def _boom(_path):
+ raise RuntimeError("simulated GGUF read failure")
+
+ backend._get_gguf_size_bytes = _boom
+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
+ assert cmd
+
+
+# --------------------------------------------------------------------------
+# 4. Reload deduplication -- the half that can regress an already-working path.
+# --------------------------------------------------------------------------
+
+
+def _intent(gguf, **kwargs):
+ params = dict(
+ gguf_path = str(gguf),
+ model_identifier = "test",
+ gpu_memory_mode = "auto",
+ tensor_parallel = True,
+ n_ctx = 4096,
+ )
+ params.update(kwargs)
+ return GgufLoadIntent(**params)
+
+
+def _reuses(backend, gguf, **kwargs):
+ """Whether a second identical /load would reuse the running server."""
+ return backend.adopt_load_intent_if_matched(_intent(gguf, **kwargs))
+
+
+def test_an_identical_auto_request_with_a_ratio_reuses_the_server(tmp_path):
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
+ assert _reuses(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1]) is True
+
+
+def test_a_changed_ratio_does_not_reuse_the_server(tmp_path):
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
+ assert _reuses(backend, gguf, tensor_split = [1, 3], gpu_ids = [0, 1]) is False
+
+
+def test_an_identical_auto_request_with_no_ratio_reuses_the_server(tmp_path):
+ """The plain auto tensor-parallel load: the user set no ratio at all."""
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ _auto_tp(backend, gguf, gpu_ids = [0, 1])
+ assert _reuses(backend, gguf, gpu_ids = [0, 1]) is True
+
+
+def test_a_planner_weighted_split_still_reuses_the_server(tmp_path):
+ """THE REGRESSION THIS FILE EXISTS FOR.
+
+ When the model does not fit evenly the planner returns its OWN weighted
+ split, in MiB, and no user ratio is involved. If the recorded auto ratio is
+ the planner's output while the comparison is against the user's request,
+ every repeat of an identical request mismatches and reloads a
+ multi-gigabyte model -- forever, because the next launch records the same
+ planner output again. This is an ordinary, already-working, pre-PR path.
+ """
+ backend, gguf = _tp_backend(
+ tmp_path,
+ memory = [(0, 24_000, 24_000), (1, 16_000, 16_000)],
+ model_gib = 30,
+ )
+ cmd = _auto_tp(backend, gguf, gpu_ids = [0, 1])
+ assert (
+ _flag(cmd, "--tensor-split") is not None
+ ), "this case is only meaningful when the planner emitted its own split"
+ assert _reuses(backend, gguf, gpu_ids = [0, 1]) is True
+
+
+def test_a_rejected_user_ratio_still_reuses_the_server(tmp_path):
+ """The other half of the same shape: the budget check declined the ratio,
+ so nothing was emitted. Repeating the identical request must not reload --
+ the second launch would decline it identically."""
+ backend, gguf = _tp_backend(
+ tmp_path,
+ memory = [(0, 16_000, 16_000), (1, 16_000, 16_000)],
+ model_gib = 25,
+ )
+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
+ assert _flag(cmd, "--tensor-split") is None
+ assert _reuses(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1]) is True
+
+
+def test_an_equivalent_ratio_reuses_the_server(tmp_path):
+ """3:1 and 6:2 are the same instruction to llama.cpp."""
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
+ assert _reuses(backend, gguf, tensor_split = [6, 2], gpu_ids = [0, 1]) is True
+
+
+def test_unload_clears_the_recorded_ratio(tmp_path):
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
+ assert backend._auto_tensor_split is not None
+ backend.unload_model()
+ assert backend._auto_tensor_split is None
+
+
+def test_a_layer_split_load_clears_the_recorded_ratio(tmp_path):
+ """Turning tensor parallelism off must not leave the ratio behind: the
+ next auto tensor load would compare against a server that never had it."""
+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])
+ _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])
+ _launch(
+ backend,
+ gguf,
+ gpu_memory_mode = "auto",
+ tensor_parallel = False,
+ gpu_ids = [0, 1],
+ n_ctx = 4096,
+ )
+ assert backend._auto_tensor_split is None
+
+
+# --------------------------------------------------------------------------
+# 6. What /status reports after a recovery rewrites the argv.
+# --------------------------------------------------------------------------
+
+
+def _crash_once_with_an_arch_error(backend):
+ """Make the first spawn die the way a binary with no kernels for the pinned
+ card does, so the arch-crash retry runs and the second spawn succeeds."""
+ calls: list[int] = []
+
+ def _health(timeout, **_kw):
+ calls.append(1)
+ return len(calls) > 1
+
+ backend._wait_for_health = _health
+ backend._kernel_image_invalid = lambda _text: True
+ return calls
+
+
+def test_the_arch_crash_retry_stops_reporting_the_ratio_it_dropped(tmp_path):
+ """The emitted ratio is not just a record, it is what `tensor_split` reports.
+
+ The retry re-masks the child onto a narrowed device set, so a split sized for
+ the crashed one weights the wrong cards and `_without_tensor_split` takes it
+ off the argv. Left recorded, /status answers with a ratio the live child does
+ not have, and here the narrowing leaves ONE device, so it would answer with a
+ ratio beside `tensor_parallel: false`. The third card is below the reserve, so
+ the planner pins two of three and the retry has somewhere to go.
+ """
+ backend, gguf = _tp_backend(
+ tmp_path,
+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 400, 24_000)],
+ )
+ _crash_once_with_an_arch_error(backend)
+
+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1])
+
+ assert _flag(cmd, "--tensor-split") is None, "the retry kept a split it re-indexed"
+ assert backend.tensor_parallel is False
+ assert backend.tensor_split is None, (
+ "/status still reports the dropped ratio: " f"{backend.tensor_split}"
+ )
+
+
+def test_a_launch_that_does_not_crash_still_reports_its_ratio(tmp_path):
+ """The control. Clearing on the recovery arm must not clear on the ordinary
+ one, or the PR's own reporting is gone."""
+ backend, gguf = _tp_backend(
+ tmp_path,
+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 400, 24_000)],
+ )
+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1])
+ assert _flag(cmd, "--tensor-split") == "3,1"
+ assert backend.tensor_split == [0.75, 0.25]
diff --git a/studio/backend/tests/test_pr10884_platform_vendor_matrix.py b/studio/backend/tests/test_pr10884_platform_vendor_matrix.py
new file mode 100644
index 00000000000..6ae01c439b2
--- /dev/null
+++ b/studio/backend/tests/test_pr10884_platform_vendor_matrix.py
@@ -0,0 +1,253 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The auto tensor-split fallback across every host Unsloth Studio supports.
+
+``test_pr10884_auto_tensor_split_sim.py`` varies the ratio, the card count and
+the planner's answer. This file varies the HOST: the Cartesian product of
+{Linux, Windows, WSL, macOS} x {NVIDIA, AMD ROCm, AMD or Intel via Vulkan,
+CPU only}, because the code the PR changed sits upstream of every one of them
+and reads `sys.platform`, `_is_wsl` and torch's ROCm markers on the way to a
+launch.
+
+Everything is simulated on a CPU-only Linux host, and the simulation is
+deliberately at the lowest level the production code reads:
+
+* the OS is ``sys.platform`` plus ``_is_wsl``, which is what the module itself
+ branches on;
+* the vendor is a fake ``torch`` in ``sys.modules`` carrying (or not carrying)
+ ``version.hip``, which is exactly what ``_host_torch_is_rocm`` reads, plus
+ the Vulkan backend flag;
+* the cards are the ``(index, free, total)`` rows ``_get_gpu_memory`` returns.
+
+What this can and cannot say is worth stating. It proves the placement
+decision, the emitted argv and the child's device mask on each host. It does
+not prove that a real ROCm or Metal driver then behaves; only hardware can say
+that, and the NVIDIA half of the table is the half that has also been run on
+two real T4s.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+from pathlib import Path
+
+_TESTS_DIR = str(Path(__file__).resolve().parent)
+if _TESTS_DIR not in sys.path:
+ sys.path.insert(0, _TESTS_DIR)
+
+import pytest # noqa: E402
+
+from test_llama_cpp_placement import _backend, _launch # noqa: E402
+
+import core.inference.llama_cpp as llama_cpp # noqa: E402
+
+GIB = 1024**3
+
+# (index, free_mib, total_mib). Two roomy cards: enough that the planner decides
+# an even share is safe and returns nothing, which is the branch the PR added.
+TWO_CARDS = [(0, 24_000, 24_000), (1, 24_000, 24_000)]
+
+OSES = {
+ # label: (sys.platform, _is_wsl)
+ "linux": ("linux", False),
+ "windows": ("win32", False),
+ "wsl": ("linux", True),
+ "macos": ("darwin", False),
+}
+
+VENDORS = ("nvidia", "rocm", "vulkan", "cpu")
+
+
+def _fake_torch(*, rocm: bool, device_count: int) -> types.ModuleType:
+ """Enough torch for the three things the launch path asks it.
+
+ ``_host_torch_is_rocm`` reads ``version.hip`` and ``__version__``;
+ ``_effective_gpu_count`` falls back to ``cuda.device_count()`` when the
+ caller pinned no ids. A real import is not wanted here: the point is to
+ put the host under test, not the host running the test.
+ """
+ torch = types.ModuleType("torch")
+ torch.__version__ = "2.11.0+rocm6.2" if rocm else "2.11.0+cu130"
+ version = types.ModuleType("torch.version")
+ version.hip = "6.2.0" if rocm else None
+ version.cuda = None if rocm else "13.0"
+ torch.version = version
+ cuda = types.SimpleNamespace(
+ is_available = lambda: device_count > 0,
+ device_count = lambda: device_count,
+ )
+ torch.cuda = cuda
+ return torch
+
+
+def host(monkeypatch):
+ """Put the process on a named OS and vendor for the duration of one test."""
+
+ def _apply(
+ os_label: str,
+ vendor: str,
+ *,
+ cards = TWO_CARDS,
+ tmp_path = None,
+ ):
+ platform, is_wsl = OSES[os_label]
+ monkeypatch.setattr(sys, "platform", platform)
+ monkeypatch.setattr(llama_cpp, "_is_wsl", lambda: is_wsl, raising = False)
+ # A real Mac is never the paravirtual case unless it is a VM; that shape
+ # has its own test in the sibling file.
+ monkeypatch.setattr(llama_cpp, "_metal_device_is_paravirtual", lambda: False, raising = False)
+ visible = [] if vendor == "cpu" else cards
+ monkeypatch.setitem(
+ sys.modules,
+ "torch",
+ _fake_torch(rocm = vendor == "rocm", device_count = len(visible)),
+ )
+ backend, gguf = _backend(tmp_path, vulkan = vendor == "vulkan", memory = list(visible))
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda *a, **k: 0
+ backend._compute_buffer_ctx_bytes = lambda *a, **k: 0
+ backend._get_gguf_size_bytes = lambda _path: 1 * GIB
+ backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = 256
+ return backend, gguf, visible
+
+ return _apply
+
+
+def _flag(cmd, name):
+ hits = [i for i, tok in enumerate(cmd) if tok == name]
+ assert len(hits) <= 1, f"{name} appears {len(hits)} times"
+ return cmd[hits[0] + 1] if hits else None
+
+
+def _auto_tp(backend, gguf, visible, **kwargs):
+ params = dict(gpu_memory_mode = "auto", tensor_parallel = True, n_ctx = 4096)
+ if visible:
+ params["gpu_ids"] = [idx for idx, *_ in visible]
+ params.update(kwargs)
+ return _launch(backend, gguf, **params)
+
+
[email protected]("os_label", sorted(OSES))
[email protected]("vendor", VENDORS)
+def test_the_user_ratio_survives_every_host(tmp_path, host, os_label, vendor):
+ """One statement per host: the load launches, and the ratio either reaches
+ llama-server or is absent for a reason this table can name."""
+ backend, gguf, visible = host(os_label, vendor, tmp_path = tmp_path)
+ captured = _auto_tp(backend, gguf, visible, tensor_split = [3, 1])
+ cmd = captured["cmd"]
+
+ if vendor == "cpu":
+ # Nothing to split. The mode is dropped before the emit block, and the
+ # new fallback must not resurrect it.
+ assert _flag(cmd, "--tensor-split") is None
+ return
+
+ assert _flag(cmd, "--split-mode") == "tensor", f"{os_label}/{vendor}"
+ assert _flag(cmd, "--tensor-split") == "3,1", f"{os_label}/{vendor}"
+
+
[email protected]("os_label", sorted(OSES))
[email protected]("vendor", VENDORS)
+def test_a_failed_plan_still_launches_on_every_host(tmp_path, host, os_label, vendor):
+ """The placement planner's except arm is a designed degradation: it drops
+ the plan, sets --fit on and launches anyway. The fallback must not turn
+ that into an exception on ANY host -- it was a KeyError on Vulkan and a
+ TypeError on CUDA and ROCm before this was gated.
+
+ Three cards, one of them below the tensor-parallel compute-buffer reserve,
+ because that is what makes the arm's rebuilt `gpu_indices` WIDER than the
+ `tp_gpus` the planner had filtered. Two equal cards never disagree, and a
+ version of this test that used them passed against the unfixed revision.
+ """
+ cards = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 200, 24_000)]
+ backend, gguf, visible = host(os_label, vendor, cards = cards, tmp_path = tmp_path)
+
+ def _boom(*args, **kwargs):
+ raise RuntimeError("simulated GPU selection failure")
+
+ backend._plan_tensor_parallel = _boom
+ cmd = _auto_tp(backend, gguf, visible, tensor_split = [2, 1, 1])["cmd"]
+ assert cmd, f"{os_label}/{vendor} did not launch"
+
+
[email protected]("os_label", sorted(OSES))
[email protected]("vendor", ("nvidia", "rocm", "vulkan"))
+def test_the_ratio_and_the_device_pin_agree_on_every_host(tmp_path, host, os_label, vendor):
+ """A split is positional over the devices the child can SEE, so the ratio
+ is meaningless unless it has one weight per pinned device. Each vendor
+ pins through a different channel, and the PR does not touch any of them
+ (the diff contains no mask line), so what is asserted here is the
+ AGREEMENT rather than a particular spelling:
+
+ * NVIDIA masks with ``CUDA_VISIBLE_DEVICES``;
+ * ROCm on Linux and WSL masks at the ROCr layer, re-indexing the CUDA mask
+ to the surviving ordinals; on Windows there is no ROCr, so it keeps the
+ HIP mask instead;
+ * Vulkan does not mask at all, it pins ``--device VulkanN`` on the argv.
+ """
+ backend, gguf, visible = host(os_label, vendor, tmp_path = tmp_path)
+ captured = _auto_tp(backend, gguf, visible, tensor_split = [3, 1])
+ env, cmd = captured["env"], captured["cmd"]
+
+ weights = _flag(cmd, "--tensor-split").split(",")
+
+ if vendor == "vulkan":
+ pinned = _flag(cmd, "--device").split(",")
+ assert pinned == ["Vulkan0", "Vulkan1"], f"{os_label}/{vendor}"
+ elif vendor == "rocm" and sys.platform == "win32":
+ pinned = env["HIP_VISIBLE_DEVICES"].split(",")
+ assert "ROCR_VISIBLE_DEVICES" not in env, "ROCr does not exist on Windows"
+ elif vendor == "rocm":
+ pinned = env["ROCR_VISIBLE_DEVICES"].split(",")
+ # With ROCr masking, HIP honours the CUDA mask, which must carry the
+ # POST-mask ordinals or a non-zero pick points out of range.
+ assert env["CUDA_VISIBLE_DEVICES"].split(",") == [
+ str(i) for i in range(len(pinned))
+ ], f"{os_label}/{vendor}"
+ else:
+ pinned = env["CUDA_VISIBLE_DEVICES"].split(",")
+
+ assert (
+ len(weights) == len(pinned) == 2
+ ), f"{os_label}/{vendor}: {len(weights)} weights for {len(pinned)} pinned devices"
+
+
[email protected]("os_label", sorted(OSES))
[email protected]("vendor", ("nvidia", "rocm", "vulkan"))
+def test_an_identical_request_reuses_the_server_on_every_host(
+ tmp_path, host, os_label, vendor, monkeypatch
+):
+ """The reload-dedupe half, per host. A fix that forwards the ratio and then
+ reloads on every identical request has traded one bug for a worse one, and
+ the comparison runs on every platform."""
+ from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend
+
+ backend, gguf, visible = host(os_label, vendor, tmp_path = tmp_path)
+ # The matcher asks the binary about MTP through a CLASSMETHOD, so the
+ # instance stub `_backend` installs does not cover it, and under a
+ # simulated `sys.platform = "win32"` the stdlib's own `shutil.which` then
+ # reaches into `_winapi`, which does not exist on this Linux host. That is
+ # this simulation's ceiling, not the product's: the real Windows answer is
+ # the windows-latest runner. Stub the probe so the axis under test is the
+ # only one being measured.
+ monkeypatch.setattr(
+ LlamaCppBackend, "probe_server_capabilities", staticmethod(lambda *a, **k: {})
+ )
+ _auto_tp(backend, gguf, visible, tensor_split = [3, 1])
+
+ def _intent(split):
+ return GgufLoadIntent(
+ gguf_path = str(gguf),
+ model_identifier = "test",
+ gpu_memory_mode = "auto",
+ tensor_parallel = True,
+ tensor_split = split,
+ gpu_ids = [idx for idx, *_ in visible],
+ n_ctx = 4096,
+ )
+
+ assert backend.adopt_load_intent_if_matched(_intent([3, 1])) is True
+ assert backend.adopt_load_intent_if_matched(_intent([1, 3])) is False
diff --git a/studio/backend/tests/test_pr10884_upgrade_compat.py b/studio/backend/tests/test_pr10884_upgrade_compat.py
new file mode 100644
index 00000000000..63ea42280c0
--- /dev/null
+++ b/studio/backend/tests/test_pr10884_upgrade_compat.py
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""What an already-installed Unsloth Studio does on its first load after this update.
+
+The auto tensor-split fallback changes what a load DOES with a ``tensor_split``
+it is handed. The question an existing install asks is narrower and more
+important: can anything already on my disk start feeding it one?
+
+Saved per-model settings live in one row of ``app_settings``
+(``openai_api_auto_switch_overrides``), and the only things that ever reach a
+load from there are what ``normalize_model_override`` allow-lists on the way in
+and what ``model_override_load_kwargs`` emits on the way out. So the upgrade
+question is answerable exactly, without a database: put a saved override
+through both and see whether a ratio can come out the other side.
+
+Forwards compatibility is the same argument run backwards. The change persists
+nothing new, so a settings row written by this version is byte-identical to one
+written before it, and an older install reading it finds nothing it does not
+already understand.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+import pytest # noqa: E402
+
+from utils.openai_auto_switch_settings import ( # noqa: E402
+ model_override_load_kwargs,
+ normalize_model_override,
+)
+
+# A saved override from an existing install, using every placement key the
+# allow-list does carry. This is the shape that replays on the first load after
+# an update.
+LEGACY_OVERRIDE = {
+ "gpu_memory_mode": "manual",
+ "gpu_layers": 99,
+ "n_cpu_moe": 4,
+ "tensor_parallel": True,
+ "gpu_ids": [0, 1],
+ "gpu_index_kind": "cuda",
+ "max_seq_length": 8192,
+ "kv_cache_dtype": "q8_0",
+ "n_parallel": 2,
+ "llama_extra_args": ["--flash-attn", "on"],
+}
+
+
+def test_a_saved_setting_can_never_carry_a_tensor_split():
+ """The allow-list is the whole attack surface for this change. If a ratio
+ cannot be stored, no existing install can begin sending one at upgrade."""
+ stored = normalize_model_override(dict(LEGACY_OVERRIDE))
+ assert "tensor_split" not in stored
+
+
+ "smuggled",
+ [
+ {"tensor_split": [3, 1]},
+ {"tensor_split": None},
+ {"tensorSplit": [3, 1]},
+ {"tensor_split": "3,1"},
+ ],
+)
+def test_a_ratio_written_into_the_settings_row_by_hand_is_dropped(smuggled):
+ """Not hypothetical: the row is JSON in a sqlite table a user can edit, and
+ a future version could try to add the key. Either way it must not reach a
+ load through this door without the allow-list being changed deliberately."""
+ stored = normalize_model_override({**LEGACY_OVERRIDE, **smuggled})
+ assert "tensor_split" not in stored
+ assert "tensorSplit" not in stored
+
+
[email protected]("is_gguf", [True, False])
+def test_the_replayed_load_asks_for_no_ratio(is_gguf):
+ """The other end of the same path: what the replay actually sends."""
+ stored = normalize_model_override(dict(LEGACY_OVERRIDE))
+ kwargs = model_override_load_kwargs(stored, is_gguf = is_gguf)
+ assert "tensor_split" not in kwargs
+
+
+def test_the_replayed_load_is_unchanged_in_every_other_respect():
+ """A pin against silent drift. If a later edit to this area changes what an
+ existing install replays, this says so in terms a reader can check against
+ the saved row above rather than against a golden blob."""
+ stored = normalize_model_override(dict(LEGACY_OVERRIDE))
+ kwargs = model_override_load_kwargs(stored, is_gguf = True)
+
+ assert kwargs["gpu_memory_mode"] == "manual"
+ assert kwargs["gpu_layers"] == 99
+ assert kwargs["n_cpu_moe"] == 4
+ assert kwargs["tensor_parallel"] is True
+ assert kwargs["gpu_ids"] == [0, 1]
+ assert kwargs["max_seq_length"] == 8192
+ assert kwargs["cache_type_kv"] == "q8_0"
+ assert kwargs["n_parallel"] == 2
+ assert kwargs["llama_extra_args"] == ["--flash-attn", "on"]
+
+
+def test_an_override_from_a_newer_version_loses_nothing_an_older_one_kept():
+ """Forwards compatibility. Round-tripping a stored row through the
+ normalizer must be stable, or an install that upgrades and then rolls back
+ finds its settings rewritten."""
+ once = normalize_model_override(dict(LEGACY_OVERRIDE))
+ twice = normalize_model_override(dict(once))
+ assert once == twice
diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.ts
index 387f4d91b93..eda05a4fa11 100644
--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts
+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts
@@ -430,10 +430,27 @@ const SETTING_CHECKS: SettingCheck[] = [
{
// The split is placement the config cannot carry: the applier clears splitRatio, so a remembered
// config asks for the default distribution while a resident manual load may run a custom one.
+ //
+ // Judged on the mode the RESIDENT server ran, not the one this pick would send. Since
+ // unslothai/unsloth#10884 an auto tensor-parallel load reports a split of its own, chosen by
+ // the planner, and the store never holds one in auto -- applyInferenceStatusToStore nulls it
+ // unless the mode is manual. Comparing the two sides there compares a field the applier
+ // cleared against a server legitimately running the planner's ratio, and declines to adopt a
+ // resident model that is exactly what was asked for. A manual load's custom ratio is still a
+ // real disagreement, and a server too old to report its mode is still compared, so nothing
+ // that used to reload stops reloading.
+ //
+ // Only when the store is holding NO ratio, though. applyInferenceStatusToStore keeps
+ // prevState.splitRatio whenever a gpu-memory edit is pending, so a ratio set under Manual
+ // survives the switch to Auto, and the load path sends store.splitRatio in either mode. Since
+ // this PR the backend honours that ratio in auto too, so adopting on the mode alone would drop
+ // a placement change the user had made and the server would have applied.
placement: true,
ggufPlacement: true,
pinned: () => true,
- agrees: (_c, s, standing) => sameList(standing.splitRatio, s.tensor_split),
+ agrees: (_c, s, standing) =>
+ (s.gpu_memory_mode === "auto" && standing.splitRatio == null) ||
+ sameList(standing.splitRatio, s.tensor_split),
},
{
// A managed override the backend would reject outright. Folding it into "no override" here would
diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.ts
index 2d5f9c381a4..2d1314b62d8 100644
--- a/studio/frontend/tests/resident-config-match.test.ts
+++ b/studio/frontend/tests/resident-config-match.test.ts
@@ -1791,3 +1791,64 @@ test("a runtime_error resident does not claim its draft depth is the default", (
);
}
});
+
+/**
+ * Auto tensor-parallel now reports a split (unslothai/unsloth#10884): the backend emits
+ * one whenever the planner sizes the load itself, and the /status echo carries it. The
+ * store never holds a split in auto mode -- applyInferenceStatusToStore nulls it unless
+ * the mode is manual -- so comparing the two sides here compares a cleared field against
+ * a server that is legitimately running a ratio, and declines to adopt a resident model
+ * that is exactly what was asked for. The split is a manual-mode opinion; in auto it is
+ * the planner's business.
+ */
+test("an auto tensor-parallel server that reports a split still adopts", () => {
+ assert.equal(
+ matches(
+ { gpu_memory_mode: "auto", tensor_parallel: true, tensor_split: [0.75, 0.25] },
+ { ...BLANK, tensorParallel: true },
+ ),
+ true,
+ );
+});
+
+/**
+ * The limit of the rule above. applyInferenceStatusToStore preserves prevState.splitRatio
+ * whenever a gpu-memory edit is pending, so a ratio set under Manual survives the switch
+ * to Auto, and the load path sends store.splitRatio in either mode. Adopting on the
+ * resident's mode alone would drop a placement change the user made and the server would
+ * have applied, since the backend honours a ratio in auto now too.
+ */
+test("a pending ratio the auto resident is not running is still a reload", () => {
+ assert.equal(
+ matches(
+ { gpu_memory_mode: "auto", tensor_parallel: true, tensor_split: [0.75, 0.25] },
+ { ...BLANK, tensorParallel: true },
+ { ...STANDING, splitRatio: [0.5, 0.5] },
+ ),
+ false,
+ );
+});
+
+test("a pending ratio the auto resident IS running adopts", () => {
+ // The other half of the guard: it must not turn into a blanket reload for anyone who
+ // ever touched the ratio.
+ assert.equal(
+ matches(
+ { gpu_memory_mode: "auto", tensor_parallel: true, tensor_split: [0.75, 0.25] },
+ { ...BLANK, tensorParallel: true },
+ { ...STANDING, splitRatio: [0.75, 0.25] },
+ ),
+ true,
+ );
+});
+
+test("a remembered manual split the resident load does not run is still a reload", () => {
+ assert.equal(
+ matches(
+ { gpu_memory_mode: "manual", tensor_split: [0.5, 0.5] },
+ { ...BLANK, gpuMemoryMode: "manual" as const, gpuLayers: 99 },
+ { ...STANDING, splitRatio: [0.75, 0.25] },
+ ),
+ false,
+ );
+});
diff --git a/tests/kaggle/studio_gpu/run_studio_gpu.py b/tests/kaggle/studio_gpu/run_studio_gpu.py
index 84686e099aa..45dd0190801 100644
--- a/tests/kaggle/studio_gpu/run_studio_gpu.py
+++ b/tests/kaggle/studio_gpu/run_studio_gpu.py
@@ -406,6 +406,75 @@ def llama_server_pids() -> list[int]:
return pids
+def llama_server_argvs() -> dict[int, list[str]]:
+ """Full argv of every llama-server child, split on NUL.
+
+ The sibling above throws away everything after argv[0], and for the flags
+ this payload asserts on that is the whole answer. It has to come from the
+ process table: ``GET /api/inference/status`` carries ``tensor_parallel``
+ but its ``tensor_split`` field is documented as the MANUAL-mode ratio and
+ is never written on the auto path, so an auto load reports
+ ``tensor_split: null`` whatever is on the child's command line.
+
+ Split on ``\\x00``, never on whitespace: a model path with a space in it
+ would otherwise become two arguments and shift every index after it.
+ """
+ out: dict[int, list[str]] = {}
+ proc_root = Path("/proc")
+ if not proc_root.is_dir():
+ return out
+ for entry in proc_root.iterdir():
+ if not entry.name.isdigit():
+ continue
+ try:
+ raw = (entry / "cmdline").read_bytes().decode("utf-8", errors = "replace")
+ except OSError:
+ continue
+ argv = [part for part in raw.split("\x00") if part]
+ if not argv:
+ continue
+ if "llama-server" in Path(argv[0]).name:
+ out[int(entry.name)] = argv
+ return out
+
+
+def argv_flag(argv: list[str], name: str) -> str | None:
+ """The value of ``name`` in an argv, accepting ``--flag v`` and ``--flag=v``.
+
+ Returns None when absent. Raises on a DUPLICATE, because two spellings of
+ the same flag mean the last one wins and an assertion that read the first
+ would be reporting on a value the server is not using.
+ """
+ hits: list[str] = []
+ for i, token in enumerate(argv):
+ if token == name:
+ if i + 1 < len(argv):
+ hits.append(argv[i + 1])
+ elif token.startswith(name + "="):
+ hits.append(token.split("=", 1)[1])
+ if len(hits) > 1:
+ raise ValueError(f"{name} appears {len(hits)} times: {hits}")
+ return hits[0] if hits else None
+
+
+def split_ratio(raw: str | None) -> tuple[float, ...] | None:
+ """A ``--tensor-split`` value as a normalized proportion.
+
+ llama.cpp normalizes the list, so ``3,1`` and ``75,25`` are the same
+ instruction and must compare equal here.
+ """
+ if raw is None:
+ return None
+ try:
+ values = [float(part) for part in raw.split(",")]
+ except ValueError:
+ return None
+ total = sum(values)
+ if total <= 0:
+ return None
+ return tuple(round(v / total, 6) for v in values)
+
+
class Payload:
def __init__(self, args: argparse.Namespace) -> None:
self.args = args
@@ -1039,6 +1108,325 @@ class Payload:
detail["failures"] = failures
return self.record("server_flags", not failures, detail)
+ # ------------------------------------------- auto tensor-parallel split
+
+ _TP_FILE = "studio/backend/core/inference/llama_cpp.py"
+
+ def _auto_tp_load(self, ratio: list[float] | None, *, force: bool) -> dict:
+ """One auto-mode tensor-parallel load, and what the child was launched with.
+
+ ``gpu_memory_mode: "auto"`` with ``tensor_parallel: true`` is the exact
+ shape unslothai/unsloth#10355 is about: on two cards of the same size
+ with a model that fits, the planner decides an even share is safe and
+ returns no ratio of its own, and the user's ratio is the only one there
+ is. ``force_reload`` is the real field name -- ``force`` is not on
+ ``LoadRequest`` and pydantic drops it -- so passing it is how this
+ drives a genuine relaunch, and omitting it is how the deduplication
+ half of the check gets a real answer.
+ """
+ body: dict = {
+ "model_path": self.args.chat_model,
+ "is_lora": False,
+ "max_seq_length": self.args.studio_ctx,
+ "gpu_memory_mode": "auto",
+ "tensor_parallel": True,
+ # f16, not a quantized cache: a quantized KV on a llama.cpp older
+ # than b9455 makes the backend drop tensor mode outright, and this
+ # check would then be measuring that drop rather than the split.
+ "cache_type_kv": "f16",
+ "force_reload": force,
+ }
+ if ratio is not None:
+ body["tensor_split"] = list(ratio)
+ if self.args.chat_variant:
+ body["gguf_variant"] = self.args.chat_variant
+ cards = gpu_inventory()
+ if len(cards) >= 2:
+ body["gpu_ids"] = list(range(len(cards)))
+
+ before = llama_server_argvs()
+ self.studio.expect("POST", "/api/inference/load", body, timeout = self.args.load_timeout)
+ code, status = self.studio.get("/api/inference/status")
+ argvs = llama_server_argvs()
+ # The child that is serving NOW. A relaunch gets a new pid, so taking
+ # the newest one is what makes the reload half of this check readable;
+ # with exactly one server there is nothing to choose between.
+ pid = None
+ if argvs:
+ fresh = [p for p in argvs if p not in before] or list(argvs)
+ pid = max(fresh)
+ argv = argvs.get(pid, []) if pid is not None else []
+ try:
+ split_mode = argv_flag(argv, "--split-mode")
+ tensor_split = argv_flag(argv, "--tensor-split")
+ duplicate = None
+ except ValueError as exc:
+ split_mode = tensor_split = None
+ duplicate = str(exc)
+ return {
+ "requested_ratio": list(ratio) if ratio is not None else None,
+ "status_code": code,
+ "status_tensor_parallel": (status or {}).get("tensor_parallel")
+ if isinstance(status, dict)
+ else None,
+ # What /status SAYS the split is, alongside what the child was
+ # actually launched with. These two disagreed for the whole life of
+ # the auto path -- the property returned the manual-mode field,
+ # which auto never writes -- so a client had no way to see the
+ # ratio at all. Reported as its own key so the report shows the
+ # pair rather than one of them.
+ "status_tensor_split": (status or {}).get("tensor_split")
+ if isinstance(status, dict)
+ else None,
+ "llama_server_pid": pid,
+ "split_mode": split_mode,
+ "tensor_split": tensor_split,
+ "tensor_split_ratio": split_ratio(tensor_split),
+ "duplicate_flag": duplicate,
+ "cards_visible": len(cards),
+ "compute_apps": nvidia_compute_apps(),
+ }
+
+ def _checkout_tp_file(self, ref: str) -> tuple[bool, str]:
+ """Put ONE file at ``ref``. The two legs then differ by the PR, and by
+ nothing else: same install, same binary, same weights, same session."""
+ fetch = run(
+ ["git", "-C", str(self.repo_root), "fetch", "--depth", "1", "origin", ref],
+ timeout = 600,
+ )
+ checkout = run(
+ ["git", "-C", str(self.repo_root), "checkout", ref, "--", self._TP_FILE],
+ timeout = 120,
+ )
+ blob = run(
+ ["git", "-C", str(self.repo_root), "hash-object", self._TP_FILE],
+ timeout = 120,
+ )
+ ok = checkout.returncode == 0
+ return ok, (
+ blob.stdout or ""
+ ).strip() or f"checkout rc={checkout.returncode} " f"fetch rc={fetch.returncode}"
+
+ def _restart_for_leg(self, label: str) -> bool:
+ """Stop Unsloth and bring it back on the code now on disk.
+
+ Restarting is the whole point: ``install.sh --local`` overlays the
+ checkout as an EDITABLE install, so the file swapped above is the file
+ the next process imports -- but only the next one. A leg that reused
+ the running server would be measuring the other revision.
+ """
+ self.stop_server()
+ for _ in range(60):
+ if not llama_server_pids():
+ break
+ time.sleep(1)
+ if not self.start_server():
+ log(f"{label}: Unsloth did not come back up")
+ return False
+ return self.authenticate()
+
+ def assert_auto_tensor_split(self) -> bool:
+ """The user's per-GPU ratio must reach llama-server in AUTO mode.
+
+ Three things are checked, and the second is the one nothing else here
+ can see:
+
+ 1. tensor mode is really on, per ``/api/inference/status``;
+ 2. ``--tensor-split`` is on the LIVE child's argv, carrying the
+ proportion that was asked for. Status cannot answer this: its
+ ``tensor_split`` field is the manual-mode ratio and is null on this
+ path whatever the child was launched with;
+ 3. the server still serves, with memory on both cards.
+
+ Then deduplication, without ``force_reload``, because a fix that
+ forwards the ratio and then reloads on every identical request has
+ traded one bug for a worse one: an identical repeat must REUSE the
+ server, and a changed ratio must replace it.
+ """
+ failures: list[str] = []
+ cards = gpu_inventory()
+ detail: dict = {"cards_visible": len(cards), "legs": {}}
+ if len(cards) < 2:
+ detail["failures"] = [
+ f"only {len(cards)} card visible, so auto tensor-parallel placement "
+ f"was NOT exercised; this proves nothing about "
+ f"unslothai/unsloth#10355"
+ ]
+ detail["void"] = True
+ return self.record("auto_tensor_split", False, detail)
+
+ ratio = [float(x) for x in self.args.auto_split_ratio.split(",")]
+ other = list(reversed(ratio))
+
+ first = self._auto_tp_load(ratio, force = True)
+ detail["legs"]["requested"] = first
+ if first.get("duplicate_flag"):
+ failures.append(f"the launch argv is ambiguous: {first['duplicate_flag']}")
+ if first.get("status_tensor_parallel") is not True:
+ failures.append(
+ f"status reports tensor_parallel={first.get('status_tensor_parallel')!r} "
+ f"after an auto tensor-parallel load, so the mode was dropped and the "
+ f"split has nothing to be applied to"
+ )
+ if first.get("split_mode") != "tensor":
+ failures.append(f"the live llama-server has --split-mode {first.get('split_mode')!r}")
+ if first.get("tensor_split") is None:
+ failures.append(
+ f"the live llama-server has NO --tensor-split after a load that asked "
+ f"for {ratio}. This is unslothai/unsloth#10355: auto mode emitted "
+ f"--split-mode tensor and dropped the ratio"
+ )
+ elif first.get("tensor_split_ratio") != split_ratio(",".join(str(x) for x in ratio)):
+ failures.append(
+ f"--tensor-split is {first.get('tensor_split')!r}, which is not the "
+ f"proportion {ratio} that was asked for"
+ )
+ else:
+ # And the API has to be able to SAY so. The argv is ground truth,
+ # but a user cannot read /proc; for the whole life of the auto path
+ # the status property returned the manual-mode field, which auto
+ # never writes, so a server running 3,1 reported null and there was
+ # no way to tell a forwarded ratio from a dropped one from outside.
+ reported = first.get("status_tensor_split")
+ if reported is None:
+ failures.append(
+ f"the live llama-server runs --tensor-split "
+ f"{first.get('tensor_split')!r} and /api/inference/status reports "
+ f"tensor_split: null, so the applied ratio is invisible to every "
+ f"client"
+ )
+ elif split_ratio(",".join(str(x) for x in reported)) != first.get("tensor_split_ratio"):
+ failures.append(
+ f"/api/inference/status reports tensor_split {reported!r} while the "
+ f"live llama-server runs {first.get('tensor_split')!r}"
+ )
+
+ # It has to still serve. A split that loaded and cannot decode is not a
+ # fix, and a CPU fallback answers a chat request just as happily.
+ try:
+ completion = self.studio.expect(
+ "POST",
+ "/v1/chat/completions",
+ {
+ "model": self.args.chat_model,
+ "messages": [{"role": "user", "content": "Say OK."}],
+ # Generous on purpose. A reasoning model spends its budget on
+ # the thinking block first, so a tight cap comes back with an
+ # empty `content` and finish_reason "length" -- a server that
+ # decoded perfectly well, scored as a failure. Measured on
+ # unsloth-t4-ci-d15ea193 with Qwen3-0.6B at max_tokens 16.
+ "max_tokens": 256,
+ "stream": False,
+ },
+ timeout = self.args.chat_timeout,
+ )
+ choice = (completion.get("choices") or [{}])[0]
+ message = choice.get("message") or {}
+ # Either channel counts as decoding: `reasoning_content` is where a
+ # thinking model's tokens land, and tokens are tokens.
+ text = (message.get("content") or "") + (message.get("reasoning_content") or "")
+ usage = completion.get("usage") or {}
+ detail["completion_chars"] = len(text)
+ detail["completion_tokens"] = usage.get("completion_tokens")
+ detail["finish_reason"] = choice.get("finish_reason")
+ if not text.strip() and not usage.get("completion_tokens"):
+ failures.append(
+ f"the split server decoded nothing: finish_reason="
+ f"{choice.get('finish_reason')!r}, usage={usage}"
+ )
+ except StudioError as exc:
+ failures.append(f"the split server could not serve a completion: {exc}"[:600])
+
+ apps = first.get("compute_apps")
+ pid = first.get("llama_server_pid")
+ if isinstance(apps, dict) and pid is not None and pid not in apps:
+ # Evidence, not a verdict: nvidia-smi's per-process view is empty
+ # inside some containers, and an absent row is not a CPU fallback.
+ detail["note_compute_apps"] = (
+ f"nvidia-smi lists no compute app for pid {pid}; GPU residency is "
+ f"unproven here, see the gpu_inference assertion"
+ )
+
+ # ---- deduplication, the half a fix can regress.
+ repeat = self._auto_tp_load(ratio, force = False)
+ detail["legs"]["repeat_same_ratio"] = repeat
+ if repeat.get("llama_server_pid") != first.get("llama_server_pid"):
+ failures.append(
+ f"an IDENTICAL auto request replaced the server "
+ f"(pid {first.get('llama_server_pid')} -> "
+ f"{repeat.get('llama_server_pid')}). Nothing changed, so this "
+ f"reloads the model on every request"
+ )
+
+ changed = self._auto_tp_load(other, force = False)
+ detail["legs"]["changed_ratio"] = changed
+ if changed.get("llama_server_pid") == first.get("llama_server_pid"):
+ failures.append(
+ f"asking for {other} reused the server launched for {ratio}, so the "
+ f"new ratio was silently ignored"
+ )
+ elif changed.get("tensor_split_ratio") != split_ratio(",".join(str(x) for x in other)):
+ failures.append(
+ f"after changing the ratio to {other} the live argv carries "
+ f"{changed.get('tensor_split')!r}"
+ )
+
+ detail["failures"] = failures
+ return self.record("auto_tensor_split", not failures, detail)
+
+ def assert_auto_tensor_split_baseline(self) -> bool:
+ """Run the same probe against the merge-base, first.
+
+ This is what makes the head result mean anything. If the base revision
+ ALSO emits the ratio then the defect is not there, the two legs prove
+ nothing about the PR, and the run is void rather than green -- so the
+ rule below is written as "the base must reproduce", not as a pass.
+ """
+ ok, blob = self._checkout_tp_file(self.args.base_sha)
+ detail: dict = {"base_sha": self.args.base_sha, "blob": blob}
+ if not ok:
+ detail["failures"] = [f"could not put {self._TP_FILE} at the base revision: {blob}"]
+ detail["void"] = True
+ return self.record("auto_tensor_split_baseline", False, detail)
+ if not self._restart_for_leg("baseline"):
+ detail["failures"] = ["Unsloth did not restart on the base revision"]
+ detail["void"] = True
+ return self.record("auto_tensor_split_baseline", False, detail)
+
+ ratio = [float(x) for x in self.args.auto_split_ratio.split(",")]
+ probe = self._auto_tp_load(ratio, force = True)
+ detail["probe"] = probe
+ failures: list[str] = []
+ if probe.get("split_mode") != "tensor":
+ failures.append(
+ f"the base revision did not enter tensor mode at all "
+ f"(--split-mode {probe.get('split_mode')!r}), so the two legs do not "
+ f"differ by the PR and this comparison is void"
+ )
+ detail["void"] = True
+ elif probe.get("tensor_split") is not None:
+ failures.append(
+ f"the base revision ALREADY emits --tensor-split "
+ f"{probe.get('tensor_split')!r}, so #10355 does not reproduce on this "
+ f"hardware and the head result proves nothing"
+ )
+ detail["void"] = True
+ detail["bug_reproduced"] = not failures
+ detail["failures"] = failures
+ return self.record("auto_tensor_split_baseline", not failures, detail)
+
+ def restore_head_revision(self) -> bool:
+ ok, blob = self._checkout_tp_file(self.args.head_sha or "HEAD")
+ self.record(
+ "auto_tensor_split_head_restored",
+ ok,
+ {
+ "blob": blob,
+ "failures": [] if ok else [f"could not restore {self._TP_FILE}: {blob}"],
+ },
+ )
+ return ok and self._restart_for_leg("head")
+
def assert_compaction(self) -> bool:
"""A conversation past the window must COMPACT, and a short one must not.
@@ -2640,6 +3028,55 @@ class Payload:
# ------------------------------------------------------------------ main
+ def execute_legs(self) -> int:
+ """Run exactly the assertions ``--legs`` names, in that order.
+
+ The standard run is one shape: install, boot, then everything. A GPU
+ session has a ceiling and a flag-level regression does not need a
+ training run to be answered, so this exists to spend the session on the
+ question being asked. It names the assertions rather than accepting an
+ arbitrary callable, so a typo is a refusal here and not an empty pass
+ forty minutes into a rented session.
+ """
+ available = {
+ "auto_tensor_split": self.assert_auto_tensor_split,
+ "gpu_inference": self.assert_gpu_inference,
+ "tool_calling": self.assert_tool_calling,
+ "code_execution": self.assert_code_execution,
+ "web_search": self.assert_web_search,
+ "server_flags": self.assert_server_flags,
+ "compaction": self.assert_compaction,
+ "api_key": self.assert_api_key,
+ "tabs": self.assert_tabs,
+ }
+ wanted = [name.strip() for name in self.args.legs.split(",") if name.strip()]
+ unknown = [name for name in wanted if name not in available]
+ if unknown:
+ self.record(
+ "legs",
+ False,
+ {
+ "requested": wanted,
+ "failures": [
+ f"unknown assertion(s) {unknown}; this run measured nothing. "
+ f"Known: {sorted(available)}"
+ ],
+ },
+ )
+ return self.finish()
+
+ # The baseline leg runs FIRST and on its own revision, so the head leg
+ # below is the same session, the same binary and the same weights with
+ # one file changed.
+ if self.args.base_sha and "auto_tensor_split" in wanted:
+ self.assert_auto_tensor_split_baseline()
+ if not self.restore_head_revision():
+ return self.finish()
+
+ for name in wanted:
+ available[name]()
+ return self.finish()
+
def execute(self) -> int:
if not self.preflight():
return self.finish()
@@ -2654,6 +3091,11 @@ class Payload:
if not self.authenticate():
return self.finish()
+ # A focused run: the named assertions, in the order named, and nothing
+ # else. The standard set below is unchanged when --legs is empty.
+ if self.args.legs:
+ return self.execute_legs()
+
# Before the GPU work: it needs nothing but a logged-in session, and
# putting it after a 20-minute training run would mean a training
# failure hides whether API keys work at all.
@@ -2868,6 +3310,35 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
action = "store_true",
help = "do not drive playwright_chat_ui.py (for debugging the API assertions alone)",
)
+ # Auto tensor-parallel split (unslothai/unsloth#10355, PR #10884).
+ ap.add_argument(
+ "--legs",
+ default = "",
+ help = "comma-separated assertion names to run, in the order listed here; "
+ "empty runs the standard set. A focused run of one flag on real "
+ "multi-GPU hardware does not need a 20-minute training leg to be "
+ "meaningful, and a Kaggle session has a ceiling.",
+ )
+ ap.add_argument(
+ "--auto-split-ratio",
+ default = "3,1",
+ help = "the per-GPU ratio the auto tensor-parallel check asks for. Skewed "
+ "on purpose: an even ratio is what the planner emits anyway, so it "
+ "cannot tell a forwarded ratio from a default one.",
+ )
+ ap.add_argument(
+ "--base-sha",
+ default = "",
+ help = "run the auto tensor-parallel check against this revision FIRST, with "
+ "only studio/backend/core/inference/llama_cpp.py swapped, so the two "
+ "legs differ by the change under test and by nothing else. The base "
+ "leg must reproduce the defect or the comparison is void.",
+ )
+ ap.add_argument(
+ "--head-sha",
+ default = "",
+ help = "the revision to restore after a --base-sha leg (default: HEAD).",
+ )
return ap.parse_args(argv)