Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Honor user tensor_split when auto tensor-parallel planner returns an even split (#10884)
Chakshu Dhannawata1598ffb507f
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex ea86b1e9575..efdfbc35151 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -6558,6 +6558,13 @@ class LlamaCppBackend:# Relative model share per GPU (--tensor-split), in GPU order; None =# default (llama.cpp splits by free VRAM).self._tensor_split: Optional[List[float]] = None+ # Same ratio, but for auto tensor-parallel mode. The planner may emit a+ # weighted split or fall back to a user-supplied ratio; recording it lets+ # _runtime_matches_intent detect a changed ratio and reload.+ self._auto_tensor_split: Optional[tuple[float, ...]] = None+ # The actual split that was emitted to llama-server in auto mode, used for+ # /status reporting. Manual mode stores this in _tensor_split.+ self._auto_tensor_split_emitted: Optional[tuple[float, ...]] = None# Tokens the tensor-spill plan added to this argv, so a retry that re-places# the model can strip them with _without_subsequence. Empty when it abstained.self._spill_plan_flags: List[str] = []@@ -7363,9 +7370,16 @@ class LlamaCppBackend:@propertydef tensor_split(self) -> Optional[List[float]]:- """Manual-mode relative model share per GPU (--tensor-split); None =- default (split by free VRAM)."""- return self._tensor_split+ """Relative model share per GPU (--tensor-split) for the active load.++ Manual mode returns the requested ratio; auto tensor-parallel mode+ returns the normalized ratio that was actually emitted, or None when+ the planner left the split to llama.cpp's default."""+ if self._tensor_split is not None:+ return self._tensor_split+ if self._auto_tensor_split_emitted is not None:+ return list(self._auto_tensor_split_emitted)+ return None@propertydef gpu_ids(self) -> Optional[List[int]]:@@ -7538,6 +7552,16 @@ class LlamaCppBackend:)):return False+ # Auto tensor-parallel mode can still carry a user ratio, which the+ # planner's own split does not describe. Two otherwise identical auto+ # requests with different ratios must not reuse the same server --+ # and two with the SAME ratio, or with none at all, must.+ if (+ intent.gpu_memory_mode != "manual"+ and self._tensor_parallel+ and self._auto_tensor_split != self._auto_split_fingerprint(intent.tensor_split)+ ):+ return Falseif not self.matches_gpu_ids(list(intent.gpu_ids) if intent.gpu_ids else None):return False@@ -7744,6 +7768,56 @@ class LlamaCppBackend:except (TypeError, ValueError, OverflowError):return []+ @staticmethod+ def _format_tensor_split(values: List[float]) -> str:+ """``--tensor-split`` text for a sanitized ratio, in plain decimal.++ Fixed point rather than ``%g``, which renders a perfectly legal+ ``1000000,1`` as ``1e+06,1``. ``std::stof`` reads that, but the argv a+ user is asked to paste into a bug report should say what they asked+ for, and a whole-number ratio should come back out as the ``3,1`` they+ typed.++ A ratio of very small numbers would round to ``0,0``, which llama.cpp+ normalizes by dividing by a zero total. Only that case is rescaled, by+ the largest weight -- free, because llama.cpp normalizes the list+ anyway, so ``3,1`` and ``75,25`` are already the same instruction.+ """++ def _render(items):+ return [f"{x:.6f}".rstrip("0").rstrip(".") or "0" for x in items]++ rendered = _render(values)+ if values and all(part == "0" for part in rendered):+ peak = max(values)+ if peak > 0:+ rendered = _render(v / peak for v in values)+ return ",".join(rendered)++ @staticmethod+ def _auto_split_fingerprint(tensor_split: Optional[List[float]]) -> Optional[tuple[float, ...]]:+ """What an auto-mode load ASKED for, as a comparable ratio.++ Requested against requested, the same rule the tuning group in+ ``_runtime_matches_intent`` uses, and it has to be: what auto mode+ EMITS is not in the caller's units. The planner's own split is a list+ of per-device MiB budgets, and a user ratio the budget check declined+ is emitted as nothing at all -- so comparing either against the+ caller's ``tensor_split`` mismatches on every repeat of a request that+ has not changed, and reloads a multi-gigabyte model forever.++ Normalized because llama.cpp normalizes: ``-ts 3,1`` and ``-ts 75,25``+ are the same instruction, so they are the same intent here. A ratio+ that sanitizes to nothing is None, which is what an absent ratio is.+ """+ if not tensor_split:+ return None+ values = LlamaCppBackend._sanitize_tensor_split(tensor_split)+ total = sum(values)+ if not values or total <= 0:+ return None+ return tuple(round(v / total, 6) for v in values)+@propertydef layer_preserves_tensor_intent(self) -> bool:"""True when a downgraded tensor request kept this layer load multi-GPU."""@@ -16134,6 +16208,8 @@ class LlamaCppBackend:self._diffusion_requested_ngl = _diffusion_manual_ngl(gpu_memory_mode, gpu_layers)self._n_cpu_moe = 0self._tensor_split = None+ self._auto_tensor_split = None+ self._auto_tensor_split_emitted = Noneself._spill_plan_flags = []# Diffusion is never tensor-parallel; clear any state left by a prior TP# chat load (load_model phase 1 only kills the process, it doesn't run@@ -18928,6 +19004,115 @@ class LlamaCppBackend:tensor_split = adjreturn effective_ctx, max_available_ctx, gpu_indices, tensor_split+ def _tensor_split_fits_budget(+ self,+ split: list[float],+ gpus: list[tuple[int, int]],+ gpu_indices: list[int],+ model_size: int,+ effective_ctx: int,+ cache_type_kv: Optional[str] = None,+ n_parallel: int = 1,+ n_ubatch: Optional[int] = None,+ kv_unified: bool = True,+ flash_attn: bool = True,+ swa_full: bool = False,+ mtp_engaged: bool = False,+ mtp_overhead_fn: Optional[Callable[[int], int]] = None,+ mtp_flat_reserve_bytes: int = 0,+ total_by_idx: Optional[dict[int, int]] = None,+ vram_fraction: Optional[float] = None,+ soft_overhead_bytes: int = 0,+ scratch_cache_type_kv: Optional[str] = None,+ ) -> bool:+ """Check that a caller-supplied tensor split fits each selected GPU.++ The auto planner returns ``None`` when an even share fits, but a user+ ratio may still overshoot one card. Validate the ratio against the same+ per-device budget the planner uses (usable VRAM minus the replicated+ compute-graph reserve and context-linear buffer).++ "The same budget" is the whole claim, so the operands are the planner's,+ including the two that are easy to leave out. ``soft_overhead_bytes`` is+ the CUDA context plus the vision and MTP reserves the planner subtracts+ from its own KV budget; omitting it makes this check OPTIMISTIC, and a+ tensor load has no ``--fit`` valve to spill into when the card it+ over-filled runs out. ``scratch_cache_type_kv`` is the lighter of an+ asymmetric K/V pair, which is what the context-linear dequant scratch is+ actually sized from; pricing it from the heavier axis misreads a+ ``-ctk q4_0 -ctv f16`` load by several gigabytes per device.+ """+ if not split or sum(split) <= 0 or len(split) != len(gpu_indices):+ return False++ _tp_frac = vram_fraction if vram_fraction is not None else _active_vram_fraction()++ def _usable(idx: int, free_mib: int) -> float:+ t = total_by_idx.get(idx, 0) if total_by_idx else 0+ return max(0.0, _vram_usable_mib(free_mib, t, _tp_frac))++ free_by_idx = {idx: free for idx, free in gpus}+ # Fail closed on a device the survey does not cover. ``gpus`` is the set+ # the tensor-parallel reserve filter admitted and ``gpu_indices`` is what+ # the load ended up pinned to; they can disagree, and indexing straight+ # into the first with the second raised KeyError out of load_model. A+ # budget that cannot be priced is not a budget that passed.+ if any(idx not in free_by_idx for idx in gpu_indices):+ return False+ usable_by_idx = {idx: _usable(idx, free_by_idx[idx]) for idx in gpu_indices}++ _reserve_bytes = self._estimate_compute_buffer_bytes(+ n_ubatch = n_ubatch, n_parallel = n_parallel, per_device_tensor = True+ )+ reserve_mib = (+ _reserve_bytes // (1024 * 1024)+ if _reserve_bytes > 0+ else self._TENSOR_PARALLEL_BUFFER_RESERVE_MIB+ )++ flat_mtp = max(0, mtp_flat_reserve_bytes)+ if mtp_engaged and mtp_overhead_fn is None:+ flat_mtp = max(flat_mtp, 2 * 1024**3)++ kv_bytes = (+ self._estimate_kv_cache_bytes(+ effective_ctx,+ cache_type_kv,+ n_parallel = n_parallel,+ swa_full = swa_full,+ kv_unified = kv_unified,+ n_ubatch = n_ubatch,+ flash_attn = flash_attn,+ )+ if self._can_estimate_kv() and effective_ctx > 0+ else 0+ )+ mtp_bytes = (+ mtp_overhead_fn(effective_ctx) if mtp_overhead_fn is not None else 0+ ) + flat_mtp+ # REPLICATED, not split: every device allocates the whole+ # context-linear buffer whatever its weight, which is why the planner+ # subtracts one per-device slice from each card BEFORE weighting rather+ # than distributing the aggregate. Weighting the aggregate is the same+ # arithmetic only for an even ratio; away from even it charges a+ # low-weight card less than the buffer it must allocate and a high-weight+ # card more, so it both under-refuses and over-refuses. Mirror the+ # planner: flat per device, alongside the compute-graph reserve.+ cc_per_device_bytes = self._compute_buffer_ctx_bytes(+ effective_ctx, n_ubatch, scratch_cache_type_kv or cache_type_kv+ )+ total_bytes = model_size + kv_bytes + mtp_bytes + max(0, soft_overhead_bytes)++ total_weight = sum(split)+ for i, idx in enumerate(gpu_indices):+ alloc_bytes = total_bytes * split[i] / total_weight+ capacity_bytes = (+ (usable_by_idx[idx] - reserve_mib) * 1024 * 1024 - cc_per_device_bytes+ )+ if alloc_bytes > capacity_bytes:+ return False+ return True+@staticmethoddef _with_mmproj_offload_disabled(cmd: Sequence[str], env: Optional[Mapping[str, str]] = None@@ -21216,6 +21401,11 @@ class LlamaCppBackend:# branch so an earlier launch's recorded drop cannot excuse this one.self._arch_gate_dropped_tensor_split = Noneself._arch_gate_dropped_tensor_parallel = False+ # Same reason, same place: the auto ratio is recorded only on the+ # auto tensor branch, so without this a manual load would carry a+ # previous auto load's ratio forward into the next comparison.+ self._auto_tensor_split = None+ self._auto_tensor_split_emitted = Noneif gpu_memory_mode == "manual" and gpu_layers >= 0:self._gpu_layers = gpu_layersself._n_cpu_moe = n_cpu_moe@@ -21224,6 +21414,8 @@ class LlamaCppBackend:self._gpu_layers = -1self._n_cpu_moe = 0self._tensor_split = None+ self._auto_tensor_split = None+ self._auto_tensor_split_emitted = Noneself._requested_gpu_ids = sorted(gpu_ids) if gpu_ids else Noneself._gpu_ids = list(self._requested_gpu_ids) if self._requested_gpu_ids else None# Manual offload skips the TP planner but still emits --split-mode@@ -21384,6 +21576,16 @@ class LlamaCppBackend:# kv_cache_bytes, so reading it from the spill arm would be an# UnboundLocalError on the path that reaches it most often._spill_inputs: Optional[dict] = None+ # Whether the tensor-parallel planner ran to completion. Bound before+ # the try for the same reason as the two above, and read AFTER it: the+ # auto ``--tensor-split`` fallback needs the planner's own working set+ # (``tp_gpus``, ``gpu_indices``, the MTP terms), and every one of those+ # is bound inside the try. The except arm restores ``tp_tensor_split =+ # None``, which is precisely the state the fallback fires on, so+ # without this it would read locals a throw left unbound -- or, worse,+ # a ``gpu_indices`` the arm rebuilt from a WIDER device set than the+ # planner's reserve filter admitted, and index a card that is not in it.+ _tp_planned = Falsetotal_by_idx: dict[int, int] = {}_gpu_mem: list[tuple[int, int, int]] = []model_size = None # set in the fit try; used by the APU RAM guard@@ -22022,7 +22224,8 @@ class LlamaCppBackend:tp_tensor_split: Optional[list[int]] = None# Whether the tensor-parallel branch below really planned this load.# Not readable from tp_tensor_split, which stays None on a tensor- # split llama.cpp is left to size itself.+ # split llama.cpp is left to size itself. Re-set here, not only+ # above the try, so a retry through this scope starts clean._tp_planned = Falseexplicit_ctx = requested_ctx > 0# Nothing speculative on the GPU: the drafter that launches is the@@ -24276,18 +24479,75 @@ class LlamaCppBackend:# See llama.cpp --split-mode.if tensor_parallel:cmd.extend(["--split-mode", "tensor"])+ _emitted_tensor_split: Optional[str] = Noneif tp_tensor_split and len(tp_tensor_split) > 1:- cmd.extend(- [- "--tensor-split",- ",".join(str(int(x)) for x in tp_tensor_split),- ]- )+ _emitted_tensor_split = ",".join(str(int(x)) for x in tp_tensor_split)+ elif (+ gpu_memory_mode != "manual" and tensor_split and _tp_planned and gpu_indices+ ):+ # Auto tensor planning may decide an even split is safe and+ # return None, but a user who set a per-GPU ratio still expects+ # it to reach llama-server. Fall back to the ratio only when+ # the planner emitted nothing and the ratio actually fits the+ # per-GPU budget the planner used.+ #+ # Gated on _tp_planned, not on tp_tensor_split alone: the fit+ # block's except arm ALSO leaves tp_tensor_split None, with+ # gpu_indices either None or rebuilt from every device rather+ # than the ones the reserve filter admitted. That arm is a+ # designed degradation -- it launches under `--fit on` -- and it+ # must not be turned into a KeyError by pricing a ratio against+ # a survey that was never taken.+ _split_gpus = self._effective_gpu_count(gpu_indices)+ _sanitized_split = self._sanitize_tensor_split(tensor_split)+ if (+ len(_sanitized_split) == _split_gpus+ and sum(_sanitized_split) > 0+ and self._tensor_split_fits_budget(+ _sanitized_split,+ tp_gpus,+ gpu_indices,+ model_size,+ effective_ctx,+ cache_type_kv = cache_type_kv,+ n_parallel = n_parallel,+ n_ubatch = _effective_ubatch,+ kv_unified = planned_kv_unified,+ flash_attn = planned_flash_attn,+ swa_full = swa_full,+ mtp_engaged = _mtp_reserves_gpu,+ mtp_overhead_fn = mtp_overhead_fn,+ mtp_flat_reserve_bytes = (+ 2 * 1024**3 if (_mtp_reserves_gpu and _mtp_kv_unsized) else 0+ ),+ total_by_idx = total_by_idx,+ vram_fraction = _vram_frac,+ # The planner's own operands, so "the same budget"+ # in the docstring is true. Both are bound by the+ # time _tp_planned is set.+ soft_overhead_bytes = _soft_overhead,+ scratch_cache_type_kv = _scratch_cache_type_kv,+ )+ ):+ _emitted_tensor_split = self._format_tensor_split(_sanitized_split)+ _emitted_values: Optional[list[float]] = None+ if _emitted_tensor_split is not None:+ cmd.extend(["--tensor-split", _emitted_tensor_split])+ _emitted_values = [float(x) for x in _emitted_tensor_split.split(",")]self._tensor_parallel = Trueself._layer_preserves_tensor_intent = False+ # Record the ratio this auto load was ASKED for, so a later+ # request with a different ratio is not incorrectly reused.+ # Not the emitted list: see _auto_split_fingerprint for why+ # recording that instead reloads on every identical repeat.+ if gpu_memory_mode != "manual":+ self._auto_tensor_split = self._auto_split_fingerprint(tensor_split)+ self._auto_tensor_split_emitted = self._auto_split_fingerprint(+ _emitted_values+ )logger.info("Tensor parallelism: --split-mode tensor, --tensor-split %s",- tp_tensor_split,+ _emitted_tensor_split,)else:self._tensor_parallel = False@@ -26896,6 +27156,12 @@ class LlamaCppBackend:# is non-None only in manual mode, which leaves gpu_indices# None, and this arm needs it truthy.self._tensor_split = None+ # The auto twin, which this arm CAN reach: an auto+ # tensor-parallel fallback emits its ratio with+ # gpu_indices pinned. Left set it feeds the+ # `tensor_split` property a ratio the respawned child+ # does not have.+ self._auto_tensor_split_emitted = None# A tensor split needs two devices, so narrowing to one makes# --split-mode tensor a no-op still REPORTED as active# (tensor_parallel drives the UI and the MTP watchdog). Strip it.@@ -28712,6 +28978,8 @@ class LlamaCppBackend:self._gpu_layers = -1self._n_cpu_moe = 0self._tensor_split = None+ self._auto_tensor_split = None+ self._auto_tensor_split_emitted = Noneself._arch_gate_forced_cpu = Falseself._layer_preserves_tensor_intent = Falseself._speculative_type = Nonediff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex cf40255eb77..005d2f17598 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -315,11 +315,15 @@ class LoadRequest(BaseModel):tensor_split: Optional[List[float]] = Field(None,description = (- "Manual mode only: relative share of the model per GPU (--tensor-split), "- "in the order of the GPUs in use, e.g. [2, 1] for 2:1. Omit it to let "- "llama.cpp use its default, which splits by free VRAM. Any list given is "- "passed through as-is, so send [1, 1] to force an even split. Ignored "- "unless gpu_memory_mode is 'manual' with gpu_layers >= 0."+ "Relative share of the model per GPU (--tensor-split), in the order of "+ "the GPUs in use, e.g. [2, 1] for 2:1. Omit it to let llama.cpp use its "+ "default, which splits by free VRAM. Values are relative, so [1, 1] "+ "forces an even split and [3, 1] and [75, 25] mean the same thing. "+ "In manual mode (gpu_layers >= 0) the list is passed through as-is. In "+ "auto mode it applies only with tensor_parallel, and only when the "+ "placement planner did not size the split itself; a ratio that does not "+ "fit the planner's per-GPU budget is dropped rather than forwarded. "+ "Ignored entirely when gpu_memory_mode is 'manual' with gpu_layers < 0."),)@@ -1274,7 +1278,7 @@ class _InferenceRuntimeFields(BaseModel):)tensor_split: Optional[List[float]] = Field(None,- description = "Manual mode: relative model share per GPU (--tensor-split); None = default (split by free VRAM).",+ description = "Relative model share per GPU (--tensor-split) used by the active load; None = default (split by free VRAM).",)n_layers: Optional[int] = Field(None,diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.pyindex f129d9cb886..5986f0303e2 100644--- a/studio/backend/tests/test_llama_cpp_placement.py+++ b/studio/backend/tests/test_llama_cpp_placement.py@@ -1790,6 +1790,136 @@ def test_an_explicit_tensor_split_leaves_the_shared_heap_uncredited(tmp_path, mo)+def test_auto_tensor_parallel_honors_user_tensor_split_when_planner_returns_none(tmp_path):+ """When auto tensor planning decides an even split is safe, the user's+ per-GPU ratio must still be emitted instead of being silently ignored.+ Regression for unslothai/unsloth#10355."""+ backend, gguf = _backend_non_vulkan(+ tmp_path,+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)],+ )+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda *args, **kwargs: 0+ backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0+ backend._get_gguf_size_bytes = lambda _path: 1 * 1024**3+ backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = 256++ cmd = _launch(+ backend,+ gguf,+ gpu_memory_mode = "auto",+ tensor_parallel = True,+ tensor_split = [3, 1],+ gpu_ids = [0, 1],+ n_ctx = 4096,+ )["cmd"]++ assert backend.tensor_parallel is True+ assert "--split-mode" in cmd+ assert cmd[cmd.index("--split-mode") + 1] == "tensor"+ assert "--tensor-split" in cmd+ assert cmd[cmd.index("--tensor-split") + 1] == "3,1"+++def test_auto_tensor_parallel_drops_user_split_when_it_exceeds_budget(tmp_path):+ """A user ratio that overshoots a GPU's usable budget must not be forwarded+ in auto mode just because an even split fits."""+ backend, gguf = _backend_non_vulkan(+ tmp_path,+ memory = [(0, 16_000, 16_000), (1, 16_000, 16_000)],+ )+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda *args, **kwargs: 0+ backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0+ backend._get_gguf_size_bytes = lambda _path: 25 * 1024**3+ backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = 256++ cmd = _launch(+ backend,+ gguf,+ gpu_memory_mode = "auto",+ tensor_parallel = True,+ tensor_split = [3, 1],+ gpu_ids = [0, 1],+ n_ctx = 4096,+ )["cmd"]++ assert backend.tensor_parallel is True+ assert cmd[cmd.index("--split-mode") + 1] == "tensor"+ assert "--tensor-split" not in cmd+++def test_auto_tensor_parallel_records_split_for_reload_matching(tmp_path):+ """An auto tensor-parallel load with a concrete ratio must not be reused+ when a later request asks for a different ratio.++ What is recorded is the ratio the load was ASKED for, normalized, not the+ list that was emitted: the planner's own split is in per-device MiB and a+ declined ratio emits nothing, so recording the emitted value would+ mismatch every identical repeat and reload forever. See+ ``_auto_split_fingerprint``.+ """+ backend, gguf = _backend_non_vulkan(+ tmp_path,+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)],+ )+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda *args, **kwargs: 0+ backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0+ backend._get_gguf_size_bytes = lambda _path: 1 * 1024**3+ backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = 256++ _launch(+ backend,+ gguf,+ gpu_memory_mode = "auto",+ tensor_parallel = True,+ tensor_split = [3, 1],+ gpu_ids = [0, 1],+ n_ctx = 4096,+ )++ assert backend._auto_tensor_split == (0.75, 0.25)+ # /status should also surface the normalized ratio that was emitted.+ assert backend.tensor_split == [0.75, 0.25]++ # The same ratio written differently is the same instruction to llama.cpp,+ # so it reuses; a different one reloads.+ def _intent(split):+ return GgufLoadIntent(+ gguf_path = str(gguf),+ model_identifier = "test",+ gpu_memory_mode = "auto",+ tensor_parallel = True,+ tensor_split = split,+ gpu_ids = [0, 1],+ n_ctx = 4096,+ )++ assert backend.adopt_load_intent_if_matched(_intent([3, 1])) is True+ assert backend.adopt_load_intent_if_matched(_intent([6, 2])) is True+ assert backend.adopt_load_intent_if_matched(_intent([1, 3])) is False++ intent_same = GgufLoadIntent(+ model_identifier = "test",+ gpu_memory_mode = "auto",+ tensor_parallel = True,+ tensor_split = (3, 1),+ gpu_ids = (0, 1),+ n_ctx = 4096,+ )+ intent_changed = GgufLoadIntent(+ model_identifier = "test",+ gpu_memory_mode = "auto",+ tensor_parallel = True,+ tensor_split = (1, 3),+ gpu_ids = (0, 1),+ n_ctx = 4096,+ )+ assert backend._runtime_matches_intent(intent_same, None) is True+ assert backend._runtime_matches_intent(intent_changed, None) is False++def _mixed_vulkan(tmp_path, monkeypatch, memory):"""A 30 GiB GGUF on a host with 4 GiB of RAM left, full manual offload."""backend, gguf = _backend(tmp_path, vulkan = True, memory = memory)diff --git a/studio/backend/tests/test_pr10884_auto_tensor_split_sim.py b/studio/backend/tests/test_pr10884_auto_tensor_split_sim.pynew file mode 100644index 00000000000..6a37d6266ea--- /dev/null+++ b/studio/backend/tests/test_pr10884_auto_tensor_split_sim.py@@ -0,0 +1,614 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Simulation matrix for the auto tensor-parallel split fallback (PR #10884).++The PR makes an AUTO-mode tensor-parallel load fall back to the user's+``tensor_split`` when the planner decides an even share fits and returns+``None``, and records the emitted ratio in ``_auto_tensor_split`` so a later+request with a different ratio reloads.++This file simulates the hardware the change can meet, on a CPU-only host, by+driving the real ``LlamaCppBackend`` against faked GPU inventories: the vendor+(CUDA / Vulkan, which is how AMD and Intel arrive), the card count, asymmetric+cards, cards whose driver reports no total, and the paravirtualised Metal+device macOS hands a VM. The reload-deduplication half is driven through the+real ``adopt_load_intent_if_matched``, because that is the method a second+``/api/inference/load`` actually reaches.+"""++from __future__ import annotations++import math+import sys+from pathlib import Path++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++import pytest # noqa: E402++from test_llama_cpp_placement import _backend, _launch # noqa: E402++from core.inference.llama_cpp import GgufLoadIntent # noqa: E402++GIB = 1024**3+++def _tp_backend(+ tmp_path: Path,+ *,+ memory,+ model_gib: float = 1.0,+ vulkan: bool = False,+ reserve_mib: int = 256,+):+ """A backend whose placement inputs are fully determined.++ KV and the context-linear compute buffer are zeroed so the only thing the+ planner and the new budget check weigh is the model size against the cards.+ That is what makes each case below a statement about ONE variable.+ """+ backend, gguf = _backend(tmp_path, vulkan = vulkan, memory = memory)+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda *a, **k: 0+ backend._compute_buffer_ctx_bytes = lambda *a, **k: 0+ backend._get_gguf_size_bytes = lambda _path: int(model_gib * GIB)+ backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = reserve_mib+ return backend, gguf+++def _auto_tp(backend, gguf, **kwargs):+ """An auto-mode tensor-parallel load, the shape the PR is about."""+ params = dict(+ gpu_memory_mode = "auto",+ tensor_parallel = True,+ n_ctx = 4096,+ )+ params.update(kwargs)+ return _launch(backend, gguf, **params)["cmd"]+++def _flag(cmd, name):+ """The value of ``name`` in an argv, or None. Refuses a duplicate."""+ hits = [i for i, tok in enumerate(cmd) if tok == name]+ assert len(hits) <= 1, f"{name} appears {len(hits)} times in {cmd}"+ if not hits:+ return None+ return cmd[hits[0] + 1]+++def _split_values(cmd):+ raw = _flag(cmd, "--tensor-split")+ if raw is None:+ return None+ return [float(part) for part in raw.split(",")]+++def _normalized(values):+ total = sum(values)+ return tuple(round(v / total, 9) for v in values)+++# --------------------------------------------------------------------------+# 1. The fix itself, across card counts, vendors and ratios.+# --------------------------------------------------------------------------++[email protected]("vulkan", [False, True], ids = ["cuda", "vulkan"])+ "memory, ratio",+ [+ # Two 24GB cards, the ordinary desktop pair.+ ([(0, 24_000, 24_000), (1, 24_000, 24_000)], [3, 1]),+ # Two T4s, which is what a Kaggle session is.+ ([(0, 15_360, 15_360), (1, 15_360, 15_360)], [3, 1]),+ # Reversed, so an off-by-order bug cannot pass both.+ ([(0, 24_000, 24_000), (1, 24_000, 24_000)], [1, 3]),+ # An even ratio, explicitly asked for rather than defaulted.+ ([(0, 24_000, 24_000), (1, 24_000, 24_000)], [1, 1]),+ # Three and four cards.+ ([(i, 24_000, 24_000) for i in range(3)], [2, 1, 1]),+ ([(i, 24_000, 24_000) for i in range(4)], [4, 3, 2, 1]),+ # Mismatched cards, the #10355 complaint exactly: a slow small card+ # that the user wants to carry less.+ ([(0, 11_000, 11_000), (1, 24_000, 24_000)], [1, 3]),+ ],+)+def test_the_user_ratio_reaches_llama_server_in_auto_mode(tmp_path, memory, ratio, vulkan):+ """The regression in #10355: auto mode dropped the user's ratio."""+ backend, gguf = _tp_backend(tmp_path, memory = memory, vulkan = vulkan)+ cmd = _auto_tp(+ backend,+ gguf,+ tensor_split = list(ratio),+ gpu_ids = [idx for idx, *_ in memory],+ )++ assert _flag(cmd, "--split-mode") == "tensor"+ assert _split_values(cmd) is not None, "the user ratio was dropped again"+ assert _normalized(_split_values(cmd)) == pytest.approx(+ _normalized([float(x) for x in ratio]), abs = 1e-6+ )+++def test_the_emitted_token_is_plain_decimal(tmp_path):+ """llama.cpp splits --tensor-split on commas and parses each field. A token+ in scientific notation, or one carrying a locale decimal comma, would either+ be read as a different number or split into extra fields."""+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ cmd = _auto_tp(backend, gguf, tensor_split = [0.75, 0.25], gpu_ids = [0, 1])+ raw = _flag(cmd, "--tensor-split")+ assert raw is not None+ assert raw.count(",") == 1, f"{raw!r} would parse as more fields than there are GPUs"+ for part in raw.split(","):+ assert "e" not in part.lower(), f"{part!r} is scientific notation"+ float(part)+++ "ratio",+ [+ [1_000_000, 1], # a ratio is relative; this is legal and means 1e6:1+ [0.0000004, 0.0000001], # ...and so is this, which means 4:1+ [1, 3],+ ],+)+def test_an_extreme_ratio_survives_formatting(tmp_path, ratio):+ """llama.cpp normalizes the list, so only the proportion has to survive.++ Two ways it can fail to: %g renders 1000000 as "1e+06", and a fixed-point+ rendering collapses a ratio of very small numbers to "0,0", which is a+ division by zero once llama.cpp normalizes it.+ """+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ cmd = _auto_tp(backend, gguf, tensor_split = list(ratio), gpu_ids = [0, 1])+ raw = _flag(cmd, "--tensor-split")+ if raw is None:+ pytest.skip("the budget check declined this ratio; formatting is not reached")++ assert raw.count(",") == len(ratio) - 1, f"{raw!r} parses as the wrong field count"+ values = [float(part) for part in raw.split(",")]+ assert sum(values) > 0, f"{raw!r} normalizes to a division by zero"+ assert _normalized(values) == pytest.approx(+ _normalized([float(x) for x in ratio]), abs = 1e-6+ ), f"{raw!r} is not the proportion that was asked for"+++# --------------------------------------------------------------------------+# 2. Ratios that must NOT be forwarded.+# --------------------------------------------------------------------------+++ "ratio, why",+ [+ ([3, 1, 1], "three weights for two cards"),+ ([3], "one weight for two cards"),+ ([0, 0], "a total of zero"),+ ([-1, -1], "negatives, which sanitize to zero"),+ ([float("nan"), 1], "a NaN weight"),+ ([float("inf"), 1], "an infinite weight"),+ ],+)+def test_a_degenerate_ratio_is_dropped_not_forwarded(tmp_path, ratio, why):+ """A bad ratio must leave llama.cpp on its own default, never reach it."""+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ cmd = _auto_tp(backend, gguf, tensor_split = list(ratio), gpu_ids = [0, 1])+ assert _flag(cmd, "--split-mode") == "tensor", why+ raw = _flag(cmd, "--tensor-split")+ if raw is not None:+ for part in raw.split(","):+ value = float(part)+ assert math.isfinite(value) and value >= 0.0, f"{raw!r} ({why})"+++def test_a_ratio_that_overshoots_one_card_is_dropped(tmp_path):+ """The planner's own budget, applied to the user's ratio. A 25GB model at+ 3:1 puts ~19GB on a 16GB card."""+ backend, gguf = _tp_backend(+ tmp_path,+ memory = [(0, 16_000, 16_000), (1, 16_000, 16_000)],+ model_gib = 25,+ )+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])+ assert _flag(cmd, "--split-mode") == "tensor"+ assert _flag(cmd, "--tensor-split") is None+++def test_the_budget_check_agrees_with_the_planner_on_an_even_share(tmp_path):+ """The check's whole claim is that it applies "the same per-device budget+ the planner uses". Then an EVEN ratio must be accepted exactly when the+ planner's own even-share rule says the load fits -- otherwise a user who+ types 1,1 is told their ratio does not fit a machine the planner just+ decided to split evenly, which is #10355 wearing a different hat.++ Sized to sit in the gap: the context compute buffer is real here, and+ charging it once (the planner's rule) accepts while charging it twice+ rejects.+ """+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ mib = 1024 * 1024+ cc_per_device_mib = 1024+ backend._compute_buffer_ctx_bytes = lambda *a, **k: cc_per_device_mib * mib++ gpus = [(0, 24_000), (1, 24_000)]+ total_by_idx = {0: 24_000, 1: 24_000}+ # usable = 24000 - max(0.03*24000, min(512, 0.03*24000)) = 23280 MiB per card,+ # reserve = 256, so the planner's ceiling is 23024 MiB a card.+ model_mib = 43_452+ even_share = (model_mib + 2 * cc_per_device_mib) / 2+ assert 22_000 < even_share < 23_024, "the case has drifted out of the gap it tests"++ assert (+ backend._tensor_split_fits_budget(+ [1.0, 1.0],+ gpus,+ [0, 1],+ model_mib * mib,+ 4096,+ n_ubatch = 512,+ total_by_idx = total_by_idx,+ vram_fraction = 0.97,+ )+ is True+ )+++def test_the_context_buffer_is_charged_flat_not_by_the_ratio(tmp_path):+ """The context-linear compute buffer is REPLICATED on every device: each one+ allocates the whole thing whatever weight it carries. Distributing the+ aggregate by the ratio instead is the same arithmetic only at an even share,+ and away from it the high-weight card is charged nearly twice the buffer --+ so a ratio the planner's own rule accepts is refused, which for a user who+ typed one is #10355 again. Sized to sit exactly in that gap: the planner+ accepts 1:9 here, the aggregate form does not.+ """+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ mib = 1024 * 1024+ cc_per_device_mib = 6 * 1024+ backend._compute_buffer_ctx_bytes = lambda *a, **k: cc_per_device_mib * mib+ model_mib = 14 * 1024++ # usable - reserve = 23024 MiB a card, as in the even-share cell above.+ ceiling_mib = 23_024+ assert model_mib / 2 + cc_per_device_mib < ceiling_mib, "the planner would not have"+ assert 0.9 * model_mib < ceiling_mib - cc_per_device_mib, "charged flat, 1:9 fits"+ assert 0.9 * (model_mib + 2 * cc_per_device_mib) > ceiling_mib, "charged by ratio, it does not"++ assert (+ backend._tensor_split_fits_budget(+ [1.0, 9.0],+ [(0, 24_000), (1, 24_000)],+ [0, 1],+ model_mib * mib,+ 4096,+ n_ubatch = 512,+ total_by_idx = {0: 24_000, 1: 24_000},+ vram_fraction = 0.97,+ )+ is True+ )+++def test_the_budget_check_still_refuses_what_does_not_fit(tmp_path):+ """The other side of the rule above: relaxing the double charge must not+ turn the check into one that accepts anything."""+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ mib = 1024 * 1024+ backend._compute_buffer_ctx_bytes = lambda *a, **k: 0++ assert (+ backend._tensor_split_fits_budget(+ [9.0, 1.0],+ [(0, 24_000), (1, 24_000)],+ [0, 1],+ 40_000 * mib,+ 4096,+ n_ubatch = 512,+ total_by_idx = {0: 24_000, 1: 24_000},+ vram_fraction = 0.97,+ )+ is False+ )+++def test_the_budget_check_fails_closed_on_an_unsurveyed_card(tmp_path):+ """gpu_indices and the tensor-parallel survey can disagree. Pricing one+ against the other used to raise KeyError out of load_model."""+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ assert (+ backend._tensor_split_fits_budget(+ [1.0, 1.0],+ [(0, 24_000)],+ [0, 1],+ 1024,+ 4096,+ total_by_idx = {0: 24_000},+ )+ is False+ )+++def test_manual_mode_is_untouched_by_the_new_branch(tmp_path):+ """Manual mode has always emitted the ratio through its own path. The new+ elif is guarded on `gpu_memory_mode != "manual"`, so this must be+ byte-identical to its pre-PR behaviour."""+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ cmd = _launch(+ backend,+ gguf,+ gpu_memory_mode = "manual",+ gpu_layers = 99,+ tensor_parallel = True,+ tensor_split = [3, 1],+ gpu_ids = [0, 1],+ n_ctx = 4096,+ )["cmd"]+ assert _flag(cmd, "--split-mode") == "tensor"+ assert _flag(cmd, "--tensor-split") == "3,1"+ assert backend._auto_tensor_split is None, "manual mode must not record an auto ratio"+++# --------------------------------------------------------------------------+# 3. Hardware shapes where tensor parallelism is not available at all.+# --------------------------------------------------------------------------+++def test_a_single_gpu_never_gets_a_split(tmp_path):+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000)])+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0])+ assert _flag(cmd, "--tensor-split") is None+++def test_no_gpu_at_all_does_not_raise(tmp_path):+ """CPU-only. The new code divides by len(gpu_indices) and by sum(split);+ neither may be reached with a zero."""+ backend, gguf = _tp_backend(tmp_path, memory = [])+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1])+ assert _flag(cmd, "--tensor-split") is None+++def test_cards_whose_driver_reports_no_total(tmp_path):+ """Some Vulkan drivers report free memory without a total. The new budget+ check reads `total_by_idx.get(idx, 0)`, so this is the shape that exercises+ the zero-total branch of _vram_usable_mib."""+ backend, gguf = _tp_backend(+ tmp_path,+ memory = [(0, 24_000, 0), (1, 24_000, 0)],+ vulkan = True,+ )+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])+ assert _flag(cmd, "--split-mode") == "tensor"+++def test_paravirtual_metal_never_launches_a_tensor_split(tmp_path, monkeypatch):+ """macOS in a VM pins to CPU and overrides --split-mode. The PR must not+ reintroduce a split there."""+ import core.inference.llama_cpp as llama_cpp++ monkeypatch.setattr(llama_cpp, "_metal_device_is_paravirtual", lambda: True)+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])+ assert _flag(cmd, "--tensor-split") is None+++# --------------------------------------------------------------------------+# 3b. The placement planner's own recovery arm.+#+# `load_model` prices the load inside one long `try`, and its `except` arm is a+# designed degradation: it drops the plan, sets `--fit on` and launches anyway.+# It resets `tp_tensor_split` to None but deliberately NOT `tensor_parallel` --+# which is exactly the state the new fallback fires in, with `gpu_indices`+# rebuilt from a WIDER set than the `tp_gpus` the planner had filtered, or not+# rebuilt at all. Three shapes, all of which launched cleanly before the PR.+# --------------------------------------------------------------------------+++def _planner_raises(backend):+ def _boom(*args, **kwargs):+ raise RuntimeError("simulated GPU selection failure")++ backend._plan_tensor_parallel = _boom+++def test_a_failed_plan_still_launches_on_vulkan(tmp_path):+ """Vulkan rebuilds gpu_indices from every DETECTED device, so it can name a+ card the planner's reserve filter had dropped."""+ backend, gguf = _tp_backend(+ tmp_path,+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 200, 24_000)],+ vulkan = True,+ )+ _planner_raises(backend)+ cmd = _auto_tp(backend, gguf, tensor_split = [2, 1, 1])+ assert cmd, "the load must still launch, as it did before the fallback existed"+++def test_a_failed_plan_still_launches_when_gpu_ids_were_pinned(tmp_path):+ """The CUDA / ROCm shape: gpu_indices comes back as sorted(gpu_ids), which+ can include a card the tensor-parallel reserve filter excluded."""+ backend, gguf = _tp_backend(+ tmp_path,+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 200, 24_000)],+ )+ _planner_raises(backend)+ cmd = _auto_tp(backend, gguf, tensor_split = [2, 1, 1], gpu_ids = [0, 1, 2])+ assert cmd+++def test_a_plan_that_fails_before_the_gpu_survey_still_launches(tmp_path):+ """The earliest throw: nothing the fallback reads has been bound yet."""+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])++ def _boom(_path):+ raise RuntimeError("simulated GGUF read failure")++ backend._get_gguf_size_bytes = _boom+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])+ assert cmd+++# --------------------------------------------------------------------------+# 4. Reload deduplication -- the half that can regress an already-working path.+# --------------------------------------------------------------------------+++def _intent(gguf, **kwargs):+ params = dict(+ gguf_path = str(gguf),+ model_identifier = "test",+ gpu_memory_mode = "auto",+ tensor_parallel = True,+ n_ctx = 4096,+ )+ params.update(kwargs)+ return GgufLoadIntent(**params)+++def _reuses(backend, gguf, **kwargs):+ """Whether a second identical /load would reuse the running server."""+ return backend.adopt_load_intent_if_matched(_intent(gguf, **kwargs))+++def test_an_identical_auto_request_with_a_ratio_reuses_the_server(tmp_path):+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])+ assert _reuses(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1]) is True+++def test_a_changed_ratio_does_not_reuse_the_server(tmp_path):+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])+ assert _reuses(backend, gguf, tensor_split = [1, 3], gpu_ids = [0, 1]) is False+++def test_an_identical_auto_request_with_no_ratio_reuses_the_server(tmp_path):+ """The plain auto tensor-parallel load: the user set no ratio at all."""+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ _auto_tp(backend, gguf, gpu_ids = [0, 1])+ assert _reuses(backend, gguf, gpu_ids = [0, 1]) is True+++def test_a_planner_weighted_split_still_reuses_the_server(tmp_path):+ """THE REGRESSION THIS FILE EXISTS FOR.++ When the model does not fit evenly the planner returns its OWN weighted+ split, in MiB, and no user ratio is involved. If the recorded auto ratio is+ the planner's output while the comparison is against the user's request,+ every repeat of an identical request mismatches and reloads a+ multi-gigabyte model -- forever, because the next launch records the same+ planner output again. This is an ordinary, already-working, pre-PR path.+ """+ backend, gguf = _tp_backend(+ tmp_path,+ memory = [(0, 24_000, 24_000), (1, 16_000, 16_000)],+ model_gib = 30,+ )+ cmd = _auto_tp(backend, gguf, gpu_ids = [0, 1])+ assert (+ _flag(cmd, "--tensor-split") is not None+ ), "this case is only meaningful when the planner emitted its own split"+ assert _reuses(backend, gguf, gpu_ids = [0, 1]) is True+++def test_a_rejected_user_ratio_still_reuses_the_server(tmp_path):+ """The other half of the same shape: the budget check declined the ratio,+ so nothing was emitted. Repeating the identical request must not reload --+ the second launch would decline it identically."""+ backend, gguf = _tp_backend(+ tmp_path,+ memory = [(0, 16_000, 16_000), (1, 16_000, 16_000)],+ model_gib = 25,+ )+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])+ assert _flag(cmd, "--tensor-split") is None+ assert _reuses(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1]) is True+++def test_an_equivalent_ratio_reuses_the_server(tmp_path):+ """3:1 and 6:2 are the same instruction to llama.cpp."""+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])+ assert _reuses(backend, gguf, tensor_split = [6, 2], gpu_ids = [0, 1]) is True+++def test_unload_clears_the_recorded_ratio(tmp_path):+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])+ assert backend._auto_tensor_split is not None+ backend.unload_model()+ assert backend._auto_tensor_split is None+++def test_a_layer_split_load_clears_the_recorded_ratio(tmp_path):+ """Turning tensor parallelism off must not leave the ratio behind: the+ next auto tensor load would compare against a server that never had it."""+ backend, gguf = _tp_backend(tmp_path, memory = [(0, 24_000, 24_000), (1, 24_000, 24_000)])+ _auto_tp(backend, gguf, tensor_split = [3, 1], gpu_ids = [0, 1])+ _launch(+ backend,+ gguf,+ gpu_memory_mode = "auto",+ tensor_parallel = False,+ gpu_ids = [0, 1],+ n_ctx = 4096,+ )+ assert backend._auto_tensor_split is None+++# --------------------------------------------------------------------------+# 6. What /status reports after a recovery rewrites the argv.+# --------------------------------------------------------------------------+++def _crash_once_with_an_arch_error(backend):+ """Make the first spawn die the way a binary with no kernels for the pinned+ card does, so the arch-crash retry runs and the second spawn succeeds."""+ calls: list[int] = []++ def _health(timeout, **_kw):+ calls.append(1)+ return len(calls) > 1++ backend._wait_for_health = _health+ backend._kernel_image_invalid = lambda _text: True+ return calls+++def test_the_arch_crash_retry_stops_reporting_the_ratio_it_dropped(tmp_path):+ """The emitted ratio is not just a record, it is what `tensor_split` reports.++ The retry re-masks the child onto a narrowed device set, so a split sized for+ the crashed one weights the wrong cards and `_without_tensor_split` takes it+ off the argv. Left recorded, /status answers with a ratio the live child does+ not have, and here the narrowing leaves ONE device, so it would answer with a+ ratio beside `tensor_parallel: false`. The third card is below the reserve, so+ the planner pins two of three and the retry has somewhere to go.+ """+ backend, gguf = _tp_backend(+ tmp_path,+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 400, 24_000)],+ )+ _crash_once_with_an_arch_error(backend)++ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1])++ assert _flag(cmd, "--tensor-split") is None, "the retry kept a split it re-indexed"+ assert backend.tensor_parallel is False+ assert backend.tensor_split is None, (+ "/status still reports the dropped ratio: " f"{backend.tensor_split}"+ )+++def test_a_launch_that_does_not_crash_still_reports_its_ratio(tmp_path):+ """The control. Clearing on the recovery arm must not clear on the ordinary+ one, or the PR's own reporting is gone."""+ backend, gguf = _tp_backend(+ tmp_path,+ memory = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 400, 24_000)],+ )+ cmd = _auto_tp(backend, gguf, tensor_split = [3, 1])+ assert _flag(cmd, "--tensor-split") == "3,1"+ assert backend.tensor_split == [0.75, 0.25]diff --git a/studio/backend/tests/test_pr10884_platform_vendor_matrix.py b/studio/backend/tests/test_pr10884_platform_vendor_matrix.pynew file mode 100644index 00000000000..6ae01c439b2--- /dev/null+++ b/studio/backend/tests/test_pr10884_platform_vendor_matrix.py@@ -0,0 +1,253 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The auto tensor-split fallback across every host Unsloth Studio supports.++``test_pr10884_auto_tensor_split_sim.py`` varies the ratio, the card count and+the planner's answer. This file varies the HOST: the Cartesian product of+{Linux, Windows, WSL, macOS} x {NVIDIA, AMD ROCm, AMD or Intel via Vulkan,+CPU only}, because the code the PR changed sits upstream of every one of them+and reads `sys.platform`, `_is_wsl` and torch's ROCm markers on the way to a+launch.++Everything is simulated on a CPU-only Linux host, and the simulation is+deliberately at the lowest level the production code reads:++* the OS is ``sys.platform`` plus ``_is_wsl``, which is what the module itself+ branches on;+* the vendor is a fake ``torch`` in ``sys.modules`` carrying (or not carrying)+ ``version.hip``, which is exactly what ``_host_torch_is_rocm`` reads, plus+ the Vulkan backend flag;+* the cards are the ``(index, free, total)`` rows ``_get_gpu_memory`` returns.++What this can and cannot say is worth stating. It proves the placement+decision, the emitted argv and the child's device mask on each host. It does+not prove that a real ROCm or Metal driver then behaves; only hardware can say+that, and the NVIDIA half of the table is the half that has also been run on+two real T4s.+"""++from __future__ import annotations++import sys+import types+from pathlib import Path++_TESTS_DIR = str(Path(__file__).resolve().parent)+if _TESTS_DIR not in sys.path:+ sys.path.insert(0, _TESTS_DIR)++import pytest # noqa: E402++from test_llama_cpp_placement import _backend, _launch # noqa: E402++import core.inference.llama_cpp as llama_cpp # noqa: E402++GIB = 1024**3++# (index, free_mib, total_mib). Two roomy cards: enough that the planner decides+# an even share is safe and returns nothing, which is the branch the PR added.+TWO_CARDS = [(0, 24_000, 24_000), (1, 24_000, 24_000)]++OSES = {+ # label: (sys.platform, _is_wsl)+ "linux": ("linux", False),+ "windows": ("win32", False),+ "wsl": ("linux", True),+ "macos": ("darwin", False),+}++VENDORS = ("nvidia", "rocm", "vulkan", "cpu")+++def _fake_torch(*, rocm: bool, device_count: int) -> types.ModuleType:+ """Enough torch for the three things the launch path asks it.++ ``_host_torch_is_rocm`` reads ``version.hip`` and ``__version__``;+ ``_effective_gpu_count`` falls back to ``cuda.device_count()`` when the+ caller pinned no ids. A real import is not wanted here: the point is to+ put the host under test, not the host running the test.+ """+ torch = types.ModuleType("torch")+ torch.__version__ = "2.11.0+rocm6.2" if rocm else "2.11.0+cu130"+ version = types.ModuleType("torch.version")+ version.hip = "6.2.0" if rocm else None+ version.cuda = None if rocm else "13.0"+ torch.version = version+ cuda = types.SimpleNamespace(+ is_available = lambda: device_count > 0,+ device_count = lambda: device_count,+ )+ torch.cuda = cuda+ return torch+++def host(monkeypatch):+ """Put the process on a named OS and vendor for the duration of one test."""++ def _apply(+ os_label: str,+ vendor: str,+ *,+ cards = TWO_CARDS,+ tmp_path = None,+ ):+ platform, is_wsl = OSES[os_label]+ monkeypatch.setattr(sys, "platform", platform)+ monkeypatch.setattr(llama_cpp, "_is_wsl", lambda: is_wsl, raising = False)+ # A real Mac is never the paravirtual case unless it is a VM; that shape+ # has its own test in the sibling file.+ monkeypatch.setattr(llama_cpp, "_metal_device_is_paravirtual", lambda: False, raising = False)+ visible = [] if vendor == "cpu" else cards+ monkeypatch.setitem(+ sys.modules,+ "torch",+ _fake_torch(rocm = vendor == "rocm", device_count = len(visible)),+ )+ backend, gguf = _backend(tmp_path, vulkan = vendor == "vulkan", memory = list(visible))+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda *a, **k: 0+ backend._compute_buffer_ctx_bytes = lambda *a, **k: 0+ backend._get_gguf_size_bytes = lambda _path: 1 * GIB+ backend._TENSOR_PARALLEL_BUFFER_RESERVE_MIB = 256+ return backend, gguf, visible++ return _apply+++def _flag(cmd, name):+ hits = [i for i, tok in enumerate(cmd) if tok == name]+ assert len(hits) <= 1, f"{name} appears {len(hits)} times"+ return cmd[hits[0] + 1] if hits else None+++def _auto_tp(backend, gguf, visible, **kwargs):+ params = dict(gpu_memory_mode = "auto", tensor_parallel = True, n_ctx = 4096)+ if visible:+ params["gpu_ids"] = [idx for idx, *_ in visible]+ params.update(kwargs)+ return _launch(backend, gguf, **params)++[email protected]("os_label", sorted(OSES))[email protected]("vendor", VENDORS)+def test_the_user_ratio_survives_every_host(tmp_path, host, os_label, vendor):+ """One statement per host: the load launches, and the ratio either reaches+ llama-server or is absent for a reason this table can name."""+ backend, gguf, visible = host(os_label, vendor, tmp_path = tmp_path)+ captured = _auto_tp(backend, gguf, visible, tensor_split = [3, 1])+ cmd = captured["cmd"]++ if vendor == "cpu":+ # Nothing to split. The mode is dropped before the emit block, and the+ # new fallback must not resurrect it.+ assert _flag(cmd, "--tensor-split") is None+ return++ assert _flag(cmd, "--split-mode") == "tensor", f"{os_label}/{vendor}"+ assert _flag(cmd, "--tensor-split") == "3,1", f"{os_label}/{vendor}"++[email protected]("os_label", sorted(OSES))[email protected]("vendor", VENDORS)+def test_a_failed_plan_still_launches_on_every_host(tmp_path, host, os_label, vendor):+ """The placement planner's except arm is a designed degradation: it drops+ the plan, sets --fit on and launches anyway. The fallback must not turn+ that into an exception on ANY host -- it was a KeyError on Vulkan and a+ TypeError on CUDA and ROCm before this was gated.++ Three cards, one of them below the tensor-parallel compute-buffer reserve,+ because that is what makes the arm's rebuilt `gpu_indices` WIDER than the+ `tp_gpus` the planner had filtered. Two equal cards never disagree, and a+ version of this test that used them passed against the unfixed revision.+ """+ cards = [(0, 24_000, 24_000), (1, 24_000, 24_000), (2, 200, 24_000)]+ backend, gguf, visible = host(os_label, vendor, cards = cards, tmp_path = tmp_path)++ def _boom(*args, **kwargs):+ raise RuntimeError("simulated GPU selection failure")++ backend._plan_tensor_parallel = _boom+ cmd = _auto_tp(backend, gguf, visible, tensor_split = [2, 1, 1])["cmd"]+ assert cmd, f"{os_label}/{vendor} did not launch"++[email protected]("os_label", sorted(OSES))[email protected]("vendor", ("nvidia", "rocm", "vulkan"))+def test_the_ratio_and_the_device_pin_agree_on_every_host(tmp_path, host, os_label, vendor):+ """A split is positional over the devices the child can SEE, so the ratio+ is meaningless unless it has one weight per pinned device. Each vendor+ pins through a different channel, and the PR does not touch any of them+ (the diff contains no mask line), so what is asserted here is the+ AGREEMENT rather than a particular spelling:++ * NVIDIA masks with ``CUDA_VISIBLE_DEVICES``;+ * ROCm on Linux and WSL masks at the ROCr layer, re-indexing the CUDA mask+ to the surviving ordinals; on Windows there is no ROCr, so it keeps the+ HIP mask instead;+ * Vulkan does not mask at all, it pins ``--device VulkanN`` on the argv.+ """+ backend, gguf, visible = host(os_label, vendor, tmp_path = tmp_path)+ captured = _auto_tp(backend, gguf, visible, tensor_split = [3, 1])+ env, cmd = captured["env"], captured["cmd"]++ weights = _flag(cmd, "--tensor-split").split(",")++ if vendor == "vulkan":+ pinned = _flag(cmd, "--device").split(",")+ assert pinned == ["Vulkan0", "Vulkan1"], f"{os_label}/{vendor}"+ elif vendor == "rocm" and sys.platform == "win32":+ pinned = env["HIP_VISIBLE_DEVICES"].split(",")+ assert "ROCR_VISIBLE_DEVICES" not in env, "ROCr does not exist on Windows"+ elif vendor == "rocm":+ pinned = env["ROCR_VISIBLE_DEVICES"].split(",")+ # With ROCr masking, HIP honours the CUDA mask, which must carry the+ # POST-mask ordinals or a non-zero pick points out of range.+ assert env["CUDA_VISIBLE_DEVICES"].split(",") == [+ str(i) for i in range(len(pinned))+ ], f"{os_label}/{vendor}"+ else:+ pinned = env["CUDA_VISIBLE_DEVICES"].split(",")++ assert (+ len(weights) == len(pinned) == 2+ ), f"{os_label}/{vendor}: {len(weights)} weights for {len(pinned)} pinned devices"++[email protected]("os_label", sorted(OSES))[email protected]("vendor", ("nvidia", "rocm", "vulkan"))+def test_an_identical_request_reuses_the_server_on_every_host(+ tmp_path, host, os_label, vendor, monkeypatch+):+ """The reload-dedupe half, per host. A fix that forwards the ratio and then+ reloads on every identical request has traded one bug for a worse one, and+ the comparison runs on every platform."""+ from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend++ backend, gguf, visible = host(os_label, vendor, tmp_path = tmp_path)+ # The matcher asks the binary about MTP through a CLASSMETHOD, so the+ # instance stub `_backend` installs does not cover it, and under a+ # simulated `sys.platform = "win32"` the stdlib's own `shutil.which` then+ # reaches into `_winapi`, which does not exist on this Linux host. That is+ # this simulation's ceiling, not the product's: the real Windows answer is+ # the windows-latest runner. Stub the probe so the axis under test is the+ # only one being measured.+ monkeypatch.setattr(+ LlamaCppBackend, "probe_server_capabilities", staticmethod(lambda *a, **k: {})+ )+ _auto_tp(backend, gguf, visible, tensor_split = [3, 1])++ def _intent(split):+ return GgufLoadIntent(+ gguf_path = str(gguf),+ model_identifier = "test",+ gpu_memory_mode = "auto",+ tensor_parallel = True,+ tensor_split = split,+ gpu_ids = [idx for idx, *_ in visible],+ n_ctx = 4096,+ )++ assert backend.adopt_load_intent_if_matched(_intent([3, 1])) is True+ assert backend.adopt_load_intent_if_matched(_intent([1, 3])) is Falsediff --git a/studio/backend/tests/test_pr10884_upgrade_compat.py b/studio/backend/tests/test_pr10884_upgrade_compat.pynew file mode 100644index 00000000000..63ea42280c0--- /dev/null+++ b/studio/backend/tests/test_pr10884_upgrade_compat.py@@ -0,0 +1,113 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""What an already-installed Unsloth Studio does on its first load after this update.++The auto tensor-split fallback changes what a load DOES with a ``tensor_split``+it is handed. The question an existing install asks is narrower and more+important: can anything already on my disk start feeding it one?++Saved per-model settings live in one row of ``app_settings``+(``openai_api_auto_switch_overrides``), and the only things that ever reach a+load from there are what ``normalize_model_override`` allow-lists on the way in+and what ``model_override_load_kwargs`` emits on the way out. So the upgrade+question is answerable exactly, without a database: put a saved override+through both and see whether a ratio can come out the other side.++Forwards compatibility is the same argument run backwards. The change persists+nothing new, so a settings row written by this version is byte-identical to one+written before it, and an older install reading it finds nothing it does not+already understand.+"""++from __future__ import annotations++import sys+from pathlib import Path++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++import pytest # noqa: E402++from utils.openai_auto_switch_settings import ( # noqa: E402+ model_override_load_kwargs,+ normalize_model_override,+)++# A saved override from an existing install, using every placement key the+# allow-list does carry. This is the shape that replays on the first load after+# an update.+LEGACY_OVERRIDE = {+ "gpu_memory_mode": "manual",+ "gpu_layers": 99,+ "n_cpu_moe": 4,+ "tensor_parallel": True,+ "gpu_ids": [0, 1],+ "gpu_index_kind": "cuda",+ "max_seq_length": 8192,+ "kv_cache_dtype": "q8_0",+ "n_parallel": 2,+ "llama_extra_args": ["--flash-attn", "on"],+}+++def test_a_saved_setting_can_never_carry_a_tensor_split():+ """The allow-list is the whole attack surface for this change. If a ratio+ cannot be stored, no existing install can begin sending one at upgrade."""+ stored = normalize_model_override(dict(LEGACY_OVERRIDE))+ assert "tensor_split" not in stored+++ "smuggled",+ [+ {"tensor_split": [3, 1]},+ {"tensor_split": None},+ {"tensorSplit": [3, 1]},+ {"tensor_split": "3,1"},+ ],+)+def test_a_ratio_written_into_the_settings_row_by_hand_is_dropped(smuggled):+ """Not hypothetical: the row is JSON in a sqlite table a user can edit, and+ a future version could try to add the key. Either way it must not reach a+ load through this door without the allow-list being changed deliberately."""+ stored = normalize_model_override({**LEGACY_OVERRIDE, **smuggled})+ assert "tensor_split" not in stored+ assert "tensorSplit" not in stored++[email protected]("is_gguf", [True, False])+def test_the_replayed_load_asks_for_no_ratio(is_gguf):+ """The other end of the same path: what the replay actually sends."""+ stored = normalize_model_override(dict(LEGACY_OVERRIDE))+ kwargs = model_override_load_kwargs(stored, is_gguf = is_gguf)+ assert "tensor_split" not in kwargs+++def test_the_replayed_load_is_unchanged_in_every_other_respect():+ """A pin against silent drift. If a later edit to this area changes what an+ existing install replays, this says so in terms a reader can check against+ the saved row above rather than against a golden blob."""+ stored = normalize_model_override(dict(LEGACY_OVERRIDE))+ kwargs = model_override_load_kwargs(stored, is_gguf = True)++ assert kwargs["gpu_memory_mode"] == "manual"+ assert kwargs["gpu_layers"] == 99+ assert kwargs["n_cpu_moe"] == 4+ assert kwargs["tensor_parallel"] is True+ assert kwargs["gpu_ids"] == [0, 1]+ assert kwargs["max_seq_length"] == 8192+ assert kwargs["cache_type_kv"] == "q8_0"+ assert kwargs["n_parallel"] == 2+ assert kwargs["llama_extra_args"] == ["--flash-attn", "on"]+++def test_an_override_from_a_newer_version_loses_nothing_an_older_one_kept():+ """Forwards compatibility. Round-tripping a stored row through the+ normalizer must be stable, or an install that upgrades and then rolls back+ finds its settings rewritten."""+ once = normalize_model_override(dict(LEGACY_OVERRIDE))+ twice = normalize_model_override(dict(once))+ assert once == twicediff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.tsindex 387f4d91b93..eda05a4fa11 100644--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts@@ -430,10 +430,27 @@ const SETTING_CHECKS: SettingCheck[] = [{// The split is placement the config cannot carry: the applier clears splitRatio, so a remembered// config asks for the default distribution while a resident manual load may run a custom one.+ //+ // Judged on the mode the RESIDENT server ran, not the one this pick would send. Since+ // unslothai/unsloth#10884 an auto tensor-parallel load reports a split of its own, chosen by+ // the planner, and the store never holds one in auto -- applyInferenceStatusToStore nulls it+ // unless the mode is manual. Comparing the two sides there compares a field the applier+ // cleared against a server legitimately running the planner's ratio, and declines to adopt a+ // resident model that is exactly what was asked for. A manual load's custom ratio is still a+ // real disagreement, and a server too old to report its mode is still compared, so nothing+ // that used to reload stops reloading.+ //+ // Only when the store is holding NO ratio, though. applyInferenceStatusToStore keeps+ // prevState.splitRatio whenever a gpu-memory edit is pending, so a ratio set under Manual+ // survives the switch to Auto, and the load path sends store.splitRatio in either mode. Since+ // this PR the backend honours that ratio in auto too, so adopting on the mode alone would drop+ // a placement change the user had made and the server would have applied.placement: true,ggufPlacement: true,pinned: () => true,- agrees: (_c, s, standing) => sameList(standing.splitRatio, s.tensor_split),+ agrees: (_c, s, standing) =>+ (s.gpu_memory_mode === "auto" && standing.splitRatio == null) ||+ sameList(standing.splitRatio, s.tensor_split),},{// A managed override the backend would reject outright. Folding it into "no override" here woulddiff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.tsindex 2d5f9c381a4..2d1314b62d8 100644--- a/studio/frontend/tests/resident-config-match.test.ts+++ b/studio/frontend/tests/resident-config-match.test.ts@@ -1791,3 +1791,64 @@ test("a runtime_error resident does not claim its draft depth is the default", ();}});++/**+ * Auto tensor-parallel now reports a split (unslothai/unsloth#10884): the backend emits+ * one whenever the planner sizes the load itself, and the /status echo carries it. The+ * store never holds a split in auto mode -- applyInferenceStatusToStore nulls it unless+ * the mode is manual -- so comparing the two sides here compares a cleared field against+ * a server that is legitimately running a ratio, and declines to adopt a resident model+ * that is exactly what was asked for. The split is a manual-mode opinion; in auto it is+ * the planner's business.+ */+test("an auto tensor-parallel server that reports a split still adopts", () => {+ assert.equal(+ matches(+ { gpu_memory_mode: "auto", tensor_parallel: true, tensor_split: [0.75, 0.25] },+ { ...BLANK, tensorParallel: true },+ ),+ true,+ );+});++/**+ * The limit of the rule above. applyInferenceStatusToStore preserves prevState.splitRatio+ * whenever a gpu-memory edit is pending, so a ratio set under Manual survives the switch+ * to Auto, and the load path sends store.splitRatio in either mode. Adopting on the+ * resident's mode alone would drop a placement change the user made and the server would+ * have applied, since the backend honours a ratio in auto now too.+ */+test("a pending ratio the auto resident is not running is still a reload", () => {+ assert.equal(+ matches(+ { gpu_memory_mode: "auto", tensor_parallel: true, tensor_split: [0.75, 0.25] },+ { ...BLANK, tensorParallel: true },+ { ...STANDING, splitRatio: [0.5, 0.5] },+ ),+ false,+ );+});++test("a pending ratio the auto resident IS running adopts", () => {+ // The other half of the guard: it must not turn into a blanket reload for anyone who+ // ever touched the ratio.+ assert.equal(+ matches(+ { gpu_memory_mode: "auto", tensor_parallel: true, tensor_split: [0.75, 0.25] },+ { ...BLANK, tensorParallel: true },+ { ...STANDING, splitRatio: [0.75, 0.25] },+ ),+ true,+ );+});++test("a remembered manual split the resident load does not run is still a reload", () => {+ assert.equal(+ matches(+ { gpu_memory_mode: "manual", tensor_split: [0.5, 0.5] },+ { ...BLANK, gpuMemoryMode: "manual" as const, gpuLayers: 99 },+ { ...STANDING, splitRatio: [0.75, 0.25] },+ ),+ false,+ );+});diff --git a/tests/kaggle/studio_gpu/run_studio_gpu.py b/tests/kaggle/studio_gpu/run_studio_gpu.pyindex 84686e099aa..45dd0190801 100644--- a/tests/kaggle/studio_gpu/run_studio_gpu.py+++ b/tests/kaggle/studio_gpu/run_studio_gpu.py@@ -406,6 +406,75 @@ def llama_server_pids() -> list[int]:return pids+def llama_server_argvs() -> dict[int, list[str]]:+ """Full argv of every llama-server child, split on NUL.++ The sibling above throws away everything after argv[0], and for the flags+ this payload asserts on that is the whole answer. It has to come from the+ process table: ``GET /api/inference/status`` carries ``tensor_parallel``+ but its ``tensor_split`` field is documented as the MANUAL-mode ratio and+ is never written on the auto path, so an auto load reports+ ``tensor_split: null`` whatever is on the child's command line.++ Split on ``\\x00``, never on whitespace: a model path with a space in it+ would otherwise become two arguments and shift every index after it.+ """+ out: dict[int, list[str]] = {}+ proc_root = Path("/proc")+ if not proc_root.is_dir():+ return out+ for entry in proc_root.iterdir():+ if not entry.name.isdigit():+ continue+ try:+ raw = (entry / "cmdline").read_bytes().decode("utf-8", errors = "replace")+ except OSError:+ continue+ argv = [part for part in raw.split("\x00") if part]+ if not argv:+ continue+ if "llama-server" in Path(argv[0]).name:+ out[int(entry.name)] = argv+ return out+++def argv_flag(argv: list[str], name: str) -> str | None:+ """The value of ``name`` in an argv, accepting ``--flag v`` and ``--flag=v``.++ Returns None when absent. Raises on a DUPLICATE, because two spellings of+ the same flag mean the last one wins and an assertion that read the first+ would be reporting on a value the server is not using.+ """+ hits: list[str] = []+ for i, token in enumerate(argv):+ if token == name:+ if i + 1 < len(argv):+ hits.append(argv[i + 1])+ elif token.startswith(name + "="):+ hits.append(token.split("=", 1)[1])+ if len(hits) > 1:+ raise ValueError(f"{name} appears {len(hits)} times: {hits}")+ return hits[0] if hits else None+++def split_ratio(raw: str | None) -> tuple[float, ...] | None:+ """A ``--tensor-split`` value as a normalized proportion.++ llama.cpp normalizes the list, so ``3,1`` and ``75,25`` are the same+ instruction and must compare equal here.+ """+ if raw is None:+ return None+ try:+ values = [float(part) for part in raw.split(",")]+ except ValueError:+ return None+ total = sum(values)+ if total <= 0:+ return None+ return tuple(round(v / total, 6) for v in values)++class Payload:def __init__(self, args: argparse.Namespace) -> None:self.args = args@@ -1039,6 +1108,325 @@ class Payload:detail["failures"] = failuresreturn self.record("server_flags", not failures, detail)+ # ------------------------------------------- auto tensor-parallel split++ _TP_FILE = "studio/backend/core/inference/llama_cpp.py"++ def _auto_tp_load(self, ratio: list[float] | None, *, force: bool) -> dict:+ """One auto-mode tensor-parallel load, and what the child was launched with.++ ``gpu_memory_mode: "auto"`` with ``tensor_parallel: true`` is the exact+ shape unslothai/unsloth#10355 is about: on two cards of the same size+ with a model that fits, the planner decides an even share is safe and+ returns no ratio of its own, and the user's ratio is the only one there+ is. ``force_reload`` is the real field name -- ``force`` is not on+ ``LoadRequest`` and pydantic drops it -- so passing it is how this+ drives a genuine relaunch, and omitting it is how the deduplication+ half of the check gets a real answer.+ """+ body: dict = {+ "model_path": self.args.chat_model,+ "is_lora": False,+ "max_seq_length": self.args.studio_ctx,+ "gpu_memory_mode": "auto",+ "tensor_parallel": True,+ # f16, not a quantized cache: a quantized KV on a llama.cpp older+ # than b9455 makes the backend drop tensor mode outright, and this+ # check would then be measuring that drop rather than the split.+ "cache_type_kv": "f16",+ "force_reload": force,+ }+ if ratio is not None:+ body["tensor_split"] = list(ratio)+ if self.args.chat_variant:+ body["gguf_variant"] = self.args.chat_variant+ cards = gpu_inventory()+ if len(cards) >= 2:+ body["gpu_ids"] = list(range(len(cards)))++ before = llama_server_argvs()+ self.studio.expect("POST", "/api/inference/load", body, timeout = self.args.load_timeout)+ code, status = self.studio.get("/api/inference/status")+ argvs = llama_server_argvs()+ # The child that is serving NOW. A relaunch gets a new pid, so taking+ # the newest one is what makes the reload half of this check readable;+ # with exactly one server there is nothing to choose between.+ pid = None+ if argvs:+ fresh = [p for p in argvs if p not in before] or list(argvs)+ pid = max(fresh)+ argv = argvs.get(pid, []) if pid is not None else []+ try:+ split_mode = argv_flag(argv, "--split-mode")+ tensor_split = argv_flag(argv, "--tensor-split")+ duplicate = None+ except ValueError as exc:+ split_mode = tensor_split = None+ duplicate = str(exc)+ return {+ "requested_ratio": list(ratio) if ratio is not None else None,+ "status_code": code,+ "status_tensor_parallel": (status or {}).get("tensor_parallel")+ if isinstance(status, dict)+ else None,+ # What /status SAYS the split is, alongside what the child was+ # actually launched with. These two disagreed for the whole life of+ # the auto path -- the property returned the manual-mode field,+ # which auto never writes -- so a client had no way to see the+ # ratio at all. Reported as its own key so the report shows the+ # pair rather than one of them.+ "status_tensor_split": (status or {}).get("tensor_split")+ if isinstance(status, dict)+ else None,+ "llama_server_pid": pid,+ "split_mode": split_mode,+ "tensor_split": tensor_split,+ "tensor_split_ratio": split_ratio(tensor_split),+ "duplicate_flag": duplicate,+ "cards_visible": len(cards),+ "compute_apps": nvidia_compute_apps(),+ }++ def _checkout_tp_file(self, ref: str) -> tuple[bool, str]:+ """Put ONE file at ``ref``. The two legs then differ by the PR, and by+ nothing else: same install, same binary, same weights, same session."""+ fetch = run(+ ["git", "-C", str(self.repo_root), "fetch", "--depth", "1", "origin", ref],+ timeout = 600,+ )+ checkout = run(+ ["git", "-C", str(self.repo_root), "checkout", ref, "--", self._TP_FILE],+ timeout = 120,+ )+ blob = run(+ ["git", "-C", str(self.repo_root), "hash-object", self._TP_FILE],+ timeout = 120,+ )+ ok = checkout.returncode == 0+ return ok, (+ blob.stdout or ""+ ).strip() or f"checkout rc={checkout.returncode} " f"fetch rc={fetch.returncode}"++ def _restart_for_leg(self, label: str) -> bool:+ """Stop Unsloth and bring it back on the code now on disk.++ Restarting is the whole point: ``install.sh --local`` overlays the+ checkout as an EDITABLE install, so the file swapped above is the file+ the next process imports -- but only the next one. A leg that reused+ the running server would be measuring the other revision.+ """+ self.stop_server()+ for _ in range(60):+ if not llama_server_pids():+ break+ time.sleep(1)+ if not self.start_server():+ log(f"{label}: Unsloth did not come back up")+ return False+ return self.authenticate()++ def assert_auto_tensor_split(self) -> bool:+ """The user's per-GPU ratio must reach llama-server in AUTO mode.++ Three things are checked, and the second is the one nothing else here+ can see:++ 1. tensor mode is really on, per ``/api/inference/status``;+ 2. ``--tensor-split`` is on the LIVE child's argv, carrying the+ proportion that was asked for. Status cannot answer this: its+ ``tensor_split`` field is the manual-mode ratio and is null on this+ path whatever the child was launched with;+ 3. the server still serves, with memory on both cards.++ Then deduplication, without ``force_reload``, because a fix that+ forwards the ratio and then reloads on every identical request has+ traded one bug for a worse one: an identical repeat must REUSE the+ server, and a changed ratio must replace it.+ """+ failures: list[str] = []+ cards = gpu_inventory()+ detail: dict = {"cards_visible": len(cards), "legs": {}}+ if len(cards) < 2:+ detail["failures"] = [+ f"only {len(cards)} card visible, so auto tensor-parallel placement "+ f"was NOT exercised; this proves nothing about "+ f"unslothai/unsloth#10355"+ ]+ detail["void"] = True+ return self.record("auto_tensor_split", False, detail)++ ratio = [float(x) for x in self.args.auto_split_ratio.split(",")]+ other = list(reversed(ratio))++ first = self._auto_tp_load(ratio, force = True)+ detail["legs"]["requested"] = first+ if first.get("duplicate_flag"):+ failures.append(f"the launch argv is ambiguous: {first['duplicate_flag']}")+ if first.get("status_tensor_parallel") is not True:+ failures.append(+ f"status reports tensor_parallel={first.get('status_tensor_parallel')!r} "+ f"after an auto tensor-parallel load, so the mode was dropped and the "+ f"split has nothing to be applied to"+ )+ if first.get("split_mode") != "tensor":+ failures.append(f"the live llama-server has --split-mode {first.get('split_mode')!r}")+ if first.get("tensor_split") is None:+ failures.append(+ f"the live llama-server has NO --tensor-split after a load that asked "+ f"for {ratio}. This is unslothai/unsloth#10355: auto mode emitted "+ f"--split-mode tensor and dropped the ratio"+ )+ elif first.get("tensor_split_ratio") != split_ratio(",".join(str(x) for x in ratio)):+ failures.append(+ f"--tensor-split is {first.get('tensor_split')!r}, which is not the "+ f"proportion {ratio} that was asked for"+ )+ else:+ # And the API has to be able to SAY so. The argv is ground truth,+ # but a user cannot read /proc; for the whole life of the auto path+ # the status property returned the manual-mode field, which auto+ # never writes, so a server running 3,1 reported null and there was+ # no way to tell a forwarded ratio from a dropped one from outside.+ reported = first.get("status_tensor_split")+ if reported is None:+ failures.append(+ f"the live llama-server runs --tensor-split "+ f"{first.get('tensor_split')!r} and /api/inference/status reports "+ f"tensor_split: null, so the applied ratio is invisible to every "+ f"client"+ )+ elif split_ratio(",".join(str(x) for x in reported)) != first.get("tensor_split_ratio"):+ failures.append(+ f"/api/inference/status reports tensor_split {reported!r} while the "+ f"live llama-server runs {first.get('tensor_split')!r}"+ )++ # It has to still serve. A split that loaded and cannot decode is not a+ # fix, and a CPU fallback answers a chat request just as happily.+ try:+ completion = self.studio.expect(+ "POST",+ "/v1/chat/completions",+ {+ "model": self.args.chat_model,+ "messages": [{"role": "user", "content": "Say OK."}],+ # Generous on purpose. A reasoning model spends its budget on+ # the thinking block first, so a tight cap comes back with an+ # empty `content` and finish_reason "length" -- a server that+ # decoded perfectly well, scored as a failure. Measured on+ # unsloth-t4-ci-d15ea193 with Qwen3-0.6B at max_tokens 16.+ "max_tokens": 256,+ "stream": False,+ },+ timeout = self.args.chat_timeout,+ )+ choice = (completion.get("choices") or [{}])[0]+ message = choice.get("message") or {}+ # Either channel counts as decoding: `reasoning_content` is where a+ # thinking model's tokens land, and tokens are tokens.+ text = (message.get("content") or "") + (message.get("reasoning_content") or "")+ usage = completion.get("usage") or {}+ detail["completion_chars"] = len(text)+ detail["completion_tokens"] = usage.get("completion_tokens")+ detail["finish_reason"] = choice.get("finish_reason")+ if not text.strip() and not usage.get("completion_tokens"):+ failures.append(+ f"the split server decoded nothing: finish_reason="+ f"{choice.get('finish_reason')!r}, usage={usage}"+ )+ except StudioError as exc:+ failures.append(f"the split server could not serve a completion: {exc}"[:600])++ apps = first.get("compute_apps")+ pid = first.get("llama_server_pid")+ if isinstance(apps, dict) and pid is not None and pid not in apps:+ # Evidence, not a verdict: nvidia-smi's per-process view is empty+ # inside some containers, and an absent row is not a CPU fallback.+ detail["note_compute_apps"] = (+ f"nvidia-smi lists no compute app for pid {pid}; GPU residency is "+ f"unproven here, see the gpu_inference assertion"+ )++ # ---- deduplication, the half a fix can regress.+ repeat = self._auto_tp_load(ratio, force = False)+ detail["legs"]["repeat_same_ratio"] = repeat+ if repeat.get("llama_server_pid") != first.get("llama_server_pid"):+ failures.append(+ f"an IDENTICAL auto request replaced the server "+ f"(pid {first.get('llama_server_pid')} -> "+ f"{repeat.get('llama_server_pid')}). Nothing changed, so this "+ f"reloads the model on every request"+ )++ changed = self._auto_tp_load(other, force = False)+ detail["legs"]["changed_ratio"] = changed+ if changed.get("llama_server_pid") == first.get("llama_server_pid"):+ failures.append(+ f"asking for {other} reused the server launched for {ratio}, so the "+ f"new ratio was silently ignored"+ )+ elif changed.get("tensor_split_ratio") != split_ratio(",".join(str(x) for x in other)):+ failures.append(+ f"after changing the ratio to {other} the live argv carries "+ f"{changed.get('tensor_split')!r}"+ )++ detail["failures"] = failures+ return self.record("auto_tensor_split", not failures, detail)++ def assert_auto_tensor_split_baseline(self) -> bool:+ """Run the same probe against the merge-base, first.++ This is what makes the head result mean anything. If the base revision+ ALSO emits the ratio then the defect is not there, the two legs prove+ nothing about the PR, and the run is void rather than green -- so the+ rule below is written as "the base must reproduce", not as a pass.+ """+ ok, blob = self._checkout_tp_file(self.args.base_sha)+ detail: dict = {"base_sha": self.args.base_sha, "blob": blob}+ if not ok:+ detail["failures"] = [f"could not put {self._TP_FILE} at the base revision: {blob}"]+ detail["void"] = True+ return self.record("auto_tensor_split_baseline", False, detail)+ if not self._restart_for_leg("baseline"):+ detail["failures"] = ["Unsloth did not restart on the base revision"]+ detail["void"] = True+ return self.record("auto_tensor_split_baseline", False, detail)++ ratio = [float(x) for x in self.args.auto_split_ratio.split(",")]+ probe = self._auto_tp_load(ratio, force = True)+ detail["probe"] = probe+ failures: list[str] = []+ if probe.get("split_mode") != "tensor":+ failures.append(+ f"the base revision did not enter tensor mode at all "+ f"(--split-mode {probe.get('split_mode')!r}), so the two legs do not "+ f"differ by the PR and this comparison is void"+ )+ detail["void"] = True+ elif probe.get("tensor_split") is not None:+ failures.append(+ f"the base revision ALREADY emits --tensor-split "+ f"{probe.get('tensor_split')!r}, so #10355 does not reproduce on this "+ f"hardware and the head result proves nothing"+ )+ detail["void"] = True+ detail["bug_reproduced"] = not failures+ detail["failures"] = failures+ return self.record("auto_tensor_split_baseline", not failures, detail)++ def restore_head_revision(self) -> bool:+ ok, blob = self._checkout_tp_file(self.args.head_sha or "HEAD")+ self.record(+ "auto_tensor_split_head_restored",+ ok,+ {+ "blob": blob,+ "failures": [] if ok else [f"could not restore {self._TP_FILE}: {blob}"],+ },+ )+ return ok and self._restart_for_leg("head")+def assert_compaction(self) -> bool:"""A conversation past the window must COMPACT, and a short one must not.@@ -2640,6 +3028,55 @@ class Payload:# ------------------------------------------------------------------ main+ def execute_legs(self) -> int:+ """Run exactly the assertions ``--legs`` names, in that order.++ The standard run is one shape: install, boot, then everything. A GPU+ session has a ceiling and a flag-level regression does not need a+ training run to be answered, so this exists to spend the session on the+ question being asked. It names the assertions rather than accepting an+ arbitrary callable, so a typo is a refusal here and not an empty pass+ forty minutes into a rented session.+ """+ available = {+ "auto_tensor_split": self.assert_auto_tensor_split,+ "gpu_inference": self.assert_gpu_inference,+ "tool_calling": self.assert_tool_calling,+ "code_execution": self.assert_code_execution,+ "web_search": self.assert_web_search,+ "server_flags": self.assert_server_flags,+ "compaction": self.assert_compaction,+ "api_key": self.assert_api_key,+ "tabs": self.assert_tabs,+ }+ wanted = [name.strip() for name in self.args.legs.split(",") if name.strip()]+ unknown = [name for name in wanted if name not in available]+ if unknown:+ self.record(+ "legs",+ False,+ {+ "requested": wanted,+ "failures": [+ f"unknown assertion(s) {unknown}; this run measured nothing. "+ f"Known: {sorted(available)}"+ ],+ },+ )+ return self.finish()++ # The baseline leg runs FIRST and on its own revision, so the head leg+ # below is the same session, the same binary and the same weights with+ # one file changed.+ if self.args.base_sha and "auto_tensor_split" in wanted:+ self.assert_auto_tensor_split_baseline()+ if not self.restore_head_revision():+ return self.finish()++ for name in wanted:+ available[name]()+ return self.finish()+def execute(self) -> int:if not self.preflight():return self.finish()@@ -2654,6 +3091,11 @@ class Payload:if not self.authenticate():return self.finish()+ # A focused run: the named assertions, in the order named, and nothing+ # else. The standard set below is unchanged when --legs is empty.+ if self.args.legs:+ return self.execute_legs()+# Before the GPU work: it needs nothing but a logged-in session, and# putting it after a 20-minute training run would mean a training# failure hides whether API keys work at all.@@ -2868,6 +3310,35 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:action = "store_true",help = "do not drive playwright_chat_ui.py (for debugging the API assertions alone)",)+ # Auto tensor-parallel split (unslothai/unsloth#10355, PR #10884).+ ap.add_argument(+ "--legs",+ default = "",+ help = "comma-separated assertion names to run, in the order listed here; "+ "empty runs the standard set. A focused run of one flag on real "+ "multi-GPU hardware does not need a 20-minute training leg to be "+ "meaningful, and a Kaggle session has a ceiling.",+ )+ ap.add_argument(+ "--auto-split-ratio",+ default = "3,1",+ help = "the per-GPU ratio the auto tensor-parallel check asks for. Skewed "+ "on purpose: an even ratio is what the planner emits anyway, so it "+ "cannot tell a forwarded ratio from a default one.",+ )+ ap.add_argument(+ "--base-sha",+ default = "",+ help = "run the auto tensor-parallel check against this revision FIRST, with "+ "only studio/backend/core/inference/llama_cpp.py swapped, so the two "+ "legs differ by the change under test and by nothing else. The base "+ "leg must reproduce the defect or the comparison is void.",+ )+ ap.add_argument(+ "--head-sha",+ default = "",+ help = "the revision to restore after a --base-sha leg (default: HEAD).",+ )return ap.parse_args(argv)