Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: catch the tensor split abort behind a gdb backtrace (#12293)

oobaboogad34cccc9ea7a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 295a48fcb56..f07587bc40b 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -29722,8 +29722,9 @@ class LlamaCppBackend:
"\n".join(self._stdout_lines[-200:])
):
_sched_abort_seen = True
+ # Scan the full buffer: gdb output can bury the assert.
_tensor_capability_crash = self._is_tensor_split_assert(
- _startup_output
+ "\n".join(self._stdout_lines)
) or self._is_tensor_quant_kv_unsupported(_startup_output)
_hip_rocr_mismatch = self._is_bundled_hip_rocr_mismatch(_startup_output)
# No fit retry reaches it, and the rung below needs this
@@ -30287,10 +30288,16 @@ class LlamaCppBackend:
_ts_out = "\n".join(self._stdout_lines[-50:])
_proc_snap2 = self._process # snapshot: re-reading races the teardown
_ts_rc = _proc_snap2.poll() if _proc_snap2 is not None else None
- if self._should_record_tensor_split_abort(_ts_rc, _ts_out):
- LlamaCppBackend._record_tensor_split_abort(
- binary, model_identifier, _planned_cache_pair
- )
+ if self._should_record_tensor_split_abort(
+ _ts_rc, "\n".join(self._stdout_lines)
+ ):
+ # A drafter crash must not disable tensor split for drafterless loads.
+ if not _extra_args_mtp_draft_path(
+ _last_spawn_cmd, env = _child_spec_env(extra_args)
+ ):
+ LlamaCppBackend._record_tensor_split_abort(
+ binary, model_identifier, _planned_cache_pair
+ )
self._kill_process()
raise RuntimeError(
"llama-server aborted on --split-mode tensor "
diff --git a/studio/backend/tests/test_tensor_split_abort_detection.py b/studio/backend/tests/test_tensor_split_abort_detection.py
new file mode 100644
index 00000000000..0f43b2c780e
--- /dev/null
+++ b/studio/backend/tests/test_tensor_split_abort_detection.py
@@ -0,0 +1,155 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Detect buried tensor-split asserts without latching drafter crashes."""
+
+from __future__ import annotations
+
+import importlib.util
+import subprocess
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+_TESTS_DIR = Path(__file__).resolve().parent
+_BACKEND_DIR = str(_TESTS_DIR.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference.llama_cpp import GgufLoadIntent, LlamaCppBackend # noqa: E402
+
+
+def _load(module_name: str, file_name: str):
+ spec = importlib.util.spec_from_file_location(module_name, _TESTS_DIR / file_name)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+_placement = _load("_placement_harness_split_abort", "test_llama_cpp_placement.py")
+
+_REAL_POPEN = subprocess.Popen
+_TWO_GPUS = [(0, 20_000, 24_000), (1, 20_000, 24_000)]
+
+# Reproduce #11308: gdb output pushes the assert beyond the last 50 lines.
+_GDB_ABORT = (
+ [
+ "common_speculative_impl_draft_dflash: - n_max=3, n_min=0, p_min=0.00\n",
+ "ggml/src/ggml-backend-meta.cpp:543: "
+ "GGML_ASSERT(src_ss[0].axis != GGML_BACKEND_SPLIT_AXIS_0) failed\n",
+ ]
+ + [f"[New LWP {400000 + i}]\n" for i in range(130)]
+ + [
+ "[Thread debugging using libthread_db enabled]\n",
+ "#1 0x... in ggml_print_backtrace () from libggml-base.so.0\n",
+ "#2 0x... in ggml_abort () from libggml-base.so.0\n",
+ ]
+)
+
+
+def crashing(tmp_path, monkeypatch):
+ for name in ("CUDA_VISIBLE_DEVICES", "LLAMA_ARG_SPLIT_MODE", "LLAMA_ARG_SPEC_TYPE"):
+ monkeypatch.delenv(name, raising = False)
+ LlamaCppBackend._tensor_split_abort_keys.clear()
+ backend, gguf = _placement._backend(tmp_path, vulkan = False, memory = list(_TWO_GPUS))
+
+ def crashed(timeout, **_kw):
+ # Drain output on exit, as the real health wait does.
+ if backend._stdout_thread is not None:
+ backend._stdout_thread.join(timeout = 2)
+ return False
+
+ backend._wait_for_health = crashed
+ spawns: list[list[str]] = []
+
+ def fake_popen(cmd, **kwargs):
+ if not cmd or str(cmd[0]) != "/fake/llama-server":
+ return _REAL_POPEN(cmd, **kwargs)
+ spawns.append([str(c) for c in cmd])
+ return type(
+ "Process",
+ (),
+ {
+ "pid": 123,
+ "returncode": -6,
+ "stdout": iter(_GDB_ABORT),
+ "poll": lambda self: -6,
+ "terminate": lambda self: None,
+ "wait": lambda self, timeout = None: -6,
+ "kill": lambda self: None,
+ },
+ )()
+
+ def load(
+ extra_args = None,
+ speculative_type = "off",
+ dflash_draft_path = None,
+ ):
+ with (
+ patch.object(subprocess, "Popen", side_effect = fake_popen),
+ pytest.raises(RuntimeError) as err,
+ ):
+ backend.load_model(
+ GgufLoadIntent(
+ gguf_path = str(gguf),
+ model_identifier = "test",
+ tensor_parallel = True,
+ speculative_type = speculative_type,
+ dflash_draft_path = dflash_draft_path,
+ extra_args = tuple(extra_args) if extra_args else None,
+ )
+ )
+ return str(err.value)
+
+ yield backend, load, spawns, tmp_path
+ LlamaCppBackend._tensor_split_abort_keys.clear()
+
+
+def _latched(backend) -> bool:
+ return LlamaCppBackend._tensor_split_aborts(
+ backend._find_llama_server_binary(), "test", ("f16", "f16")
+ )
+
+
+def test_assert_behind_a_gdb_dump_raises_to_layer_split_on_the_first_spawn(crashing):
+ backend, load, spawns, _tmp = crashing
+
+ message = load()
+
+ assert "split-axis geometry" in message
+ assert len(spawns) == 1, "the --fit and flash-attn retries cannot fix a split-axis abort"
+ assert "--split-mode" in spawns[0] and "tensor" in spawns[0]
+ assert _latched(backend)
+
+
[email protected]("source", ["auto_sidecar", "extra_args", "environment"])
+def test_abort_with_a_separate_drafter_is_not_latched_for_the_model(crashing, monkeypatch, source):
+ backend, load, spawns, tmp_path = crashing
+ drafter = tmp_path / "dflash-test-Q4_K_M.gguf"
+ drafter.write_bytes(b"GGUF")
+
+ if source == "auto_sidecar":
+ # Let Auto select DFlash with the fake binary.
+ probe = backend.probe_server_capabilities
+ backend.probe_server_capabilities = lambda *a, **kw: {
+ **probe(*a, **kw),
+ "supports_dflash": True,
+ }
+ message = load(speculative_type = "auto", dflash_draft_path = str(drafter))
+ elif source == "extra_args":
+ message = load(["--model-draft", str(drafter), "--spec-type", "draft-dflash"])
+ else:
+ # Explicit --spec-type preserves the drafter environment variable.
+ monkeypatch.setenv("LLAMA_ARG_SPEC_DRAFT_MODEL", str(drafter))
+ message = load(["--spec-type", "draft-dflash"])
+
+ assert "split-axis geometry" in message
+ assert len(spawns) == 1
+ if source == "environment":
+ assert "--model-draft" not in spawns[0]
+ else:
+ assert spawns[0][spawns[0].index("--model-draft") + 1] == str(drafter)
+ assert not _latched(backend), "a drafter's abort must not keep tensor off for other loads"
diff --git a/studio/backend/tests/test_tp_vision_regression.py b/studio/backend/tests/test_tp_vision_regression.py
index b736866f810..a1e8d4251c6 100644
--- a/studio/backend/tests/test_tp_vision_regression.py
+++ b/studio/backend/tests/test_tp_vision_regression.py
@@ -400,7 +400,7 @@ def test_tensor_split_abort_raises_early_to_layer_fallback():
f"retry runs after the projector has already been discarded (#6659)"
)
# gated on the marker-plus-crash helper, which also drives the record just above
- guard = src[max(0, raise_idx - 600) : raise_idx]
+ guard = src[max(0, raise_idx - 900) : raise_idx]
assert "_should_record_tensor_split_abort" in guard
rec_idx = src.find("LlamaCppBackend._record_tensor_split_abort(")
assert rec_idx != -1 and rec_idx < raise_idx