Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: sample Vulkan drafts on the CPU when the vocab misaligns the logits rows (#15258)
Daniel Hanf1fbcf8e456f
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex c9988583550..9320f736dd0 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -6736,6 +6736,27 @@ def _extra_args_mtp_draft_path(_HF_DRAFT_FLAGS = frozenset({"--spec-draft-hf", "-hfd", "-hfrd", "--hf-repo-draft"})_LOCAL_DRAFT_FLAGS = frozenset({"--model-draft", "--spec-draft-model", "-md"})+# Vulkan TOP_K cannot bind a misaligned view, and backend draft sampling reads logits row r at+# r * n_vocab * 4 bytes (#14987). n_vocab % 64 == 0 keeps rows 256-byte aligned, the spec maximum.+_VULKAN_ALIGNED_VOCAB_MULTIPLE = 64+_NO_DRAFT_BACKEND_SAMPLING_FLAG = "--no-spec-draft-backend-sampling"+++def _draft_backend_sampling_off(+ spec_flags: list[str], caps: Mapping[str, object], vocab_size: Optional[int], vulkan: bool+) -> list[str]:+ if not vulkan or "--spec-type" not in spec_flags:+ return []+ i = spec_flags.index("--spec-type")+ spec_types = str(spec_flags[i + 1]).split(",") if i + 1 < len(spec_flags) else []+ if all(t == "none" or t.startswith("ngram") for t in spec_types):+ return []+ if vocab_size and vocab_size % _VULKAN_ALIGNED_VOCAB_MULTIPLE == 0:+ return []+ if _NO_DRAFT_BACKEND_SAMPLING_FLAG not in (caps.get("flags") or {}):+ return []+ return [_NO_DRAFT_BACKEND_SAMPLING_FLAG]+def _extra_args_mtp_draft_source(extra_args: Optional[Iterable[str]], env: Optional[Mapping[str, str]] = None@@ -32436,6 +32457,16 @@ class LlamaCppBackend:"llama-server has no draft KV cache flags; skipping the requested %s.",_draft_cache_type,)+ _no_backend_draft = _draft_backend_sampling_off(+ spec_flags, server_caps, self._vocab_size, self._is_vulkan_backend(binary)+ )+ if _no_backend_draft:+ logger.info(+ "Vulkan: sampling drafts on the CPU, the %s-token vocab misaligns "+ "the logits rows the backend sampler reads.",+ self._vocab_size or "unknown",+ )+ spec_flags.extend(_no_backend_draft)# Remember where the spec block sits so a drafter-load failure# can be retried with these flags swapped out (see below)._spec_start = len(cmd)diff --git a/studio/backend/tests/test_vulkan_draft_backend_sampling.py b/studio/backend/tests/test_vulkan_draft_backend_sampling.pynew file mode 100644index 00000000000..67de2eebd2b--- /dev/null+++ b/studio/backend/tests/test_vulkan_draft_backend_sampling.py@@ -0,0 +1,209 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Vulkan drafts sample on the CPU when the vocab misaligns the logits rows (#14987)."""++from __future__ import annotations++import struct+import subprocess+import sys+import types as _types+from pathlib import Path+from unittest.mock import patch++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++_loggers_stub = _types.ModuleType("loggers")+_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)+sys.modules.setdefault("loggers", _loggers_stub)+_structlog_stub = _types.ModuleType("structlog")+_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("structlog")+sys.modules.setdefault("structlog", _structlog_stub)+if not hasattr(sys.modules["structlog"], "get_logger"):+ sys.modules["structlog"].get_logger = _structlog_stub.get_logger++from core.inference.llama_cpp import ( # noqa: E402+ GgufLoadIntent,+ LlamaCppBackend,+ _draft_backend_sampling_off,+)++_FLAG = "--no-spec-draft-backend-sampling"+_CAPS = {"flags": {_FLAG: "", "--spec-draft-backend-sampling": ""}}+_DSPARK = [+ "--model-draft",+ "/m/dspark.gguf",+ "--spec-type",+ "draft-dspark",+ "--spec-draft-n-max",+ "3",+]+_REAL_POPEN = subprocess.Popen++_MINICPM_V47_VOCAB = 248094+++ "spec_flags",+ [+ _DSPARK,+ [+ "--model-draft",+ "/m/dflash.gguf",+ "--spec-type",+ "draft-dflash",+ "--spec-draft-n-max",+ "3",+ ],+ ["--spec-type", "mtp", "--spec-draft-n-max", "2"],+ ["--spec-type", "draft-mtp,ngram-mod", "--spec-draft-n-max", "2"],+ ],+)+def test_misaligned_vocab_on_vulkan_samples_drafts_on_the_cpu(spec_flags):+ assert _draft_backend_sampling_off(spec_flags, _CAPS, _MINICPM_V47_VOCAB, True) == [_FLAG]+++def test_unknown_vocab_on_vulkan_takes_the_safe_side():+ assert _draft_backend_sampling_off(_DSPARK, _CAPS, None, True) == [_FLAG]++[email protected]("vocab", [151936, 248320, 262144, 128256, 201088])+def test_aligned_vocabs_keep_backend_sampling(vocab):+ assert _draft_backend_sampling_off(_DSPARK, _CAPS, vocab, True) == []+++def test_other_backends_are_untouched():+ assert _draft_backend_sampling_off(_DSPARK, _CAPS, _MINICPM_V47_VOCAB, False) == []+++ "spec_flags",+ [[], ["--spec-default"], ["--spec-type", "ngram-mod", "--spec-ngram-mod-n-max", "8"]],+)+def test_no_draft_model_means_no_flag(spec_flags):+ assert _draft_backend_sampling_off(spec_flags, _CAPS, _MINICPM_V47_VOCAB, True) == []+++def test_a_build_without_the_flag_is_not_handed_it():+ # llama-server exits on an argument it does not know, taking the load with it.+ assert _draft_backend_sampling_off(_DSPARK, {"flags": {}}, _MINICPM_V47_VOCAB, True) == []+ assert _draft_backend_sampling_off(_DSPARK, {}, _MINICPM_V47_VOCAB, True) == []+++def _write_gguf(path: Path) -> Path:+ def string(value: str) -> bytes:+ data = value.encode()+ return struct.pack("<Q", len(data)) + data++ metadata = string("general.architecture") + struct.pack("<I", 8) + string("qwen35")+ path.write_bytes(struct.pack("<IIQQ", 0x46554747, 3, 0, 1) + metadata)+ return path+++def _launch(+ tmp_path,+ monkeypatch,+ *,+ vulkan,+ vocab,+ extra_args = None,+):+ caps = dict(LlamaCppBackend.probe_server_capabilities("/nonexistent/llama-server"))+ caps.update(+ found = True,+ supports_dspark = True,+ mtp_probe_inconclusive = False,+ spec_draft_n_max_flag = "--spec-draft-n-max",+ help_probe_ok = True,+ flags = dict(_CAPS["flags"]),+ )+ monkeypatch.setattr(+ LlamaCppBackend, "probe_server_capabilities", classmethod(lambda cls, b = None: caps)+ )+ backend = LlamaCppBackend()+ backend._get_gpu_memory = lambda _binary = None, **_kw: []+ backend._get_gpu_free_memory = lambda _binary = None, **_kw: []++ def read_metadata(_path):+ backend._vocab_size = vocab++ backend._read_gguf_metadata = read_metadata+ backend._can_estimate_kv = lambda: False+ backend._get_gguf_size_bytes = lambda _path: 1024+ backend._mmproj_vram_bytes = lambda _path: 0+ backend._resolve_launch_mmproj_path = lambda **kwargs: None+ backend._apu_ram_shortfall_message = lambda *a, **k: None+ backend._amd_apu_wants_unified_memory = lambda *a, **k: False+ backend._find_llama_server_binary = lambda include_denied = False: "/fake/llama-server"+ backend._is_vulkan_backend = lambda _binary = None: vulkan+ backend._wait_for_health = lambda timeout, **_kw: True+ backend._detect_audio_type_strict = lambda: None+ backend._apply_detected_audio = lambda _detected: True+ draft = tmp_path / "dspark-model-F16.gguf"+ draft.write_bytes(b"GGUF")++ captured = {}++ def fake_popen(cmd, **kwargs):+ if not cmd or str(cmd[0]) != "/fake/llama-server":+ return _REAL_POPEN(cmd, **kwargs)+ captured["cmd"] = list(cmd)+ return type(+ "Process",+ (),+ {+ "pid": 123,+ "stdout": (),+ "poll": lambda self: None,+ "terminate": lambda self: None,+ "wait": lambda self, timeout = None: 0,+ "kill": lambda self: None,+ },+ )()++ with patch.object(subprocess, "Popen", side_effect = fake_popen):+ backend.load_model(+ GgufLoadIntent(+ gguf_path = str(_write_gguf(tmp_path / "model.gguf")),+ dspark_draft_path = str(draft),+ model_identifier = "openbmb/MiniCPM-V-4.7-1B-gguf",+ speculative_type = "dspark",+ n_ctx = 4096,+ extra_args = extra_args,+ )+ )+ return captured["cmd"]+++def test_load_model_launches_a_misaligned_vulkan_dspark_without_backend_draft_sampling(+ tmp_path, monkeypatch+):+ cmd = _launch(tmp_path, monkeypatch, vulkan = True, vocab = _MINICPM_V47_VOCAB)+ assert "draft-dspark" in cmd+ assert cmd.count(_FLAG) == 1++[email protected]("vulkan, vocab", [(False, _MINICPM_V47_VOCAB), (True, 248320)])+def test_load_model_leaves_every_other_dspark_launch_as_it_was(+ tmp_path, monkeypatch, vulkan, vocab+):+ cmd = _launch(tmp_path, monkeypatch, vulkan = vulkan, vocab = vocab)+ assert "draft-dspark" in cmd+ assert _FLAG not in cmd+++def test_an_explicit_backend_sampling_extra_still_wins(tmp_path, monkeypatch):+ # llama.cpp is last-wins and the extras follow the spec block.+ cmd = _launch(+ tmp_path,+ monkeypatch,+ vulkan = True,+ vocab = _MINICPM_V47_VOCAB,+ extra_args = ["--spec-draft-backend-sampling"],+ )+ assert cmd.index("--spec-draft-backend-sampling") > cmd.index(_FLAG)