Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: add per-model custom llama.cpp INI configuration (#10783)

Etherl864fec40e8a8
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 7ada48eef08..ea2395c8023 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -56,6 +56,13 @@ from typing import (
import httpx
from core.inference.chat_template_helpers import build_dac_tts_prompt
+from core.inference.llama_custom_config import (
+ CompiledCustomConfig,
+ CustomConfigError,
+ CustomConfigSource,
+ compile_custom_config,
+ parse_config_source,
+)
from core.inference.context_window import (
_COMPACTION_HEADROOM_RATIO,
clamp_compaction_headroom_ratio,
@@ -625,8 +632,10 @@ class GgufLoadIntent:
force_reload: bool = False
cpu_fallback: bool = False
+ llama_cpp_config: Optional[CustomConfigSource] = None
def __post_init__(self):
+ object.__setattr__(self, "llama_cpp_config", parse_config_source(self.llama_cpp_config))
for key in ("tensor_split", "gpu_ids", "extra_args"):
value = getattr(self, key)
if value is not None:
@@ -635,6 +644,11 @@ class GgufLoadIntent:
object.__setattr__(self, "gpu_ids", None)
+def _is_custom(intent) -> bool:
+ source = getattr(intent, "llama_cpp_config", None)
+ return source is not None and source.mode == "custom"
+
+
class _CpuFallbackRuntime(NamedTuple):
tempdir: tempfile.TemporaryDirectory
source_binary: Path
@@ -7754,6 +7768,7 @@ class LlamaCppBackend:
self._requested_n_ctx: int = 0
# Last healthy caller intent for crash recovery. Memory-only and never logged.
self._last_load_intent: Optional[GgufLoadIntent] = None
+ self._compiled_custom_config: Optional[CompiledCustomConfig] = None
self._mtp_runtime_fallback_lock = threading.Lock()
self._mtp_runtime_fallback_in_progress = False
# Background watchdog so an MTP+tensor crash recovers even when no request
@@ -8487,6 +8502,8 @@ class LlamaCppBackend:
never competed with."""
if self._arch_gate_forced_cpu:
return True
+ if self._compiled_custom_config is not None:
+ return self._compiled_custom_config.cpu_only and self._gpu_offload_active is False
return (
self._gpu_memory_mode == "manual"
and self._gpu_layers == 0
@@ -8571,6 +8588,29 @@ class LlamaCppBackend:
def last_load_intent(self) -> Optional[GgufLoadIntent]:
return self._last_load_intent
+ @property
+ def requested_llama_cpp_config(self) -> Optional[dict]:
+ source = getattr(self._last_load_intent, "llama_cpp_config", None)
+ return source.to_wire() if source is not None else None
+
+ @property
+ def llama_cpp_config_summary(self) -> Optional[dict]:
+ compiled = getattr(self, "_compiled_custom_config", None)
+ return compiled.summary() if compiled is not None else None
+
+ def _custom_runtime_matches(self, intent: GgufLoadIntent) -> bool:
+ resident = self._last_load_intent
+ return (
+ not intent.force_reload
+ and getattr(self, "_compiled_custom_config", None) is not None
+ and resident is not None
+ and resident.llama_cpp_config == intent.llama_cpp_config
+ and intent.model_identifier == self._model_identifier
+ and (intent.hf_variant or None) == (self._hf_variant or None)
+ and intent.disable_vision == resident.disable_vision
+ and not self._binary_changed_since_launch()
+ )
+
def _runtime_matches_intent(
self, intent: GgufLoadIntent, effective_extra_args: Optional[list[str]]
) -> bool:
@@ -8578,6 +8618,8 @@ class LlamaCppBackend:
if intent.force_reload:
return False
+ if _is_custom(intent) or getattr(self, "_compiled_custom_config", None) is not None:
+ return _is_custom(intent) and self._custom_runtime_matches(intent)
if self._requested_n_ctx != int(intent.n_ctx):
return False
if not self._is_diffusion and self._requested_n_parallel != max(1, int(intent.n_parallel)):
@@ -23313,6 +23355,291 @@ class LlamaCppBackend:
self._stdout_thread.start()
return True
+ def prepare_custom_config(self, intent: GgufLoadIntent) -> CompiledCustomConfig:
+ """Compile the INI against the selected llama-server without touching the resident model."""
+ if getattr(self, "_llama_update_in_progress", False):
+ raise CustomConfigError("llama.cpp is updating; try again in a moment")
+ binary = self._exec_path_for_launch(self._find_llama_server_binary())
+ if not binary:
+ raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL)
+ caps = self.probe_server_capabilities(binary)
+ if not caps.get("help_probe_ok"):
+ raise CustomConfigError("Could not read the selected llama-server's options")
+ return compile_custom_config(
+ intent.llama_cpp_config, caps.get("flags") or {}, caps.get("switch_flags") or ()
+ )
+
+ def _load_custom_model(
+ self,
+ intent: GgufLoadIntent,
+ *,
+ load_cancel_event: Optional[threading.Event] = None,
+ ) -> bool:
+ """Launch Studio's transport flags plus the compiled INI: no tuning, no fallback."""
+ from core.inference.model_ids import public_model_id
+
+ self._cancel_event.clear()
+ epoch = self._unload_epoch
+
+ def cancelled():
+ return (
+ self._cancel_event.is_set()
+ or self._unload_epoch != epoch
+ or bool(load_cancel_event is not None and load_cancel_event.is_set())
+ )
+
+ compiled = self.prepare_custom_config(intent)
+ model_path, projector = intent.gguf_path, intent.mmproj_path
+ download_cancel = (
+ _CombinedCancelEvent(self._cancel_event, load_cancel_event)
+ if load_cancel_event
+ else self._cancel_event
+ )
+ if intent.hf_repo and not model_path:
+ with _hf_offline_if_unreachable():
+ model_path = self._download_gguf(
+ hf_repo = _resolve_repo_id_casing(intent.hf_repo),
+ hf_variant = intent.hf_variant,
+ hf_token = intent.hf_token,
+ cancel_event = download_cancel,
+ allow_smaller_fallback = False,
+ )
+ if cancelled():
+ return False
+ if not model_path or not Path(model_path).is_file():
+ raise CustomConfigError("The selected GGUF file is unavailable")
+ refusal = self._non_chat_gguf_refusal_for_path(model_path, intent.model_identifier)
+ if refusal:
+ raise CustomConfigError(refusal)
+ # The route could not classify an uncached file; diffusion GGUFs run outside llama-server.
+ if self._gguf_path_is_diffusion(model_path, intent.model_identifier):
+ raise CustomConfigError(
+ "Diffusion GGUFs run on the diffusion runner; use Studio settings"
+ )
+ # The INI's own projector switch-off wins over Studio's --mmproj (last wins on argv).
+ if {"--no-mmproj", "--no-mmproj-auto"} & set(compiled.options):
+ projector = None
+ # Downloaded even with vision off, as managed loads do: only its header tells an
+ # audio-only encoder (kept) from an image tower (dropped below).
+ elif intent.is_vision and not projector and intent.hf_repo:
+ with _hf_offline_if_unreachable():
+ projector = self._download_mmproj(
+ hf_repo = intent.hf_repo,
+ hf_token = intent.hf_token,
+ cancel_event = download_cancel,
+ near_path = model_path,
+ )
+ if projector:
+ projector = self._resolve_launch_mmproj_path(
+ model_path = model_path, mmproj_path = projector
+ )
+ if projector and intent.disable_vision and not _mmproj_env_is_audio_only(projector):
+ projector = None
+
+ binary = self._exec_path_for_launch(self._find_llama_server_binary())
+ caps = self.probe_server_capabilities(binary)
+ # The INI is the only configuration source: inherited LLAMA_ARG_* would be a second one.
+ env = {
+ k: v
+ for k, v in self._llama_server_env_for_binary(binary).items()
+ if not k.startswith("LLAMA_ARG_")
+ }
+ scrub_denied_env(env) # LLAMA_API_KEY: the child would refuse Studio's own requests
+ # HSA enumeration dies on an arch the build lacks before --device is read: mask like managed loads.
+ survivors = self._arch_gate_survivors(binary)
+ if survivors:
+ self._emit_child_gpu_visibility(
+ env, ",".join(str(i) for i in survivors), prefer_rocr = True
+ )
+ port = self._find_free_port()
+ cmd = [binary, "-m", model_path, "--host", "127.0.0.1", "--port", str(port)]
+ cmd += ["--alias", public_model_id(intent.model_identifier), "--jinja"]
+ if compiled.n_parallel is not None:
+ cmd += ["--parallel", str(compiled.n_parallel)]
+ if projector:
+ cmd += ["--mmproj", projector]
+ api_key = None
+ if os.getenv("UNSLOTH_DIRECT_STREAM", "0") == "1":
+ import secrets
+ api_key = secrets.token_urlsafe(32)
+ cmd += ["--api-key", api_key]
+ cmd += list(compiled.argv)
+ resolved = replace(intent, gguf_path = model_path, mmproj_path = projector, verified_gguf = None)
+
+ with self._lock:
+ if cancelled():
+ return False
+ with self._spawn_lock:
+ if self._spawn_is_stale():
+ return False
+ self._begin_load_warnings()
+ self._kill_process()
+ self._cleanup_cpu_fallback_runtime()
+ self._model_identifier = intent.model_identifier
+ self._read_gguf_metadata(model_path)
+ self._gguf_path = model_path
+ self._gguf_load_identity = self._gguf_load_source_identity(model_path, projector)
+ self._hf_repo, self._hf_variant = intent.hf_repo, intent.hf_variant
+ self._port, self._api_key = port, api_key
+ self._launch_binary_revision = self._binary_revision(binary)
+ self._slot_save_dir = self._chat_template_override = None
+ self._cpu_fallback_reason = self._mmproj_fallback_reason = None
+ self._mtp_draft_path = self._mtp_draft_suppressed_path = None
+ self._spec_fallback_reason = self._spec_drafter_kind = None
+ self._mtp_runtime_fallback_active = self._dflash_retry_needed = False
+ self._memory_state = self._memory_direct_io = self._memory_pending_launch = None
+ self._is_vision = bool(projector)
+ self._disable_vision = self._vision_disabled_by_user = intent.disable_vision
+ self._mmproj_has_audio, self._mmproj_accepts_image = False, True
+ self._mmproj_projector_type = None
+ if projector:
+ from utils.models.gguf_metadata import (
+ mmproj_capabilities,
+ read_mmproj_projector_type,
+ )
+ self._mmproj_has_audio, self._mmproj_accepts_image = mmproj_capabilities(projector)
+ self._mmproj_projector_type = read_mmproj_projector_type(projector)
+ self._is_audio = self._audio_probed = self._has_audio_input = False
+ self._audio_type = None
+ # Every managed knob reads as unset: the INI, not Studio, chose them.
+ self._extra_args, self._requested_extra_args = [], []
+ self._extra_args_source = (intent.model_identifier, intent.hf_variant)
+ self._requested_n_batch = self._requested_n_ubatch = None
+ self._requested_load_mode = self._requested_spec_draft_cache_type = None
+ self._requested_ctx_checkpoints = self._requested_cache_ram = None
+ self._effective_cache_types = (
+ compiled.option("-ctk", "--cache-type-k") or "f16",
+ compiled.option("-ctv", "--cache-type-v") or "f16",
+ )
+ self._requested_cache_types = self._effective_cache_types
+ self._cache_type_kv = compiled.option("-ctk", "--cache-type-k")
+ self._speculative_type = self._requested_spec_mode = self._spec_draft_n_max = None
+ self._fit_load_mode_flags, self._vram_fraction_launched = [], None
+ self._gpu_memory_mode, self._gpu_layers, self._n_cpu_moe = "auto", -1, 0
+ self._tensor_parallel = self._arch_gate_forced_cpu = False
+ self._tensor_split = self._gpu_ids = self._requested_gpu_ids = None
+ self._auto_tensor_split = self._auto_tensor_split_emitted = None
+ self._spill_plan_flags = []
+ self._layer_preserves_tensor_intent = self._capability_probe_inconclusive = False
+ self._flash_attn_enabled = self._swa_full = self._kv_cache_unified = False
+ self._prompt_cache_disabled = "--no-cache-prompt" in compiled.options
+ budget = compiled.option("--reasoning-budget")
+ self._reasoning_budget = self._requested_reasoning_budget = (
+ int(budget) if budget is not None and re.fullmatch(r"-?[0-9]+", budget) else -1
+ )
+ self._reasoning_budget_message = self._requested_reasoning_budget_message = (
+ compiled.option("--reasoning-budget-message") or ""
+ )
+ try:
+ if cancelled() or not self._start_llama_process(
+ cmd,
+ env,
+ child_gpu_physical_ids = None if compiled.cpu_only else tuple(survivors) or None,
+ ):
+ self._kill_process()
+ return False
+ if not self._wait_for_health(timeout = 600.0, cancelled = cancelled):
+ was_cancelled = cancelled() or getattr(self, "_health_wait_cancelled", False)
+ self._kill_process()
+ if was_cancelled:
+ return False
+ # llama-server's own reason (unknown device, bad value) is otherwise only in the log.
+ reason = next(
+ (
+ l.strip()
+ for l in reversed(self._stdout_lines[-50:])
+ if "error" in l.lower()
+ ),
+ "check the server log",
+ )
+ if survivors and "device" in reason.lower():
+ reason += f" (only supported GPUs are visible, numbered from 0 to {len(survivors) - 1})"
+ raise CustomConfigError(
+ f"llama-server did not start with this configuration: {reason}"
+ )
+ props = self._query_server_props() or {}
+ n_ctx = (props.get("default_generation_settings") or {}).get("n_ctx")
+ slots = props.get("total_slots")
+ if not isinstance(n_ctx, int) or not isinstance(slots, int) or slots < 1:
+ raise CustomConfigError("llama-server did not report its context and slots")
+ template = props.get("chat_template")
+ if isinstance(template, str):
+ self._chat_template = template
+ flags = detect_reasoning_flags(
+ template, intent.model_identifier, log_source = "custom config"
+ )
+ for name in (
+ "supports_reasoning",
+ "reasoning_style",
+ "reasoning_always_on",
+ "reasoning_effort_levels",
+ "supports_preserve_thinking",
+ "preserve_thinking_default",
+ "supports_tools",
+ ):
+ setattr(self, "_" + name, flags[name])
+ self._commit_effective_parallel_slots(slots)
+ self._requested_n_parallel = slots
+ self._requested_n_ctx = 0
+ self._effective_context_length = n_ctx
+ self._max_context_length = self._context_length
+ # A unified cache is one shared pool (llama-server enables it for np auto):
+ # trust its startup line, else the options.
+ unified = next(
+ (
+ m[1] == "true"
+ for line in reversed(self._stdout_lines)
+ if (m := re.search(r"kv_unified = '(true|false)'", line))
+ ),
+ compiled.n_parallel is None
+ or "--kv-unified" in compiled.options
+ or "-kvu" in compiled.options,
+ )
+ if "--no-kv-unified" in compiled.options or "-no-kvu" in compiled.options:
+ unified = False
+ self._kv_cache_unified = unified
+ per_slot = compiled.option("--kv-unified-per-slot")
+ total = n_ctx if unified and not per_slot else n_ctx * slots
+ # An explicit -c pins a unified pool; only c=0 sizes it to np x per-slot (server.cpp).
+ pinned = compiled.option("-c", "--ctx-size")
+ if unified and pinned and re.fullmatch(r"[0-9]+", pinned) and int(pinned) > 0:
+ total = min(total, int(pinned))
+ self._kv_cache_context_total = total
+ self._has_video_input = bool((props.get("modalities") or {}).get("video"))
+ self._idle_slot_clearing_active = _idle_slot_clearing_active(
+ cmd, supports_cache_ram = bool(caps.get("supports_cache_ram"))
+ )
+ self._gpu_offload_active = (
+ False
+ if compiled.cpu_only and not projector
+ else classify_gpu_offload_lines(self._stdout_lines)
+ )
+ self._compiled_custom_config = compiled
+ self._last_load_intent = resolved
+ if cancelled() or not self._publish_healthy():
+ self._kill_process()
+ return False
+ except Exception:
+ self._kill_process()
+ raise
+ finally:
+ if not self._healthy:
+ self._compiled_custom_config = None
+ self._last_load_intent = None
+ # Outside self._lock (non-reentrant; _apply_detected_audio takes it), as managed loads
+ # do. The duplicate-load fast path requires the probe.
+ try:
+ detected = self._detect_audio_type_strict()
+ self._audio_probed = True
+ except Exception as exc:
+ logger.debug("Audio probe failed: %s", exc)
+ detected = None
+ return (
+ self._apply_detected_audio(detected)
+ if intent.audio_codec_path is None
+ else self._apply_detected_audio(detected, intent.audio_codec_path)
+ )
+
@contextlib.contextmanager
def _serial_load_scope(self):
"""Hold the serial load lock and release pending launch state with it.
@@ -23419,6 +23746,9 @@ class LlamaCppBackend:
if getattr(self, "_llama_update_in_progress", False):
raise RuntimeError("llama.cpp is updating; try again in a moment.")
+ if _is_custom(intent):
+ return self._load_custom_model(intent, load_cancel_event = load_cancel_event)
+
intent = self._preserve_cpu_fallback_intent(intent)
tensor_parallel = intent.tensor_parallel
gpu_memory_mode = intent.gpu_memory_mode
@@ -24170,6 +24500,7 @@ class LlamaCppBackend:
if _load_cancelled():
logger.info("Load cancelled before diffusion server start")
return False
+ self._compiled_custom_config = None
return self._start_diffusion_server(
model_path = model_path,
gguf_path = gguf_path,
@@ -31605,6 +31936,7 @@ class LlamaCppBackend:
# dropped: a respawn replays this snapshot arbitrarily later, so recovery
# re-resolves the file instead of trusting a path this request verified.
self._last_load_intent = replace(intent, verified_gguf = None)
+ self._compiled_custom_config = None
self._mtp_runtime_fallback_active = _mtp_active_for_launched_server
self._start_mtp_crash_watchdog()
@@ -32324,6 +32656,13 @@ class LlamaCppBackend:
if self._binary_changed_since_launch():
logger.info("llama-server selection changed since launch; forcing a reload")
return False
+ if _is_custom(intent) or getattr(self, "_compiled_custom_config", None) is not None:
+ if not (_is_custom(intent) and self._custom_runtime_matches(intent)):
+ return False
+ self._last_load_intent = replace(
+ intent, gguf_path = intent.gguf_path or self._gguf_path, verified_gguf = None
+ )
+ return True
intent = self._preserve_cpu_fallback_intent(intent, source_matches = True)
# The stored state is what LAUNCHED, and on a virtualised Metal device that is
# the CPU-pinned rewrite, not the request. Apply the same rewrite here
@@ -32553,6 +32892,7 @@ class LlamaCppBackend:
self._launch_binary_revision = ()
self._cpu_fallback_reason = None
self._last_load_intent = None
+ self._compiled_custom_config = None
self._mtp_runtime_fallback_active = False
self._hf_variant = None
self._is_vision = False
@@ -34406,6 +34746,8 @@ class LlamaCppBackend:
"""
# Cheap async-safe gate: only our live MTP+tensor launch, not cancelled,
# with a snapshot to replay.
+ if getattr(self, "_compiled_custom_config", None) is not None:
+ return False
if self._cancel_event.is_set():
return False
if not self._mtp_runtime_fallback_active:
diff --git a/studio/backend/core/inference/llama_custom_config.py b/studio/backend/core/inference/llama_custom_config.py
new file mode 100644
index 00000000000..9f463a61c2e
--- /dev/null
+++ b/studio/backend/core/inference/llama_custom_config.py
@@ -0,0 +1,265 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Compile a per-model llama.cpp preset INI into llama-server argv.
+
+Grammar follows llama.cpp common/preset.cpp: keys before any header form the ``default``
+section, only ``[*]`` applies to every section, ``;``/``#`` start comments and the last
+duplicate key wins.
+Option names and arity come from the selected binary's ``--help`` catalog.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+import re
+
+from . import llama_server_args as policy
+
+MAX_INI_BYTES = 64 * 1024
+_TRUE = frozenset({"on", "enabled", "true", "1"})
+_FALSE = frozenset({"off", "disabled", "false", "0"})
+# Studio selects the model and projector; preset files commonly carry both.
+_IGNORED = frozenset({"-m", "--model", "-mm", "--mmproj"})
+_PARALLEL = frozenset({"-np", "--parallel", "--n-parallel"})
+# Router metadata in upstream preset files; llama-server itself has no such options.
+_ROUTER_ONLY = frozenset({"load-on-startup", "stop-timeout", "dedup-cache-models", "version"})
+# Flag -> (request field, Studio chat API bounds in models/inference.py).
+_SAMPLING = {
+ "--temp": ("temperature", 0.0, 2.0),
+ "--top-p": ("top_p", 0.0, 1.0),
+ "--top-k": ("top_k", -1, 100),
+ "--min-p": ("min_p", 0.0, 1.0),
+ "--repeat-penalty": ("repetition_penalty", 1.0, 2.0),
+ "--presence-penalty": ("presence_penalty", 0.0, 2.0),
+}
+
+
+class CustomConfigError(ValueError):
+ """A configuration error whose message never echoes INI values."""
+
+
+@dataclass(frozen = True)
+class CustomConfigSource:
+ version: int = 1
+ mode: str = "managed"
+ ini: str | None = None
+ section: str | None = None
+
+ def to_wire(self) -> dict:
+ if self.mode != "custom":
+ return {"version": 1, "mode": "managed"}
+ return {"version": 1, "mode": "custom", "ini": self.ini, "section": self.section}
+
+
+def parse_config_source(value) -> CustomConfigSource | None:
+ if value is None:
+ return None
+ if isinstance(value, CustomConfigSource):
+ value = value.to_wire()
+ if not isinstance(value, Mapping) or value.get("version") != 1:
+ raise CustomConfigError("llama_cpp_config must be an object with version 1")
+ mode = value.get("mode")
+ if mode == "managed" and set(value) <= {"version", "mode"}:
+ return CustomConfigSource()
+ if mode != "custom" or set(value) - {"version", "mode", "ini", "section"}:
+ raise CustomConfigError("llama_cpp_config mode must be managed or custom")
+ ini, section = value.get("ini"), value.get("section")
+ if not isinstance(ini, str) or not ini.strip():
+ raise CustomConfigError("Custom configuration requires INI text")
+ if len(ini.encode("utf-8", "surrogatepass")) > MAX_INI_BYTES:
+ raise CustomConfigError("INI text exceeds the 64 KiB limit")
+ if any((ord(c) < 32 and c not in "\r\n\t") or ord(c) == 127 for c in ini):
+ raise CustomConfigError("INI text contains control characters")
+ if section is not None and (
+ not isinstance(section, str) or not section or len(section) > 512 or "]" in section
+ ):
+ raise CustomConfigError("Select a valid INI section name")
+ return CustomConfigSource(1, "custom", ini, section)
+
+
+@dataclass(frozen = True)
+class CompiledCustomConfig:
+ source: CustomConfigSource
+ argv: tuple[str, ...]
+ n_parallel: int | None
+ options: dict = field(default_factory = dict)
+ request_defaults: dict = field(default_factory = dict)
+ diagnostics: tuple[str, ...] = ()
+ # Every help-block spelling of a valued option, so lookups match whichever alias the INI used.
+ aliases: dict = field(default_factory = dict)
+
+ def option(self, *names: str):
+ return next(
+ (table[n] for table in (self.options, self.aliases) for n in names if n in table),
+ None,
+ )
+
+ @property
+ def cpu_only(self) -> bool:
+ # -ngl 0 still offloads ops and the projector; --device none also moves the projector
+ # (common/arg.cpp), but -mmdev and a drafter pick their own devices.
+ return (
+ self.option("-dev", "--device") == "none"
+ and self.option("-mmdev", "--mmproj-device") in (None, "none")
+ and (
+ self.option("-md", "-hfd") is None
+ or self.option("-devd", "--device-draft") == "none"
+ )
+ )
+
+ def summary(self) -> dict:
+ return {
+ "mode": "custom",
+ "section": self.source.section,
+ "options": dict(self.options),
+ "request_defaults": dict(self.request_defaults),
+ "diagnostics": list(self.diagnostics),
+ }
+
+
+def _sections(ini: str) -> dict[str, dict[str, str]]:
+ sections: dict[str, dict[str, str]] = {"default": {}}
+ current = "default"
+ for number, raw in enumerate(re.split(r"\r\n|\r|\n", ini), 1):
+ header = re.fullmatch(r"\[[ \t]*([^\]]+)\][ \t]*(?:[;#].*)?", raw)
+ if header:
+ current = header[1].strip()
+ sections[current] = {}
+ continue
+ line = re.split(r"[;#]", raw, maxsplit = 1)[0].strip()
+ if not line:
+ continue
+ entry = re.fullmatch(r"([A-Za-z_][A-Za-z0-9_.-]*)\s*=\s*(.*)", line)
+ if not entry:
+ raise CustomConfigError(f"Invalid INI syntax at line {number}")
+ sections.setdefault(current, {})[entry[1]] = entry[2]
+ return sections
+
+
+def _polarity(block: str) -> tuple[list[str], list[str]]:
+ """Positive and negative spellings from a help block's declaration line.
+
+ llama.cpp prints args then args_neg, each run ending in its long form:
+ ``-kvo, --kv-offload, -nkvo, --no-kv-offload``."""
+ head = re.match(r"(-[\w.-]+(?:, -[\w.-]+)*)", block or "")
+ names = head[1].split(", ") if head else []
+ cut = next((i + 1 for i, n in enumerate(names) if n.startswith("--")), len(names))
+ return names[:cut], names[cut:]
+
+
+def _spelling(key: str, flags: Mapping) -> str | None:
+ found = next((f for f in (f"--{key}", f"-{key}") if f in flags), None)
+ if found is None and re.fullmatch(r"[A-Z][A-Z0-9_]*", key):
+ # Presets may name an option by its env variable, which maps to the option itself.
+ block = next((d for d in flags.values() if d and f"(env: {key})" in d), None)
+ if block is not None:
+ positive = [f for f in _polarity(block)[0] if f in flags]
+ found = max(positive, key = len, default = None)
+ return found
+
+
+def compile_custom_config(source, flags: Mapping, switch_flags) -> CompiledCustomConfig:
+ """``flags``/``switch_flags`` are the selected binary's probed help catalog."""
+ source = parse_config_source(source)
+ if source is None or source.mode != "custom":
+ raise CustomConfigError("Compilation requires custom mode")
+ if "--port" not in flags:
+ raise CustomConfigError(
+ "The selected llama-server did not report its options; check the install"
+ )
+ switches = set(switch_flags)
+ sections = _sections(source.ini)
+ named = [s for s in sections if s != "*" and (s != "default" or sections[s])]
+ section = source.section
+ if section is None and named:
+ if named != ["default"]:
+ raise CustomConfigError("Select which INI section to use")
+ section = "default"
+ if section is not None and section not in named:
+ raise CustomConfigError("The selected INI section does not exist")
+ # llama.cpp resolves keys to options before cascading: the section overrides [*] per option.
+ entries: dict = {}
+ for name in ("*", section):
+ owners: dict = {}
+ for key, value in sections.get(name or "", {}).items():
+ if key in _ROUTER_ONLY:
+ continue
+ flag = _spelling(key, flags)
+ if flag is None:
+ raise CustomConfigError(
+ f"'{key[:80]}' is not an option of the selected llama-server"
+ )
+ # Two aliases in one section: llama.cpp keeps one arbitrarily, so refuse both.
+ identity = flags[flag] or flag
+ other = owners.setdefault(identity, key)
+ if other != key:
+ raise CustomConfigError(f"'{other[:80]}' and '{key[:80]}' set the same option")
+ entries[identity] = (key, flag, value)
+
+ argv: list[str] = []
+ options: dict = {}
+ aliases: dict = {}
+ defaults: dict = {}
+ diagnostics: list[str] = []
+ n_parallel = None
+ for identity, (key, flag, value) in entries.items():
+ if flag in _IGNORED:
+ note = "Studio selects the model and projector; m and mm entries are ignored."
+ if note not in diagnostics:
+ diagnostics.append(note)
+ continue
+ if flag in _PARALLEL:
+ if (
+ not re.fullmatch(r"[0-9]+", value)
+ or not policy.PARALLEL_MIN <= int(value) <= policy.PARALLEL_MAX
+ ):
+ raise CustomConfigError(
+ f"'{key}' must be between {policy.PARALLEL_MIN} and {policy.PARALLEL_MAX}"
+ )
+ n_parallel = int(value)
+ continue
+ if policy.is_managed_flag(flag):
+ raise CustomConfigError(f"'{key[:80]}' is managed by Studio and cannot be set here")
+ if flag in switches:
+ if value.lower() not in _TRUE | _FALSE:
+ raise CustomConfigError(f"'{key[:80]}' needs true or false")
+ # Like common/preset.cpp: a negative alias (-nkvo, --no-x) inverts the value, and
+ # false on a switch with no negative form is dropped.
+ positive, negative = _polarity(identity)
+ if flag not in positive + negative:
+ positive, negative = [flag], []
+ on = (value.lower() in _TRUE) != (flag in negative)
+ pick = [f for f in (positive if on else negative) if f in flags]
+ if not pick:
+ continue
+ flag = flag if flag in pick else max(pick, key = len)
+ argv.append(flag)
+ options[flag] = True
+ continue
+ if not value:
+ raise CustomConfigError(f"'{key[:80]}' needs a value")
+ argv += [flag, value]
+ options[flag] = value
+ aliases.update((f, value) for f, d in flags.items() if d and d == identity)
+ sampling = next((s for s in _SAMPLING if s in flags and flags[s] == flags[flag]), None)
+ if sampling is not None:
+ name, low, high = _SAMPLING[sampling]
+ try:
+ number = int(value) if sampling == "--top-k" else float(value)
+ except ValueError:
+ raise CustomConfigError(f"'{key[:80]}' needs a number") from None
+ # It becomes the chat default, so it must pass the chat request schema.
+ if not low <= number <= high:
+ raise CustomConfigError(f"'{key[:80]}' must be between {low} and {high}")
+ defaults[name] = number
+ if "--no-jinja" in options:
+ raise CustomConfigError("jinja = false would break Studio's tool calling")
+ try:
+ policy.validate_extra_args(argv)
+ except ValueError as exc:
+ raise CustomConfigError(str(exc)) from None
+ return CompiledCustomConfig(
+ source, tuple(argv), n_parallel, options, defaults, tuple(diagnostics), aliases
+ )
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index 716a9816f33..ac59d7be0a4 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -89,6 +89,7 @@ _DENYLIST_GROUPS: tuple[frozenset[str], ...] = (
frozenset({"--path"}),
frozenset({"--api-prefix"}),
frozenset({"--reuse-port"}),
+ frozenset({"--rpc"}),
# Auth / TLS: Unsloth terminates auth; upstream --api-key / TLS shadows Unsloth's key and breaks the proxy hop
frozenset({"--api-key"}),
frozenset({"--api-key-file"}),
@@ -1803,6 +1804,7 @@ DENIED_ENV_VARS: tuple[str, ...] = (
"LLAMA_ARG_HOST",
"LLAMA_ARG_PORT",
"LLAMA_ARG_REUSE_PORT",
+ "LLAMA_ARG_RPC",
"LLAMA_ARG_N_PARALLEL",
"LLAMA_ARG_POOLING",
"LLAMA_ARG_EMBEDDINGS",
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 79783323cb9..6c90ef4735b 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -31,6 +31,7 @@ from core.inference.llama_server_args import (
from core.inference.runtime_context import MAX_REQUESTABLE_CONTEXT
from core.inference.video_families import MAX_VIDEO_NUM_FRAMES
from picker.schemas import MAX_CHAT_TEMPLATE_BYTES
+from models.llama_custom_config import LlamaCppConfigFields
from utils.reasoning_budget import validate_reasoning_budget_message
@@ -51,7 +52,7 @@ def resolve_inventory_handle(value: str) -> str:
return resolved
-class LoadRequest(BaseModel):
+class LoadRequest(LlamaCppConfigFields):
"""Request to load a model for inference"""
model_path: str = Field(..., description = "Model identifier or local path")
@@ -509,7 +510,7 @@ class SttLoadRequest(BaseModel):
)
-class ValidateModelRequest(BaseModel):
+class ValidateModelRequest(LlamaCppConfigFields):
"""Check whether an identifier resolves to a ModelConfig; does NOT load weights."""
model_path: str = Field(..., description = "Model identifier or local path")
@@ -794,6 +795,7 @@ class ValidateModelResponse(BaseModel):
"""
valid: bool = Field(..., description = "Whether the model identifier looks valid")
+ llama_cpp_config_summary: Optional[Dict[str, Any]] = None
message: str = Field(..., description = "Human-readable validation message")
identifier: Optional[str] = Field(None, description = "Resolved model identifier")
resident: bool = Field(
@@ -1254,6 +1256,9 @@ class GenerateRequest(BaseModel):
class _InferenceRuntimeFields(BaseModel):
"""Runtime fields shared by load and status responses."""
+ requested_llama_cpp_config: Optional[Dict[str, Any]] = None
+ llama_cpp_config_summary: Optional[Dict[str, Any]] = None
+
is_vision: bool = Field(False, description = "Whether model is a vision model")
is_diffusion: bool = Field(
False, description = "Whether model is a block-diffusion model (DiffusionGemma)"
diff --git a/studio/backend/models/llama_custom_config.py b/studio/backend/models/llama_custom_config.py
new file mode 100644
index 00000000000..c36f673f32c
--- /dev/null
+++ b/studio/backend/models/llama_custom_config.py
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Shared wire fields for a selected model's llama.cpp configuration."""
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class LlamaCppConfigFields(BaseModel):
+ llama_cpp_config: Optional[dict[str, Any]] = Field(
+ None,
+ description = "Versioned per-model INI configuration, or an explicit managed-mode reset.",
+ )
+
+ @field_validator("llama_cpp_config", mode = "before")
+ @classmethod
+ def validate_llama_cpp_config(cls, value):
+ if value is None:
+ return None
+ from core.inference.llama_custom_config import parse_config_source
+ return parse_config_source(value).to_wire()
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 5526d54d93a..d59a9b08bea 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -8078,6 +8078,8 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
# running with no extras would come back carrying the arguments of the load
# that just failed. None stays for "nothing was ever set", which is the only
# case where inheriting is the right answer.
+ requested_llama_cpp_config = getattr(llama_backend, "requested_llama_cpp_config", None),
+ llama_cpp_config_summary = getattr(llama_backend, "llama_cpp_config_summary", None),
requested_llama_extra_args = (
None
if llama_backend.is_diffusion
@@ -8139,9 +8141,15 @@ def _gguf_load_response(
is_lora = False,
is_gguf = True,
is_local_model = is_local_model,
- inference = load_inference_config(
- inference_identifier or llama_backend.model_identifier or model
- ),
+ # A custom INI's sampling seeds the chat controls like a model recommendation.
+ inference = {
+ **load_inference_config(
+ inference_identifier or llama_backend.model_identifier or model
+ ),
+ **(getattr(llama_backend, "llama_cpp_config_summary", None) or {}).get(
+ "request_defaults", {}
+ ),
+ },
# Advisory, and None on nearly every load. Recorded by load_model when the
# weights outgrow fast memory, so the client can say why generation is slow.
# getattr: older/custom backend doubles predate this additive field.
@@ -15215,6 +15223,8 @@ async def _prepare_load_placement(
request: LoadRequest | ValidateModelRequest,
extra_args: Optional[list[str]],
) -> _LoadPlacement:
+ if _custom_llama_config(request):
+ return _LoadPlacement(None, None, False, _classify_diffusion_gguf(config))
requested = request.gpu_ids or None
if not config.is_gguf:
return _LoadPlacement(requested, None, False, False)
@@ -15599,6 +15609,12 @@ def _guard_chat_load_against_training(
)
return
+ if _custom_llama_config(request):
+ raise HTTPException(
+ status_code = 409,
+ detail = "Custom llama.cpp placement cannot be verified beside active training. Stop training before loading this configuration.",
+ )
+
from core.inference.llama_cpp import _diffusion_manual_ngl, _scale_diffusion_required_gb
is_gguf = bool(getattr(config, "is_gguf", False))
@@ -15822,6 +15838,61 @@ def _guard_chat_load_against_training(
raise HTTPException(status_code = 409, detail = detail)
+def _custom_llama_config(request) -> bool:
+ source = getattr(request, "llama_cpp_config", None)
+ return isinstance(source, dict) and source.get("mode") == "custom"
+
+
+def _resolve_llama_cpp_config(
+ request,
+ config = None,
+ model_identifier = None,
+):
+ """Request's own source, else this model row's saved override, else the resident same-model load's."""
+ from core.inference.llama_custom_config import parse_config_source
+ from utils.openai_auto_switch_settings import resolve_override_for_load
+
+ source = getattr(request, "llama_cpp_config", None)
+ identifier = model_identifier or request.model_path
+ variant = getattr(request, "gguf_variant", None) or getattr(config, "gguf_variant", None)
+ if source is None:
+ _, override = resolve_override_for_load(
+ identifier, getattr(config, "identifier", None), variant
+ )
+ source = override.get("llama_cpp_config")
+ if source is None:
+ resident = get_llama_cpp_backend()
+ intent = getattr(resident, "last_load_intent", None)
+ if (
+ intent is not None
+ and _same_loaded_identifier(getattr(intent, "model_identifier", None), identifier)
+ and (getattr(intent, "hf_variant", None) or "").casefold() == (variant or "").casefold()
+ ):
+ source = getattr(intent, "llama_cpp_config", None)
+ if source is None:
+ return request
+ source = parse_config_source(source)
+ return request.model_copy(update = {"llama_cpp_config": source.to_wire()})
+
+
+async def _preflight_custom_llama_config(request, config):
+ """Compile a custom config before any backend or GPU owner can be evicted."""
+ if not _custom_llama_config(request):
+ return None
+ if not config.is_gguf or _classify_diffusion_gguf(config) is True:
+ raise HTTPException(
+ status_code = 400,
+ detail = "Custom llama.cpp configuration needs a GGUF chat model.",
+ )
+ intent = GgufLoadIntent(
+ model_identifier = config.identifier, llama_cpp_config = request.llama_cpp_config
+ )
+ try:
+ return await asyncio.to_thread(get_llama_cpp_backend().prepare_custom_config, intent)
+ except ValueError as exc:
+ raise HTTPException(status_code = 400, detail = redact_native_paths(str(exc))) from exc
+
+
def _resolve_inherited_extra_args(
request,
config: ModelConfig,
@@ -15832,6 +15903,8 @@ def _resolve_inherited_extra_args(
"""Effective pass-through extras for a GGUF request that omitted the field:
the previous same-model load's extras, shadow-stripped, so a settings-Apply
reload (which does not round-trip the extras field) keeps them (#5401)."""
+ if _custom_llama_config(request):
+ return []
if getattr(request, "llama_extra_args", None) is not None:
return extra_llama_args
if not getattr(config, "is_gguf", False):
@@ -17083,17 +17156,27 @@ async def _load_model_impl(
gguf_load_stack = ExitStack()
token_rejections = gguf_load_stack.enter_context(collecting_hub_token_rejections())
try:
+ requested_llama_cpp_config = request.llama_cpp_config
+
# Validate user pass-through args up front so a managed-flag collision
- # returns 400 before any model work.
- try:
- extra_llama_args = validate_extra_args(request.llama_extra_args)
- except ValueError as exc:
- # Keep the curated validation message (names the flag); just strip paths.
- logger.warning("inference.validate_extra_args_failed: %s", exc)
- raise HTTPException(
- status_code = 400,
- detail = redact_native_paths(str(exc)),
- )
+ # returns 400 before any model work. Custom mode launches without them.
+ def _validated_extra_args(req):
+ if _custom_llama_config(req):
+ return []
+ try:
+ return validate_extra_args(req.llama_extra_args)
+ except ValueError as exc:
+ # Keep the curated validation message (names the flag); just strip paths.
+ logger.warning("inference.validate_extra_args_failed: %s", exc)
+ raise HTTPException(
+ status_code = 400,
+ detail = redact_native_paths(str(exc)),
+ )
+
+ extra_llama_args = _validated_extra_args(request)
+ request = _resolve_llama_cpp_config(request)
+ if _custom_llama_config(request):
+ extra_llama_args = []
# Re-narrow []-from-None back to None so the inheritance path below can
# tell "caller omitted" from "caller explicit []".
extra_llama_args: Optional[list[str]] = (
@@ -17277,7 +17360,12 @@ async def _load_model_impl(
)
is_direct_gguf_request = model_identifier.lower().endswith(".gguf")
- if llama_backend.is_loaded and (request.gguf_variant or is_direct_gguf_request):
+ if (
+ llama_backend.is_loaded
+ and (request.gguf_variant or is_direct_gguf_request)
+ and getattr(request, "llama_cpp_config", None) is None
+ and getattr(llama_backend, "requested_llama_cpp_config", None) is None
+ ):
reused = _reuse_loaded_gguf(
_active_gguf_intent(
request,
@@ -17295,7 +17383,9 @@ async def _load_model_impl(
if not llama_backend.holds_no_vram:
await asyncio.to_thread(acquire_for_request, CHAT)
return reused
- if not (request.gguf_variant or is_direct_gguf_request):
+ if not (request.gguf_variant or is_direct_gguf_request) and not _custom_llama_config(
+ request
+ ):
_inherit_resident_load_in_4bit(backend, request, model_identifier)
if (
_same_loaded_identifier(backend.active_model_name, model_identifier)
@@ -17413,6 +17503,17 @@ async def _load_model_impl(
)
await asyncio.to_thread(_require_resolved_base_access, config)
+ request = _resolve_llama_cpp_config(
+ request.model_copy(update = {"llama_cpp_config": requested_llama_cpp_config}),
+ config,
+ public_model_identifier,
+ )
+ extra_llama_args = _validated_extra_args(request)
+ custom_compiled = await _preflight_custom_llama_config(request, config)
+ if custom_compiled is not None:
+ _n_parallel = custom_compiled.n_parallel or _n_parallel
+ effective_chat_template_override = None
+
# Resolve inherited extras once before command-dependent preflights.
extra_llama_args = _resolve_inherited_extra_args(
request,
@@ -17652,6 +17753,8 @@ async def _load_model_impl(
request.speculative_type,
)
)
+ if custom_compiled is not None:
+ chat_load_needs_gpu = not custom_compiled.cpu_only
# Ahead of the arbiter: acquire_for evicts a resident Images/Video pipeline and the
# confirmation below cancels the running generations, both before load_model's own
# copy of this check runs. A header-sized read spares them. Fails open into that copy.
@@ -17858,13 +17961,18 @@ async def _load_model_impl(
# loading; the response reports the backend's actual tensor_parallel
# state so the UI toggle reflects the fallback.
try:
- success = await load_with_tensor_fallback(
- _attempt_gguf_load,
- requested_tensor = request.tensor_parallel,
- extra_args = extra_llama_args,
- label = config.identifier,
- cancelled = lambda: _gguf_load_cancelled(llama_backend, load_cancel_event),
- )
+ if custom_compiled is not None:
+ success = await _run_gguf_load_attempt(
+ llama_backend, load_intent, load_cancel_event
+ )
+ else:
+ success = await load_with_tensor_fallback(
+ _attempt_gguf_load,
+ requested_tensor = request.tensor_parallel,
+ extra_args = extra_llama_args,
+ label = config.identifier,
+ cancelled = lambda: _gguf_load_cancelled(llama_backend, load_cancel_event),
+ )
except Exception:
# A GGUF load can raise before tearing down the old llama-server (e.g. an
# update-in-progress guard fires before _kill_process), leaving the prior
@@ -18504,6 +18612,11 @@ async def validate_model(
detail = f"Invalid model identifier: {model_log_label}",
)
+ request = _resolve_llama_cpp_config(
+ request, config, _public_model_identifier(request.model_path, model_identifier)
+ )
+ custom_compiled = await _preflight_custom_llama_config(request, config)
+
# The caller's own list when it sent one, or the resolver hands back this
# fourth argument unchanged and a --ctx-size the load is about to use would
# be missing from the estimate that approves it.
@@ -18805,6 +18918,9 @@ async def validate_model(
return restore_inventory_handles(
ValidateModelResponse(
valid = True,
+ llama_cpp_config_summary = custom_compiled.summary()
+ if custom_compiled is not None
+ else None,
message = " ".join(
["Model identifier is valid."] + _hub_access_warnings(token_rejections)
),
@@ -19502,6 +19618,9 @@ async def estimate_memory(
from core.inference.llama_cpp import _args_place_tensors_on_cpu
from core.inference.llama_server_args import _effective_tensor_parallel
+ request = _resolve_llama_cpp_config(request)
+ if _custom_llama_config(request):
+ return EstimateMemoryResponse(available = False, reason = "unsizable")
if is_ollama_manifest_ref(request.model_path):
# Resolving one writes a .gguf link to disk; that belongs to the load path.
return EstimateMemoryResponse(available = False, reason = "unsupported_source")
@@ -26305,6 +26424,19 @@ async def delete_openai_container(
await client.close()
+def _custom_request_defaults(model_id) -> dict:
+ backend = get_llama_cpp_backend()
+ summary = getattr(backend, "llama_cpp_config_summary", None)
+ if not isinstance(summary, dict) or not _same_loaded_identifier(
+ getattr(backend, "model_identifier", None), model_id
+ ):
+ return {}
+ defaults = dict(summary.get("request_defaults") or {})
+ if "repeat_penalty" in defaults:
+ defaults["repetition_penalty"] = defaults.pop("repeat_penalty")
+ return defaults
+
+
_REASONING_EFFORT_VALUES = {"none", "minimal", "low", "medium", "high", "max", "xhigh"}
@@ -26415,7 +26547,9 @@ def _fill_recommended_sampling_openai(payload, model_id) -> None:
f: (getattr(payload, f) if f in payload.model_fields_set else None)
for f in SAMPLING_FIELD_NAMES
}
- effective = resolve_effective_sampling(model_id, explicit)
+ effective = resolve_effective_sampling(
+ model_id, explicit, preset_defaults = _custom_request_defaults(model_id)
+ )
for field, value in effective.items():
setattr(payload, field, value)
@@ -26440,7 +26574,9 @@ def _fill_recommended_sampling_completions(body: dict, model_id) -> None:
from utils.inference.inference_config import resolve_effective_sampling, SAMPLING_FIELD_NAMES
explicit = {f: body.get(_COMPLETIONS_SAMPLING_BODY_KEY.get(f, f)) for f in SAMPLING_FIELD_NAMES}
- effective = resolve_effective_sampling(model_id, explicit, fill_defaults = False)
+ effective = resolve_effective_sampling(
+ model_id, explicit, fill_defaults = False, preset_defaults = _custom_request_defaults(model_id)
+ )
for field, value in effective.items():
body[_COMPLETIONS_SAMPLING_BODY_KEY.get(field, field)] = value
@@ -37434,6 +37570,9 @@ async def anthropic_messages(
"repetition_penalty": payload.repetition_penalty,
"presence_penalty": payload.presence_penalty,
},
+ preset_defaults = _custom_request_defaults(
+ getattr(llama_backend, "model_identifier", None) or model_name
+ ),
)
temperature = _anthropic_sampling["temperature"]
top_p = _anthropic_sampling["top_p"]
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 8d0cb5306fe..b172fc5c980 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -47,6 +47,7 @@ from core.rag.config import (
effective_gguf_repo_for_embedding_model,
)
from loggers import get_logger
+from models.llama_custom_config import LlamaCppConfigFields
from utils.utils import safe_curated_detail, safe_error_detail, log_and_http_error
from utils.personalization_settings import (
MAX_AVATAR_DATA_URL_BYTES,
@@ -917,7 +918,7 @@ MAX_GGUF_VARIANT_KEY_LEN = 4096
MAX_GPU_IDS = MAX_GPU_ID + 1
-class ModelOverridePayload(BaseModel):
+class ModelOverridePayload(LlamaCppConfigFields):
"""One model's saved launch config, applied when the API loads that model.
Everything past ``model_id`` is optional and omitted means "app default", so a
@@ -2303,7 +2304,8 @@ def update_openai_auto_switch_override(
_carried_fields = (() if payload.mirrors_server_tuning else _tuning_fields) + (
() if payload.mirrors_reasoning_budget else _reasoning_fields
)
- if _carried_fields and not is_removal:
+ _stored_row = None
+ if not is_removal and (_carried_fields or payload.llama_cpp_config is None):
# The same spellings the extra-args carry-over walks: a cached repo is not an ordinary folded match,
# so a save under the repo id would find nothing and retire the alias with its tuning.
_alias_ids = [payload.model_id]
@@ -2320,15 +2322,15 @@ def update_openai_auto_switch_override(
# Taken as a unit from the first row that exists, not field by field down the list: a load stops at the
# first non-empty row rather than merging, so filling a gap in the winner from a loser would switch
# dormant tuning on.
- for _alias_id in _alias_ids:
- _stored_tuning = get_model_override(_alias_id)
- if not _stored_tuning:
- continue
- for name in _carried_fields:
- if _kept_tuning[name] is None:
- _kept_tuning[name] = _stored_tuning.get(name)
- break
+ _stored_row = next((row for row in map(get_model_override, _alias_ids) if row), None)
+ if _stored_row:
+ for name in _carried_fields:
+ if _kept_tuning[name] is None:
+ _kept_tuning[name] = _stored_row.get(name)
removed_keys: list[str] = []
+ kept_custom_config = payload.llama_cpp_config
+ if kept_custom_config is None and _stored_row:
+ kept_custom_config = _stored_row.get("llama_cpp_config")
if payload.remove is True:
# An explicit remove wins over any other field. Remove the key a load resolves to, not the literal one sent
# (the browser normalizes casing), and every spelling: clearing one of two leaves the survivor as the sole
@@ -2409,6 +2411,7 @@ def update_openai_auto_switch_override(
set_model_override(
target_id,
llama_extra_args = extra_args,
+ llama_cpp_config = kept_custom_config,
keep_empty_extra_args = keep_empty,
max_seq_length = max_seq_length,
custom_context_length = custom_context_length,
diff --git a/studio/backend/tests/test_llama_custom_config.py b/studio/backend/tests/test_llama_custom_config.py
new file mode 100644
index 00000000000..2162fa10353
--- /dev/null
+++ b/studio/backend/tests/test_llama_custom_config.py
@@ -0,0 +1,420 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from types import SimpleNamespace
+
+import pytest
+
+from core.inference import llama_cpp as llama
+from core.inference.llama_custom_config import (
+ CustomConfigError,
+ compile_custom_config,
+ parse_config_source,
+)
+
+_OPTIONS = (
+ "--port; -c --ctx-size; -ngl --gpu-layers; -fa --flash-attn; -np --parallel; -m --model; "
+ "-mm --mmproj; --temp --temperature; --top-k; --top-p; --warmup --no-warmup; "
+ "--mmproj-offload --no-mmproj-offload; --no-mmproj; --mmproj-auto --no-mmproj-auto; --jinja --no-jinja; -dev --device; "
+ "-ctk --cache-type-k; --host; --cache-prompt --no-cache-prompt; -mmdev --mmproj-device; "
+ "-sp --special; -cb --cont-batching -nocb --no-cont-batching; "
+ "-kvo --kv-offload -nkvo --no-kv-offload; "
+ "-md --model-draft --spec-draft-model; -hfd --hf-repo-draft --spec-draft-hf; "
+ "-devd --device-draft; --kv-unified-per-slot; --reasoning-budget; --reasoning-budget-message"
+)
+# Like the probe: every spelling of one option maps to that option's help block.
+FLAGS = {
+ flag: ", ".join(group.split())
+ + " (env: LLAMA_ARG_"
+ + group.split()[-1].lstrip("-").upper().replace("-", "_")
+ + ")"
+ for group in _OPTIONS.split("; ")
+ for flag in group.split()
+}
+SWITCHES = {
+ "-kvo",
+ "--kv-offload",
+ "-nkvo",
+ "--no-kv-offload",
+ "--no-mmproj",
+ "--mmproj-auto",
+ "--no-mmproj-auto",
+ "-sp",
+ "--special",
+ "-cb",
+ "--cont-batching",
+ "-nocb",
+ "--no-cont-batching",
+ "--warmup",
+ "--no-warmup",
+ "--mmproj-offload",
+ "--no-mmproj-offload",
+ "--jinja",
+ "--no-jinja",
+ "--cache-prompt",
+ "--no-cache-prompt",
+}
+
+
+def source(ini, section = None):
+ return {"version": 1, "mode": "custom", "ini": ini, "section": section}
+
+
+def compile_ini(ini, section = None):
+ return compile_custom_config(source(ini, section), FLAGS, SWITCHES)
+
+
+def test_preset_grammar_compiles_to_argv_and_defaults():
+ compiled = compile_ini(
+ "; comment\n[*]\nc = 8192\nngl=-1\n[qwen]\nc=128000 # wins\nfa=on\nwarmup=false\n"
+ "mmproj-offload=off\nnp=4\nm=/ignored.gguf\ntemp=1\ntop-k=20",
+ "qwen",
+ )
+ assert compiled.argv == (
+ "-c",
+ "128000",
+ "-ngl",
+ "-1",
+ "-fa",
+ "on",
+ "--no-warmup",
+ "--no-mmproj-offload",
+ "--temp",
+ "1",
+ "--top-k",
+ "20",
+ )
+ assert compiled.n_parallel == 4
+ assert compiled.request_defaults == {"temperature": 1.0, "top_k": 20}
+ assert compiled.diagnostics and not compiled.cpu_only
+
+
+ "ini,section,message",
+ [
+ ("[a]\nc=1\n[b]\nc=2", None, "Select which"),
+ ("c=1", "missing", "does not exist"),
+ ("bogus=1", None, "not an option"),
+ ("host=0.0.0.0", None, "managed by Studio"),
+ ("np=65", None, "between"),
+ ("jinja=false", None, "tool calling"),
+ ("warmup=maybe", None, "true or false"),
+ ("temp=hot", None, "number"),
+ ("not an entry", None, "line 1"),
+ ],
+)
+def test_refusals_name_the_key_not_the_value(ini, section, message):
+ with pytest.raises(CustomConfigError, match = message):
+ compile_ini(ini, section)
+
+
[email protected]("ini", ["c=1024\nctx-size=2048", "warmup=true\nno-warmup=true"])
+def test_two_spellings_of_one_option_are_refused(ini):
+ with pytest.raises(CustomConfigError, match = "set the same option"):
+ compile_ini(ini)
+
+
+def test_sampling_aliases_and_router_keys():
+ compiled = compile_ini(
+ "temperature=0.2\nload-on-startup=true\nstop-timeout=5\ndedup-cache-models=1"
+ )
+ assert compiled.request_defaults == {"temperature": 0.2}
+ assert compiled.argv == ("--temperature", "0.2")
+
+
[email protected]("ini", ["top-k=180", "temp=nan", "top-p=inf", "temp=-1"])
+def test_defaults_outside_the_chat_schema_are_refused(ini):
+ with pytest.raises(CustomConfigError, match = "between"):
+ compile_ini(ini)
+
+
+ "ini,argv",
+ [
+ ("special=false", ()),
+ ("special=true", ("--special",)),
+ ("nocb=false", ("--cont-batching",)),
+ ("nocb=true", ("-nocb",)),
+ ("cont-batching=off", ("--no-cont-batching",)),
+ ("no-warmup=true", ("--no-warmup",)),
+ ("nkvo=false", ("--kv-offload",)),
+ ("nkvo=true", ("-nkvo",)),
+ ("LLAMA_ARG_NO_KV_OFFLOAD=true", ("--kv-offload",)),
+ ("LLAMA_ARG_NO_KV_OFFLOAD=false", ("--no-kv-offload",)),
+ ],
+)
+def test_switches_follow_upstream_preset_semantics(ini, argv):
+ assert compile_ini(ini).argv == argv
+
+
+def test_env_variable_keys_resolve_like_upstream_presets():
+ flags = {**FLAGS, "-c": "-c, --ctx-size N (env: LLAMA_ARG_CTX_SIZE)"}
+ flags["--ctx-size"] = flags["-c"]
+ compiled = compile_custom_config(source("LLAMA_ARG_CTX_SIZE=4096"), flags, SWITCHES)
+ assert compiled.argv == ("--ctx-size", "4096")
+
+
+def test_unprobed_binary_is_refused():
+ with pytest.raises(CustomConfigError, match = "did not report"):
+ compile_custom_config(source("c=1"), {}, set())
+
+
+ "value",
+ [
+ {"version": 2, "mode": "custom", "ini": "c=1"},
+ {"version": 1, "mode": "custom", "ini": " "},
+ {"version": 1, "mode": "custom", "ini": "c=1\x00"},
+ {"version": 1, "mode": "custom", "ini": "x" * (64 * 1024 + 1)},
+ {"version": 1, "mode": "custom", "ini": "c=1", "section": "a]b"},
+ {"version": 1, "mode": "managed", "ini": "c=1"},
+ ],
+)
+def test_wire_validation(value):
+ with pytest.raises(CustomConfigError):
+ parse_config_source(value)
+
+
+ "ini,cpu_only",
+ [
+ ("dev=none", True),
+ ("dev=none\nmmdev=none", True),
+ ("dev=none\nmd=d.gguf\ndevd=none", True),
+ ("ngl=0", False), # op offload and the projector still use the GPU
+ ("dev=none\nmmdev=CUDA0", False),
+ ("dev=none\nmd=d.gguf", False),
+ ("dev=none\nspec-draft-model=d.gguf", False),
+ ("dev=none\nspec-draft-hf=org/d", False),
+ ("dev=none\nspec-draft-hf=org/d\ndevice-draft=none", True),
+ ],
+)
+def test_cpu_only_counts_every_companion(ini, cpu_only):
+ assert compile_ini(ini).cpu_only is cpu_only
+
+
+def test_a_section_overrides_global_through_another_alias():
+ compiled = compile_ini("[*]\nc=1024\n[large]\nctx-size=2048", "large")
+ assert compiled.argv == ("--ctx-size", "2048")
+ assert compiled.option("-c") == "2048"
+
+
+def test_unsectioned_keys_are_the_default_section_not_global():
+ ini = "c=1000\n[*]\nngl=-1\n[large]\nc=9000"
+ assert compile_ini(ini, "large").option("-c") == "9000"
+ assert compile_ini("fa=on\n[large]\nc=9000", "large").option("-fa") is None
+ assert compile_ini("fa=on\n[large]\nc=9000", "default").option("-fa") == "on"
+ assert compile_ini("c=1000\n[*]\nngl=-1").argv == ("-ngl", "-1", "-c", "1000")
+ with pytest.raises(CustomConfigError, match = "Select which"):
+ compile_ini("fa=on\n[large]\nc=9000")
+
+
+def launch(monkeypatch, tmp_path):
+ backend = llama.LlamaCppBackend(manages_processes = False)
+ model, binary = tmp_path / "selected.gguf", tmp_path / "llama-server"
+ model.touch()
+ binary.touch()
+ caps = {"help_probe_ok": True, "flags": FLAGS, "switch_flags": SWITCHES}
+ for name, value in {
+ "_find_llama_server_binary": lambda: str(binary),
+ "_exec_path_for_launch": lambda p: p,
+ "probe_server_capabilities": lambda *a, **k: caps,
+ "_binary_revision": lambda p: (str(p), 1),
+ "_cuda_sm_gate_error": lambda p: None,
+ "_arch_gate_survivors": lambda p: [],
+ "_non_chat_gguf_refusal_for_path": lambda *a: None,
+ "_gguf_path_is_diffusion": lambda *a: False,
+ "_spawn_is_stale": lambda: False,
+ "_find_free_port": lambda: 8123,
+ "_wait_for_health": lambda **k: True,
+ "_detect_audio_type_strict": lambda: None,
+ "_llama_server_env_for_binary": lambda p: {
+ "PATH": "libs",
+ "LLAMA_ARG_CTX_SIZE": "1",
+ "LLAMA_API_KEY": "x",
+ "KEEP": "1",
+ },
+ "_query_server_props": lambda: {
+ "total_slots": 2,
+ "default_generation_settings": {"n_ctx": 28000},
+ "modalities": {"vision": False},
+ },
+ }.items():
+ monkeypatch.setattr(backend, name, value)
+ monkeypatch.setattr(llama, "_metal_device_is_paravirtual", lambda: False, raising = False)
+ monkeypatch.setattr(
+ llama.LlamaCppBackend,
+ "_read_gguf_metadata",
+ lambda self, path: setattr(self, "_context_length", 128000),
+ )
+ captured = []
+
+ def spawn(cmd, env, **kwargs):
+ captured.append((list(cmd), dict(env)))
+ backend._process = SimpleNamespace(poll = lambda: None, pid = 1)
+ return True
+
+ monkeypatch.setattr(backend, "_start_llama_process", spawn)
+ monkeypatch.setattr(
+ backend, "_kill_process", lambda *a, **k: setattr(backend, "_process", None)
+ )
+ intent = llama.GgufLoadIntent(
+ model_identifier = "selected",
+ gguf_path = str(model),
+ n_ctx = 4096,
+ n_parallel = 8,
+ extra_args = ("--ctx-size", "99999"),
+ llama_cpp_config = source("[*]\nnp=2\nc=56000\nngl=-1\nno-warmup=true\ntemp=0.3"),
+ )
+ return SimpleNamespace(backend = backend, intent = intent, captured = captured)
+
+
+def test_custom_launch_runs_only_the_ini(launch):
+ assert launch.backend.load_model(launch.intent)
+ cmd, env = launch.captured[0]
+ assert cmd[cmd.index("--parallel") + 1] == "2"
+ assert "99999" not in cmd and "--jinja" in cmd
+ assert cmd[cmd.index("-c") + 1] == "56000" and "--no-warmup" in cmd
+ assert not any(k.startswith("LLAMA_ARG_") or k == "LLAMA_API_KEY" for k in env)
+ assert env["KEEP"] == "1"
+ summary = launch.backend.llama_cpp_config_summary
+ assert summary["request_defaults"] == {"temperature": 0.3}
+ assert launch.backend.extra_args == []
+ # The duplicate-load fast path only reuses a probed server.
+ assert launch.backend._audio_probed is True
+
+
+def test_cpu_only_custom_load_still_masks_unsupported_rocm_cards(launch, monkeypatch):
+ from dataclasses import replace
+
+ masked = []
+ monkeypatch.setattr(launch.backend, "_arch_gate_survivors", lambda p: [1])
+ monkeypatch.setattr(
+ launch.backend, "_emit_child_gpu_visibility", lambda env, ids, **k: masked.append(ids)
+ )
+ cpu = replace(launch.intent, llama_cpp_config = parse_config_source(source("dev=none")))
+ assert launch.backend.load_model(cpu)
+ assert masked == ["1"]
+ # The GPU arbiter leaves image/video work alone for a zero-VRAM resident server.
+ assert launch.backend.holds_no_vram
+
+
+def test_custom_launch_reports_the_presets_reasoning_budget(launch):
+ from dataclasses import replace
+
+ launch.backend._reasoning_budget = launch.backend._requested_reasoning_budget = 512
+ launch.backend._reasoning_budget_message = "stale"
+ assert launch.backend.load_model(launch.intent)
+ assert (launch.backend._reasoning_budget, launch.backend._reasoning_budget_message) == (-1, "")
+ ini = parse_config_source(source("reasoning-budget=256\nreasoning-budget-message=done"))
+ assert launch.backend.load_model(replace(launch.intent, llama_cpp_config = ini))
+ assert launch.backend._requested_reasoning_budget == 256
+ assert launch.backend._reasoning_budget_message == "done"
+
+
+ "log,ini,total",
+ [
+ ("", "np=2\nc=56000", 56000), # split cache: 2 slots x 28000
+ ("srv init: n_slots = 2, n_ctx_slot = 28000, kv_unified = 'true'", "np=2", 28000),
+ ("", "c=56000", 28000), # np auto: llama-server unifies the cache
+ # kv-unified-per-slot with an explicit -c: the pinned pool, not slots x per-slot.
+ ("kv_unified = 'true'", "np=2\nc=30000\nkv-unified-per-slot=28000", 30000),
+ ],
+)
+def test_published_kv_capacity_follows_the_unified_cache(launch, log, ini, total):
+ from dataclasses import replace
+
+ launch.backend._stdout_lines = [log] if log else []
+ intent = replace(launch.intent, llama_cpp_config = parse_config_source(source(ini)))
+ assert launch.backend.load_model(intent)
+ assert launch.backend._kv_cache_context_total == total
+
+
+def test_gpu_custom_load_holds_vram(launch):
+ assert launch.backend.load_model(launch.intent)
+ assert not launch.backend.holds_no_vram
+
+
+def test_audio_probe_runs_after_the_load_lock_is_released(launch, monkeypatch):
+ held = []
+ monkeypatch.setattr(launch.backend, "_detect_audio_type_strict", lambda: "snac")
+ monkeypatch.setattr(
+ launch.backend,
+ "_apply_detected_audio",
+ lambda *a: held.append(launch.backend._lock.locked()) or True,
+ )
+ assert launch.backend.load_model(launch.intent)
+ assert held == [False]
+
+
+def test_a_downloaded_diffusion_gguf_is_refused_before_launch(launch, monkeypatch):
+ monkeypatch.setattr(launch.backend, "_gguf_path_is_diffusion", lambda *a: True)
+ with pytest.raises(CustomConfigError, match = "diffusion"):
+ launch.backend.load_model(launch.intent)
+ assert launch.captured == []
+
+
[email protected]("ini", ["no-mmproj=true", "mmproj-auto=false"])
+def test_an_ini_that_disables_the_projector_launches_text_only(launch, monkeypatch, tmp_path, ini):
+ from dataclasses import replace
+
+ projector = tmp_path / "mmproj.gguf"
+ projector.touch()
+ monkeypatch.setattr(launch.backend, "_resolve_launch_mmproj_path", lambda **k: str(projector))
+ launch.backend._auto_tensor_split_emitted = "3,1"
+ intent = replace(
+ launch.intent,
+ mmproj_path = str(projector),
+ llama_cpp_config = parse_config_source(source(ini)),
+ )
+ assert launch.backend.load_model(intent)
+ assert "--mmproj" not in launch.captured[-1][0]
+ assert launch.backend._auto_tensor_split_emitted is None
+
+
[email protected]("ini,downloads", [("no-mmproj=true", 0), ("c=4096", 1)])
+def test_projector_download_follows_the_ini_not_the_vision_toggle(
+ launch, monkeypatch, ini, downloads
+):
+ from dataclasses import replace
+
+ fetched = []
+ monkeypatch.setattr(launch.backend, "_download_mmproj", lambda **k: fetched.append(k) or None)
+ intent = replace(
+ launch.intent,
+ is_vision = True,
+ disable_vision = True, # an audio-only encoder survives this; its header decides
+ hf_repo = "org/model-GGUF",
+ llama_cpp_config = parse_config_source(source(ini)),
+ )
+ assert launch.backend.load_model(intent)
+ assert len(fetched) == downloads
+
+
+def test_force_reload_never_adopts_a_resident_custom_server(launch):
+ from dataclasses import replace
+
+ assert launch.backend.load_model(launch.intent)
+ assert launch.backend._custom_runtime_matches(launch.intent)
+ assert not launch.backend._custom_runtime_matches(replace(launch.intent, force_reload = True))
+
+
+def test_bad_config_never_spawns(launch):
+ from dataclasses import replace
+
+ bad = replace(launch.intent, llama_cpp_config = parse_config_source(source("bogus=1")))
+ with pytest.raises(CustomConfigError):
+ launch.backend.load_model(bad)
+ assert launch.captured == []
+ assert launch.backend.llama_cpp_config_summary is None
+
+
+def test_start_failure_carries_llama_servers_reason(launch, monkeypatch):
+ launch.backend._stdout_lines = ["load_model: error: invalid device: ROCm1"]
+ monkeypatch.setattr(launch.backend, "_wait_for_health", lambda **k: False)
+ monkeypatch.setattr(launch.backend, "_arch_gate_survivors", lambda p: [1])
+ monkeypatch.setattr(launch.backend, "_emit_child_gpu_visibility", lambda *a, **k: None)
+ with pytest.raises(CustomConfigError, match = "invalid device: ROCm1.*0 to 0"):
+ launch.backend.load_model(launch.intent)
diff --git a/studio/backend/tests/test_llama_custom_routes.py b/studio/backend/tests/test_llama_custom_routes.py
new file mode 100644
index 00000000000..203db0b2e4a
--- /dev/null
+++ b/studio/backend/tests/test_llama_custom_routes.py
@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Custom configuration persistence, preflight and request-default boundaries."""
+
+from types import SimpleNamespace
+from pathlib import Path
+import sys
+
+import pytest
+from fastapi import HTTPException
+from pydantic import ValidationError
+
+from models.inference import ChatCompletionRequest, LoadRequest, ValidateModelRequest
+from routes import inference as routes
+from utils import openai_auto_switch_settings as settings
+from utils.inference import inference_config as sampling
+from core.inference.llama_custom_config import CustomConfigError
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from test_openai_auto_switch import _put, override_store # noqa: E402, F401
+
+
+CUSTOM = {"version": 1, "mode": "custom", "ini": "[*]\nnp=1\nc=56000\nfit=off", "section": None}
+MANAGED = {"version": 1, "mode": "managed"}
+MODEL = "unsloth/example-GGUF:Q4_K_M"
+
+
+def test_override_roundtrip_and_managed_reset(override_store):
+ _put(MODEL, llama_cpp_config = CUSTOM, llama_extra_args = ["--temp", "0.7"])
+ row = settings.get_model_override(MODEL)
+ assert row["llama_cpp_config"] == CUSTOM
+ assert settings.model_override_load_kwargs(row, is_gguf = True) == {"llama_cpp_config": CUSTOM}
+ _put(MODEL, llama_cpp_config = MANAGED)
+ row = settings.get_model_override(MODEL)
+ assert row["llama_cpp_config"] == MANAGED
+ assert row["llama_extra_args"] == ["--temp", "0.7"]
+ assert settings.model_override_load_kwargs(row, is_gguf = True)["llama_extra_args"] == [
+ "--temp",
+ "0.7",
+ ]
+
+
+def test_older_client_save_preserves_custom_source(override_store):
+ _put(MODEL, llama_cpp_config = CUSTOM)
+ _put(MODEL, custom_context_length = 8192)
+ assert settings.get_model_override(MODEL)["llama_cpp_config"] == CUSTOM
+
+
+def test_quant_reset_stops_bare_model_fallback(override_store):
+ _put("unsloth/example-GGUF", llama_cpp_config = CUSTOM)
+ _put(MODEL, llama_cpp_config = MANAGED)
+ _, row = settings.resolve_override_for_load("unsloth/example-GGUF", variant = "Q4_K_M")
+ assert row["llama_cpp_config"] == MANAGED
+
+
[email protected]("schema", [LoadRequest, ValidateModelRequest])
+def test_source_schema_fails_closed(schema):
+ with pytest.raises(ValidationError):
+ schema(model_path = "model.gguf", llama_cpp_config = {"version": 2, "mode": "custom"})
+
+
+def test_corrupt_saved_source_does_not_become_managed():
+ with pytest.raises(ValueError):
+ settings.normalize_model_override({"llama_cpp_config": {"version": 1, "mode": "custom"}})
+
+
+def test_source_inheritance_is_model_scoped_and_reset_is_explicit(monkeypatch):
+ monkeypatch.setattr(settings, "resolve_override_for_load", lambda *a, **k: (None, {}))
+ resident = SimpleNamespace(
+ last_load_intent = SimpleNamespace(
+ model_identifier = "/models/Original.gguf", hf_variant = None, llama_cpp_config = CUSTOM
+ )
+ )
+ monkeypatch.setattr(routes, "get_llama_cpp_backend", lambda: resident)
+ same = routes._resolve_llama_cpp_config(LoadRequest(model_path = "/models/Original.gguf"))
+ other = routes._resolve_llama_cpp_config(LoadRequest(model_path = "/models/Other.gguf"))
+ reset = routes._resolve_llama_cpp_config(
+ LoadRequest(model_path = "/models/Original.gguf", llama_cpp_config = MANAGED)
+ )
+ assert same.llama_cpp_config == CUSTOM
+ assert other.llama_cpp_config is None
+ assert reset.llama_cpp_config == MANAGED
+
+
+def test_custom_ignores_legacy_extras():
+ request = LoadRequest(
+ model_path = "model.gguf", llama_cpp_config = CUSTOM, llama_extra_args = ["--ctx-size", "128000"]
+ )
+ assert (
+ routes._resolve_inherited_extra_args(
+ request, SimpleNamespace(is_gguf = True), "model.gguf", request.llama_extra_args
+ )
+ == []
+ )
+
+
+def preset_backend(monkeypatch):
+ backend = SimpleNamespace(
+ model_identifier = "model.gguf",
+ llama_cpp_config_summary = {
+ "mode": "custom",
+ "request_defaults": {"temperature": 0.25, "top_k": 80},
+ },
+ )
+ monkeypatch.setattr(routes, "get_llama_cpp_backend", lambda: backend)
+ monkeypatch.setattr(
+ sampling, "_recommended_sampling", lambda _: {"temperature": 0.8, "top_k": 20, "top_p": 0.9}
+ )
+ for field in sampling.SAMPLING_FIELD_NAMES:
+ monkeypatch.delenv(sampling._SAMPLING_FIELDS[field][0], raising = False)
+ return backend
+
+
+def _chat(**fields):
+ return ChatCompletionRequest(messages = [{"role": "user", "content": "hi"}], **fields)
+
+
+def test_ini_sampling_ranks_between_client_and_recommendation(preset_backend):
+ payload = _chat(top_k = 5)
+ routes._fill_recommended_sampling_openai(payload, "model.gguf")
+ assert (payload.temperature, payload.top_k, payload.top_p) == (0.25, 5, 0.9)
+
+
+def test_operator_pin_still_wins(preset_backend, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", "1.5")
+ payload = _chat()
+ routes._fill_recommended_sampling_openai(payload, "model.gguf")
+ assert payload.temperature == 1.5
+
+
+def test_ini_defaults_never_leak_to_another_model(preset_backend):
+ payload = _chat()
+ routes._fill_recommended_sampling_openai(payload, "other.gguf")
+ assert payload.temperature == 0.8
+
+
+def test_raw_completions_get_ini_values(preset_backend):
+ body = {"prompt": "hi"}
+ routes._fill_recommended_sampling_completions(body, "model.gguf")
+ assert (body["temperature"], body["top_k"]) == (0.25, 80)
+
+
+async def test_preflight_refuses_a_bad_config_before_any_eviction(monkeypatch):
+ calls = []
+
+ def prepare(intent):
+ calls.append(intent.llama_cpp_config.ini)
+ raise CustomConfigError("'bogus' is not an option of the selected llama-server")
+
+ monkeypatch.setattr(
+ routes, "get_llama_cpp_backend", lambda: SimpleNamespace(prepare_custom_config = prepare)
+ )
+ monkeypatch.setattr(routes, "_classify_diffusion_gguf", lambda config: False)
+ request = LoadRequest(model_path = "model.gguf", llama_cpp_config = CUSTOM)
+ config = SimpleNamespace(is_gguf = True, identifier = "model.gguf")
+ with pytest.raises(HTTPException) as caught:
+ await routes._preflight_custom_llama_config(request, config)
+ assert caught.value.status_code == 400 and "bogus" in caught.value.detail
+ assert calls == [CUSTOM["ini"]]
+
+
+async def test_preflight_refuses_non_gguf():
+ request = LoadRequest(model_path = "org/model", llama_cpp_config = CUSTOM)
+ with pytest.raises(HTTPException):
+ await routes._preflight_custom_llama_config(request, SimpleNamespace(is_gguf = False))
diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py
index 4985fa95ebf..ee75f428dfb 100644
--- a/studio/backend/tests/test_llama_server_args.py
+++ b/studio/backend/tests/test_llama_server_args.py
@@ -192,6 +192,7 @@ def test_a_bare_positional_is_rejected():
"--path",
"--api-prefix",
"--reuse-port",
+ "--rpc",
# Auth / TLS
"--api-key",
"--api-key-file",
@@ -1403,6 +1404,7 @@ def test_every_denied_flag_with_a_twin_in_the_help_is_scrubbed():
("LLAMA_ARG_HF_REPO", "--hf-repo"),
("LLAMA_ARG_HOST", "--host"),
("LLAMA_ARG_PORT", "--port"),
+ ("LLAMA_ARG_RPC", "--rpc"),
("LLAMA_ARG_N_PARALLEL", "--parallel"),
("LLAMA_ARG_SSL_KEY_FILE", "--ssl-key-file"),
("LLAMA_ARG_SSL_CERT_FILE", "--ssl-cert-file"),
diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.py
index c91919db76b..4f6f110111b 100644
--- a/studio/backend/utils/inference/inference_config.py
+++ b/studio/backend/utils/inference/inference_config.py
@@ -204,8 +204,9 @@ def resolve_effective_sampling(
explicit: Dict[str, Any],
*,
fill_defaults: bool = True,
+ preset_defaults: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
- """Resolve the effective sampling params for a request. ``explicit`` maps each field in :data:`SAMPLING_FIELD_NAMES` to the client-sent value, or ``None`` when the client omitted it. Precedence, highest first: an operator ``UNSLOTH_SAMPLING_*`` pin, the client's explicit value, the per-model recommendation, then the static schema default. When ``fill_defaults`` is False a field with none of the first three is omitted rather than set to the static default, so a raw proxy body (``/v1/completions``) keeps llama-server's own default for that field."""
+ """Resolve the effective sampling params for a request. ``explicit`` maps each field in :data:`SAMPLING_FIELD_NAMES` to the client-sent value, or ``None`` when the client omitted it. Precedence, highest first: an operator ``UNSLOTH_SAMPLING_*`` pin, the client's explicit value, a custom llama.cpp INI's value (``preset_defaults``), the per-model recommendation, then the static schema default. When ``fill_defaults`` is False a field with none of the first four is omitted rather than set to the static default, so a raw proxy body (``/v1/completions``) keeps llama-server's own default for that field."""
recommended = _recommended_sampling(model_id or "")
effective: Dict[str, Any] = {}
for field, (_env, default, _lo, _hi, _int) in _SAMPLING_FIELDS.items():
@@ -214,6 +215,8 @@ def resolve_effective_sampling(
effective[field] = override
elif explicit.get(field) is not None:
effective[field] = explicit[field]
+ elif preset_defaults and field in preset_defaults:
+ effective[field] = preset_defaults[field]
elif field in recommended:
effective[field] = recommended[field]
elif fill_defaults:
diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py
index 576713f8dd7..791c659b0c3 100644
--- a/studio/backend/utils/openai_auto_switch_settings.py
+++ b/studio/backend/utils/openai_auto_switch_settings.py
@@ -427,6 +427,12 @@ def normalize_model_override(
"""Validate one per-model launch config, dropping anything unusable. Silently drops rather than raising: an override is a convenience mirror of the UI's config, so one stale field (a KV dtype this llama.cpp build lost, a GPU id from another host) must not block persisting the rest or fail the API load that reads it. ``validate_extra_args`` is the caller's job, since it lives in the llama_server_args allow-list module this one must not import. ``keep_empty_extra_args`` keeps an explicit empty list, the difference between "this model has no launch flags" and "nothing is stored for this model": the same thing everywhere except under a fallback, where a quant whose row is gone reads the bare repository row instead and a cleared box would come back holding whatever that legacy row carries."""
entry: dict[str, Any] = {}
+ if payload.get("llama_cpp_config") is not None:
+ from core.inference.llama_custom_config import parse_config_source
+
+ # A broken custom configuration must not silently become a managed load.
+ entry["llama_cpp_config"] = parse_config_source(payload["llama_cpp_config"]).to_wire()
+
extra_args = payload.get("llama_extra_args")
if isinstance(extra_args, (list, tuple)) and extra_args:
entry["llama_extra_args"] = [str(arg) for arg in extra_args]
@@ -576,6 +582,16 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
return {}
kwargs: dict[str, Any] = {}
+ if is_gguf and override.get("llama_cpp_config") is not None:
+ from core.inference.llama_custom_config import parse_config_source
+
+ custom = parse_config_source(override["llama_cpp_config"])
+ kwargs["llama_cpp_config"] = custom.to_wire()
+ if custom.mode == "custom":
+ if override.get("disable_vision") is not None:
+ kwargs["disable_vision"] = override["disable_vision"]
+ return kwargs
+
max_seq_length = resolve_fit_max_seq_length(override, is_gguf = is_gguf)
if max_seq_length is not None:
kwargs["max_seq_length"] = max_seq_length
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index aecec6f0fe9..5d685d84ec5 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+// eslint-disable-next-line no-restricted-imports -- The picker barrel imports chat; this payload helper is import-free.
+import { llamaCppConfigPayload } from "@/features/model-picker/model-config/llama-cpp-config";
import { attachedMediaUnavailableReason } from "../lib/attached-media-gate";
import { externalModelLabel } from "../lib/external-model-label";
import { mlxRuntimeStateFrom } from "../lib/mlx-runtime-state";
@@ -219,9 +221,11 @@ import {
type PendingImageEditReference,
type RagAutoInject,
GPU_LAYERS_AUTO,
- loadedGpuMemoryFields,
+ managedGpuMemoryFields,
reconcilePersistedGpuIds,
- resolveLoadedSpeculativeSettings,
+ loadedLlamaCppConfigFields,
+ managedKvCacheFields,
+ managedSpeculativeSettings,
resolveSpeculativeSettingsForLoad,
persistGpuMemoryModeOnLoad,
resolvePreserveThinkingOnLoad,
@@ -3497,6 +3501,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
// it disagrees with the launch.
...serverTuningLoadPayload(config),
// Checked with the same arguments the load sends, or a list the backend refuses would pass this gate.
+ ...llamaCppConfigPayload(config.llamaCppConfig, { isDiffusion }),
...(resolvedExtraArgs !== undefined
? { llama_extra_args: resolvedExtraArgs ?? [] }
: {}),
@@ -3561,6 +3566,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
...serverTuningLoadPayload(config),
// Remembered pass-through args: nothing is resident at startup to inherit them from.
// Undefined predates the field; a cleared list is an explicit none.
+ ...llamaCppConfigPayload(config.llamaCppConfig, { isDiffusion }),
...(resolvedExtraArgs !== undefined
? { llama_extra_args: resolvedExtraArgs ?? [] }
: {}),
@@ -3656,8 +3662,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
preserveThinking: resolvePreserveThinkingOnLoad(loadResp),
supportsTools: loadResp.supports_tools ?? false,
...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false),
- kvCacheDtype: loadResp.cache_type_kv ?? null,
- loadedKvCacheDtype: loadResp.cache_type_kv ?? null,
+ ...managedKvCacheFields(loadResp),
...mlxRuntimeStateFrom(loadResp),
// Click-time value, not the resolved backend echo (see performLoad).
nParallel: committedSlots,
@@ -3693,6 +3698,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
...committedServerTuningState(config, loadResp.is_diffusion ?? false),
// What this launch is running, for a later rollback: the status applier cannot seed it while
// the model-loading lease is held, and a failed switch would restore the wrong args.
+ ...loadedLlamaCppConfigFields(loadResp, config.llamaCppConfig),
loadedLlamaExtraArgs:
loadResp.requested_llama_extra_args !== undefined
? (loadResp.requested_llama_extra_args ?? [])
@@ -3704,7 +3710,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
// loaded projector, and the next Apply would send it.
disableVision: loadResp.disable_vision ?? false,
loadedVisionDisabledByUser: loadResp.vision_disabled_by_user ?? false,
- ...loadedGpuMemoryFields(loadResp),
+ ...managedGpuMemoryFields(loadResp),
loadedCustomContextLength: keepCustomCtx,
defaultChatTemplate: loadResp.chat_template ?? null,
chatTemplateOverride: effectiveChatTemplateOverride,
@@ -3715,7 +3721,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
mmprojFallbackReason: loadResp.mmproj_fallback_reason ?? null,
loadedIsDiffusion: loadResp.is_diffusion ?? false,
activeModelIsLocal: loadResp.is_local_model ?? false,
- ...resolveLoadedSpeculativeSettings(loadResp),
+ ...managedSpeculativeSettings(loadResp),
});
} else {
useChatRuntimeStore.setState({
@@ -3728,8 +3734,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
preserveThinking: resolvePreserveThinkingOnLoad(loadResp),
supportsTools: loadResp.supports_tools ?? false,
...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false),
- kvCacheDtype: loadResp.cache_type_kv ?? null,
- loadedKvCacheDtype: loadResp.cache_type_kv ?? null,
+ ...managedKvCacheFields(loadResp),
...mlxRuntimeStateFrom(loadResp),
nParallel: committedSlots,
loadedNParallel: committedSlots,
@@ -3747,6 +3752,9 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
// Same reason, and the baseline must clear so a rollback to THIS model does not resend a
// GGUF's arguments.
loadedLlamaExtraArgs: null,
+ llamaCppConfig: undefined,
+ loadedLlamaCppConfig: null,
+ llamaCppConfigSummary: null,
tensorParallel: loadResp.tensor_parallel ?? false,
loadedTensorParallel: loadResp.tensor_parallel ?? false,
loadedDisableVision: loadResp.disable_vision ?? false,
@@ -3756,7 +3764,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
loadedVisionDisabledByUser:
loadResp.vision_disabled_by_user ?? false,
// Non-GGUF response: clears any stale GPU baseline a prior manual-GPU GGUF load left.
- ...loadedGpuMemoryFields(loadResp),
+ ...managedGpuMemoryFields(loadResp),
defaultChatTemplate: loadResp.chat_template ?? null,
chatTemplateOverride: effectiveChatTemplateOverride,
loadedChatTemplateOverride: effectiveChatTemplateOverride,
@@ -3768,7 +3776,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
...loadedContextFields(loadResp),
activeNativePathToken: null,
activeNativePathExpiresAtMs: null,
- ...resolveLoadedSpeculativeSettings(loadResp),
+ ...managedSpeculativeSettings(loadResp),
loadedIsMultimodal: isMultimodalResponse(loadResp),
mmprojFallbackReason: loadResp.mmproj_fallback_reason ?? null,
loadedIsDiffusion: loadResp.is_diffusion ?? false,
@@ -4071,8 +4079,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
preserveThinking: resolvePreserveThinkingOnLoad(loadResp),
supportsTools: loadResp.supports_tools ?? false,
...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false),
- kvCacheDtype: loadResp.cache_type_kv ?? null,
- loadedKvCacheDtype: loadResp.cache_type_kv ?? null,
+ ...managedKvCacheFields(loadResp),
...mlxRuntimeStateFrom(loadResp),
// The request above omits n_parallel: a staged override would read as applied and be re-sent
// by the next Apply.
@@ -4096,7 +4103,9 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
disableVision: loadResp.disable_vision ?? false,
loadedVisionDisabledByUser:
loadResp.vision_disabled_by_user ?? false,
- ...loadedGpuMemoryFields(loadResp),
+ ...managedGpuMemoryFields(loadResp),
+ // The request omits llama_cpp_config, so a saved custom source may be what launched.
+ ...loadedLlamaCppConfigFields(loadResp, undefined),
// Drives the GPU Memory controls' diffusion gate; set on every load path so the gate cannot read stale.
loadedIsDiffusion: loadResp.is_diffusion ?? false,
defaultChatTemplate: loadResp.chat_template ?? null,
@@ -4104,7 +4113,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
loadedIsMultimodal: isMultimodalResponse(loadResp),
mmprojFallbackReason: loadResp.mmproj_fallback_reason ?? null,
activeModelIsLocal: loadResp.is_local_model ?? false,
- ...resolveLoadedSpeculativeSettings(loadResp),
+ ...managedSpeculativeSettings(loadResp),
});
recordLastLocalModelLoad({
id: DEFAULT_CHAT_MODEL_REPO,
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index ec4d23f1212..a6ced08c1ce 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -392,6 +392,9 @@ export async function validateModel(
reasoning_budget_message: payload.reasoning_budget_message ?? "",
// A --ctx-size or cache override in here changes the estimate, so a preflight that dropped them
// would approve a different command from the one that runs.
+ ...(payload.llama_cpp_config !== undefined
+ ? { llama_cpp_config: payload.llama_cpp_config }
+ : {}),
...(payload.llama_extra_args !== undefined
? // biome-ignore lint/style/useNamingConvention: API schema
{ llama_extra_args: payload.llama_extra_args }
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index de0d1f8efe6..ffad8d70c9e 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+// eslint-disable-next-line no-restricted-imports -- The picker barrel imports chat; this payload helper is import-free.
+import { llamaCppConfigPayload } from "@/features/model-picker/model-config/llama-cpp-config";
import {
CACHE_MISS_DOWNLOAD_DESCRIPTION,
EMPTY_CACHE_MISS_WATCH,
@@ -97,7 +99,10 @@ import {
import {
GPU_LAYERS_AUTO,
isLocalModelPath,
- loadedGpuMemoryFields,
+ loadedLlamaCppConfigFields,
+ managedGpuMemoryFields,
+ managedKvCacheFields,
+ managedSpeculativeSettings,
noteLoadedModelReasoningMode,
persistGpuMemoryModeOnLoad,
pinHoldsLiveEffort,
@@ -2095,6 +2100,12 @@ export function useChatModelRuntime() {
// them", and the route preserves the stored flags when the field is omitted, so a fallback
// would clear flags the user set elsewhere.
const loadLlamaExtraArgs = pendingLoadConfig?.llamaExtraArgs;
+ const loadLlamaCppConfig =
+ pendingLoadConfig?.llamaCppConfig ??
+ (currentCheckpoint === modelId &&
+ previousVariant === (ggufVariant ?? null)
+ ? stateBeforeUnload.llamaCppConfig
+ : undefined);
let loadNBatch =
pendingLoadConfig?.nBatch ?? stateBeforeUnload.nBatch;
let loadNUbatch =
@@ -2221,6 +2232,9 @@ export function useChatModelRuntime() {
// The same list the load below sends: a --ctx-size or cache override changes the memory this
// preflight estimates, so omitting it approves a different command and /load then refuses
// the target with the real arguments.
+ ...llamaCppConfigPayload(loadLlamaCppConfig, {
+ isDiffusion: targetIsDiffusion,
+ }),
...(!targetIsDiffusion && loadLlamaExtraArgs !== undefined
? { llama_extra_args: loadLlamaExtraArgs ?? [] }
: {}),
@@ -2318,10 +2332,11 @@ export function useChatModelRuntime() {
// preflight, so a rejected target truncates replies for a model that never loads. Idle,
// unload first and free VRAM early.
if (!forceCancelActive) {
- await unloadModel({ model_path: currentCheckpoint });
- // Only a real /unload removes the resident model. The forced path leaves
- // it to /load, so cancellation must not treat it as gone.
- loadRun.residentModelUnloaded = true;
+ if (loadLlamaCppConfig?.mode !== "custom") {
+ await unloadModel({ model_path: currentCheckpoint });
+ // Custom mode leaves replacement to /load, which preflights before evicting.
+ loadRun.residentModelUnloaded = true;
+ }
}
// Set either way: /load can still leave no model resident, and an unneeded rollback hits
// already_loaded before the gate.
@@ -2495,6 +2510,11 @@ export function useChatModelRuntime() {
isGguf && !targetIsDiffusion ? loadReasoningBudgetMessage : "",
// Sent only once known, and [] is the explicit "launch with none": the flags are llama-server's,
// so neither a transformers load nor a diffusion GGUF carries them.
+ ...(isGguf
+ ? llamaCppConfigPayload(loadLlamaCppConfig, {
+ isDiffusion: targetIsDiffusion,
+ })
+ : {}),
...(isGguf && !targetIsDiffusion && loadLlamaExtraArgs !== undefined
? { llama_extra_args: loadLlamaExtraArgs ?? [] }
: {}),
@@ -2596,11 +2616,7 @@ export function useChatModelRuntime() {
}
}
}
- const loadedKv = loadResponse.cache_type_kv ?? null;
const loadedTp = loadResponse.tensor_parallel ?? false;
- const loadedSpec = normalizeSpeculativeType(
- loadResponse.speculative_type,
- );
const committedSlots =
((loadResponse.is_gguf ?? false) && !(loadResponse.is_diffusion ?? false)) ||
(loadResponse.is_mlx ?? false)
@@ -2686,8 +2702,7 @@ export function useChatModelRuntime() {
codeToolsEnabled: stateBeforeUnload.codeToolsEnabled,
}
: resolveToolsEnabledOnLoad(supportsTools)),
- kvCacheDtype: loadedKv,
- loadedKvCacheDtype: loadedKv,
+ ...managedKvCacheFields(loadResponse),
...mlxRuntimeStateFrom(loadResponse),
tensorParallel: loadedTp,
loadedTensorParallel: loadedTp,
@@ -2699,11 +2714,8 @@ export function useChatModelRuntime() {
// Set alongside loadedIsMultimodal so the composer can say WHY images are unavailable.
loadedVisionDisabledByUser:
loadResponse.vision_disabled_by_user ?? false,
- ...loadedGpuMemoryFields(loadResponse),
- speculativeType: loadedSpec,
- loadedSpeculativeType: loadedSpec,
- specDraftNMax: loadResponse.spec_draft_n_max ?? null,
- loadedSpecDraftNMax: loadResponse.spec_draft_n_max ?? null,
+ ...managedGpuMemoryFields(loadResponse),
+ ...managedSpeculativeSettings(loadResponse),
// Keep the click-time value: the echo is the resolved count, and adopting it would pin a blank
// "server default" control.
nParallel: committedSlots,
@@ -2745,6 +2757,7 @@ export function useChatModelRuntime() {
// process's list, so the last thing we knew still holds unless this was a different model. An
// explicit empty list is recorded as empty, not null, since omitting the field is what makes /load
// inherit. The server's echo comes first, as the only account of what the launch carried.
+ ...loadedLlamaCppConfigFields(loadResponse, loadLlamaCppConfig),
loadedLlamaExtraArgs:
loadResponse.requested_llama_extra_args !== undefined
? (loadResponse.requested_llama_extra_args ?? [])
@@ -2885,6 +2898,9 @@ export function useChatModelRuntime() {
}),
// Explicit, unlike the batch pair above: the failed switch left the TARGET resident, so an
// omitted field here inherits across models, which the route refuses.
+ ...llamaCppConfigPayload(
+ rollbackState.loadedLlamaCppConfig ?? undefined,
+ ),
...(rollbackState.loadedLlamaExtraArgs != null
? { llama_extra_args: rollbackState.loadedLlamaExtraArgs }
: {}),
@@ -2910,6 +2926,12 @@ export function useChatModelRuntime() {
rollbackResponse.speculative_type,
);
useChatRuntimeStore.setState({
+ llamaCppConfig:
+ rollbackState.loadedLlamaCppConfig ?? undefined,
+ loadedLlamaCppConfig: rollbackState.loadedLlamaCppConfig,
+ llamaCppConfigSummary:
+ rollbackResponse.llama_cpp_config_summary ??
+ rollbackState.llamaCppConfigSummary,
activeModelIsLocal: rollbackResponse.is_local_model ?? false,
activeLoadId: previousActiveLoadId ?? null,
activeNativePathToken: previousActiveNativePathToken ?? null,
@@ -2956,17 +2978,29 @@ export function useChatModelRuntime() {
rollbackState.loadedCtxCheckpoints ?? null,
cacheRam: previousServerTuning.cacheRam ?? null,
loadedCacheRam: rollbackState.loadedCacheRam ?? null,
- loadedSpeculativeType: rollbackSpeculativeType,
- loadedSpecDraftNMax:
- rollbackResponse.spec_draft_n_max ?? null,
- loadedKvCacheDtype: rollbackResponse.cache_type_kv ?? null,
+ // A custom echo carries the INI's tuning, not the managed baselines (as managedKvCacheFields).
+ ...(rollbackResponse.requested_llama_cpp_config?.mode ===
+ "custom"
+ ? {
+ loadedSpeculativeType:
+ rollbackState.loadedSpeculativeType,
+ loadedSpecDraftNMax: rollbackState.loadedSpecDraftNMax,
+ loadedKvCacheDtype: rollbackState.loadedKvCacheDtype,
+ }
+ : {
+ loadedSpeculativeType: rollbackSpeculativeType,
+ loadedSpecDraftNMax:
+ rollbackResponse.spec_draft_n_max ?? null,
+ loadedKvCacheDtype:
+ rollbackResponse.cache_type_kv ?? null,
+ }),
...mlxRuntimeStateFrom(rollbackResponse),
// After the spread, which seeds the control from the echo; the control keeps its intent, like
// nParallel above.
mlxKvQuant: previousMlxKvQuant,
loadedChatTemplateOverride:
rollbackState.loadedChatTemplateOverride,
- ...loadedGpuMemoryFields(rollbackResponse),
+ ...managedGpuMemoryFields(rollbackResponse),
tensorParallel: rollbackResponse.tensor_parallel ?? false,
loadedTensorParallel:
rollbackResponse.tensor_parallel ?? false,
diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
index e27d5e7d6d4..cc90062920e 100644
--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
@@ -469,7 +469,9 @@ export function applyActiveModelStatusToStore(
specDrafterKind: status.spec_drafter_kind ?? null,
// Controls follow the server while clean; loaded baselines always describe
// the settled resident, including same-model reloads from another client.
+ // Never from a custom load: its echo is the INI's tuning.
...(seedLoadParams &&
+ status.requested_llama_cpp_config?.mode !== "custom" &&
(status.speculative_type !== undefined ||
prevState.loadedSpeculativeType === null ||
hydratingExistingModel) && {
@@ -481,6 +483,7 @@ export function applyActiveModelStatusToStore(
}),
}),
...(seedLoadParams &&
+ status.requested_llama_cpp_config?.mode !== "custom" &&
status.spec_draft_n_max !== undefined && {
loadedSpecDraftNMax: status.spec_draft_n_max ?? null,
...((hydratingExistingModel ||
@@ -489,6 +492,7 @@ export function applyActiveModelStatusToStore(
}),
}),
...(seedLoadParams &&
+ status.requested_llama_cpp_config?.mode !== "custom" &&
status.cache_type_kv !== undefined && {
loadedKvCacheDtype: status.cache_type_kv,
...((prevState.loadedKvCacheDtype === null ||
@@ -622,6 +626,11 @@ export function applyActiveModelStatusToStore(
// status, not just the first: another client can reload the SAME model with different
// arguments, and a pinned baseline would resurrect arguments that are not running.
// seedLoadParams still guards it, so a mid-switch poll cannot overwrite performLoad.
+ ...(status.requested_llama_cpp_config !== undefined && seedLoadParams ? {
+ llamaCppConfig: status.requested_llama_cpp_config ?? undefined,
+ loadedLlamaCppConfig: status.requested_llama_cpp_config ?? null,
+ llamaCppConfigSummary: status.llama_cpp_config_summary ?? null,
+ } : {}),
...resolveLlamaExtraArgsSeed({
incoming: status.requested_llama_extra_args,
isGguf: status.is_gguf ?? true,
@@ -673,8 +682,9 @@ export function applyActiveModelStatusToStore(
}),
// Re-seed on first hydration, model/variant changes, or a same-model backend
// placement change. placementAndContextFields preserves dirty local edits in the last
- // case while advancing their loaded baselines.
+ // case while advancing their loaded baselines. Never from a custom load (see above).
...(seedLoadParams &&
+ status.requested_llama_cpp_config?.mode !== "custom" &&
(prevState.loadedGpuMemoryMode === null ||
hydratingExistingModel ||
placementOrContextChanged) &&
diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.ts
index b34e0b869db..2af39044f6f 100644
--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts
+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts
@@ -36,6 +36,7 @@ type ResidentRuntime = Pick<
| "disable_vision"
| "chat_template_override"
| "requested_llama_extra_args"
+ | "requested_llama_cpp_config"
| "gpu_memory_mode"
| "gpu_layers"
| "n_cpu_moe"
@@ -574,6 +575,10 @@ export function residentRuntimeMatchesConfig(
if (!config) {
return true;
}
+ if (config.llamaCppConfig?.mode === "custom" || status.requested_llama_cpp_config?.mode === "custom") {
+ // Only the server can tell whether a custom config is unchanged (binary, resources).
+ return false;
+ }
const placementPreserved =
// A virtualised Metal device pins every GGUF request to the CPU before either comparator runs, so
// placement cannot tell two requests apart there.
diff --git a/studio/frontend/src/features/chat/presets/preset-policy.ts b/studio/frontend/src/features/chat/presets/preset-policy.ts
index e9af04b1e19..b6537724e77 100644
--- a/studio/frontend/src/features/chat/presets/preset-policy.ts
+++ b/studio/frontend/src/features/chat/presets/preset-policy.ts
@@ -274,6 +274,7 @@ interface BackendInferenceDefaults {
top_k?: number;
min_p?: number;
presence_penalty?: number;
+ repetition_penalty?: number;
trust_remote_code?: boolean;
}
@@ -330,6 +331,9 @@ export function mergeBackendRecommendedInference({
presencePenalty:
toFiniteNumber(inference?.presence_penalty) ??
defaultInferenceParams.presencePenalty,
+ // Only a custom llama.cpp config reports one; otherwise the slider keeps its value.
+ repetitionPenalty:
+ toFiniteNumber(inference?.repetition_penalty) ?? next.repetitionPenalty,
};
}
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index b919c159185..b15ec7ffb43 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+// eslint-disable-next-line no-restricted-imports -- Keep the import-free payload helper independent of the picker UI.
+import { llamaCppConfigPayload } from "@/features/model-picker/model-config/llama-cpp-config";
import { useChatArtifactsStore } from "./artifacts/store";
import { mlxRuntimeStateFrom } from "./lib/mlx-runtime-state";
import { offloadCountsFrom, offloadWarning } from "./lib/partial-offload";
@@ -203,10 +205,12 @@ import {
shouldPinDiffusionPlacement,
} from "./lib/gpu-placement";
import {
- loadedGpuMemoryFields,
+ loadedLlamaCppConfigFields,
+ managedGpuMemoryFields,
type ReasoningEffort,
reconcilePersistedGpuIds,
- resolveLoadedSpeculativeSettings,
+ managedKvCacheFields,
+ managedSpeculativeSettings,
resolvePreserveThinkingOnLoad,
persistGpuMemoryModeOnLoad,
resolveSpeculativeSettingsForLoad,
@@ -1728,6 +1732,9 @@ export function SharedComposer({
: ownConfig.reasoningBudgetMessage,
// Only when this panel has read the stored value: omitted, the load inherits it, which is what
// keeps CLI-set flags working.
+ ...llamaCppConfigPayload(ownConfig.llamaCppConfig, {
+ isDiffusion: resolvedIsDiffusion === true,
+ }),
...(ownConfig.llamaExtraArgs !== undefined
? // biome-ignore lint/style/useNamingConvention: API schema
{ llama_extra_args: ownConfig.llamaExtraArgs ?? [] }
@@ -1827,6 +1834,9 @@ export function SharedComposer({
n_cpu_moe: effectiveNCpuMoe,
tensor_split: compareLoadKnobs.splitRatio ?? undefined,
gpu_ids: effectiveSelectedGpuIds ?? undefined,
+ ...llamaCppConfigPayload(ownConfig.llamaCppConfig, {
+ isDiffusion: resolvedIsDiffusion === true,
+ }),
...(ownConfig.llamaExtraArgs !== undefined
? // biome-ignore lint/style/useNamingConvention: API schema
{ llama_extra_args: ownConfig.llamaExtraArgs ?? [] }
@@ -1923,8 +1933,7 @@ export function SharedComposer({
supportsPreserveThinking: resp.supports_preserve_thinking ?? false,
preserveThinking: resolvePreserveThinkingOnLoad(resp),
supportsTools: resp.supports_tools ?? false,
- kvCacheDtype: resp.cache_type_kv ?? null,
- loadedKvCacheDtype: resp.cache_type_kv ?? null,
+ ...managedKvCacheFields(resp),
...mlxRuntimeStateFrom(resp),
// Click-time value, not the resolved echo (see the single-model load).
nParallel: committedSlots,
@@ -1962,6 +1971,7 @@ export function SharedComposer({
: clearedServerTuningState()),
// What this pane's launch is running, for a later rollback: the status applier is held off for
// the whole load, so a switch straight after would snapshot the other model's list.
+ ...loadedLlamaCppConfigFields(resp, ownConfig.llamaCppConfig),
loadedLlamaExtraArgs:
resp.requested_llama_extra_args !== undefined
? (resp.requested_llama_extra_args ?? [])
@@ -1980,7 +1990,7 @@ export function SharedComposer({
loadedCustomContextLength: keepCustomCtx,
// Adopt the load response's GPU-memory fields (mode/layers/MoE/split/pick plus loaded baselines)
// so the GPU controls round-trip. The context group and native-path token/expiry clear below.
- ...loadedGpuMemoryFields(resp),
+ ...managedGpuMemoryFields(resp),
// Drives the GPU Memory controls' diffusion gate; set alongside the GPU fields on every load path
// so the gate cannot read stale.
loadedIsDiffusion: resp.is_diffusion ?? false,
@@ -1997,7 +2007,7 @@ export function SharedComposer({
// lease. Clear any prior picked file's token/expiry so the reload path never sends a stale one.
activeNativePathToken: null,
activeNativePathExpiresAtMs: null,
- ...resolveLoadedSpeculativeSettings(resp),
+ ...managedSpeculativeSettings(resp),
});
if (!targetIsGguf) {
// Non-GGUF panes carry their context in params.maxSeqLength.
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index d1bb7849c2a..b4661f07cf8 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -1,6 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+// eslint-disable-next-line no-restricted-imports -- The picker barrel imports this store; this leaf is import-free.
+import type {
+ LlamaCppConfig,
+ LlamaCppConfigSummary,
+} from "@/features/model-picker/model-config/llama-cpp-config";
import { authFetch } from "@/features/auth";
import type { ImageDisclosure } from "../api/mcp-image";
import {
@@ -2099,6 +2104,55 @@ export function requestedGpuIdsFromResponse(resp: {
: (resp.gpu_ids ?? null);
}
+type LlamaCppConfigEcho = { requested_llama_cpp_config?: { mode: string } | null };
+
+function isCustomLlamaLoad(resp: LlamaCppConfigEcho): boolean {
+ return resp.requested_llama_cpp_config?.mode === "custom";
+}
+
+/** The config a load ran with: the server's echo, else what was sent. */
+export function loadedLlamaCppConfigFields(
+ resp: {
+ requested_llama_cpp_config?: LlamaCppConfig | null;
+ llama_cpp_config_summary?: LlamaCppConfigSummary | null;
+ },
+ sent: LlamaCppConfig | undefined,
+) {
+ const config = resp.requested_llama_cpp_config ?? sent;
+ return {
+ llamaCppConfig: config,
+ loadedLlamaCppConfig: config ?? null,
+ llamaCppConfigSummary: resp.llama_cpp_config_summary ?? null,
+ };
+}
+
+// A custom load echoes its INI's tuning; adopting it would carry that into the next managed load.
+export function managedKvCacheFields(
+ resp: { cache_type_kv?: string | null } & LlamaCppConfigEcho,
+) {
+ if (isCustomLlamaLoad(resp)) return {};
+ const kv = resp.cache_type_kv ?? null;
+ return { kvCacheDtype: kv, loadedKvCacheDtype: kv };
+}
+
+export function managedSpeculativeSettings(
+ resp: Parameters<typeof resolveLoadedSpeculativeSettings>[0] & LlamaCppConfigEcho,
+) {
+ return isCustomLlamaLoad(resp) ? {} : resolveLoadedSpeculativeSettings(resp);
+}
+
+export function managedGpuMemoryFields(
+ resp: Parameters<typeof loadedGpuMemoryFields>[0] & LlamaCppConfigEcho,
+) {
+ if (isCustomLlamaLoad(resp)) {
+ return {
+ ggufLayerCount: resp.n_layers ?? null,
+ moeLayerCount: resp.n_moe_layers ?? null,
+ };
+ }
+ return loadedGpuMemoryFields(resp);
+}
+
// Store fields derived from a load/status response's GPU-memory settings, shared by every
// load path so the manual-knob round-trip cannot drift.
export function loadedGpuMemoryFields(resp: {
@@ -2442,6 +2496,9 @@ type ChatRuntimeStore = {
/** Pass-through args the resident model is running, as far as this client knows. A rollback
* resends them: by then the target load has replaced the backend's inheritance source. */
loadedLlamaExtraArgs: string[] | null;
+ llamaCppConfig?: LlamaCppConfig;
+ loadedLlamaCppConfig: LlamaCppConfig | null;
+ llamaCppConfigSummary: LlamaCppConfigSummary | null;
/** user --ubatch-size override for gguf loads (null = llama.cpp default 512) */
nUbatch: number | null;
loadedNUbatch: number | null;
@@ -4211,6 +4268,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
nBatch: null,
loadedNBatch: null,
loadedLlamaExtraArgs: null,
+ loadedLlamaCppConfig: null,
+ llamaCppConfigSummary: null,
nUbatch: null,
loadedNUbatch: null,
specDraftCacheDtype: null,
@@ -5159,6 +5218,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
nBatch: null,
loadedNBatch: null,
loadedLlamaExtraArgs: null,
+ loadedLlamaCppConfig: null,
+ llamaCppConfigSummary: null,
nUbatch: null,
loadedNUbatch: null,
specDraftCacheDtype: null,
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 119bd98a5aa..8c8344e04f4 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -1,6 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+// eslint-disable-next-line no-restricted-imports -- Share the config wire types without expanding the picker UI barrel.
+import type {
+ LlamaCppConfig,
+ LlamaCppConfigSummary,
+} from "@/features/model-picker/model-config/llama-cpp-config";
import type { TransformersUpgradeInfo } from "@/features/transformers-upgrade";
export type CpuFallbackReason = "vulkan_startup_crash";
@@ -101,6 +106,7 @@ export interface LoadModelRequest {
* flag. Omit/null inherits the stored per-model value; [] launches with none. GGUF only. */
// biome-ignore lint/style/useNamingConvention: API schema
llama_extra_args?: string[] | null;
+ llama_cpp_config?: LlamaCppConfig;
/** Split the model across GPUs by tensor (--split-mode tensor) instead of by layer for GGUF models.
* Multi-GPU only. */
tensor_parallel?: boolean | null;
@@ -127,6 +133,8 @@ export interface LoadModelRequest {
}
export interface ValidateModelResponse {
+ requested_llama_cpp_config?: LlamaCppConfig | null;
+ llama_cpp_config_summary?: LlamaCppConfigSummary | null;
valid: boolean;
message: string;
identifier?: string | null;
@@ -325,6 +333,8 @@ export interface LoadModelResponse {
requested_cache_ram?: number | null;
/** Pass-through llama-server arguments the running load was invoked with. */
requested_llama_extra_args?: string[] | null;
+ requested_llama_cpp_config?: LlamaCppConfig | null;
+ llama_cpp_config_summary?: LlamaCppConfigSummary | null;
}
export interface UnloadModelRequest {
@@ -441,6 +451,8 @@ export interface InferenceStatusResponse {
requested_cache_ram?: number | null;
/** Pass-through llama-server arguments the running load was invoked with. */
requested_llama_extra_args?: string[] | null;
+ requested_llama_cpp_config?: LlamaCppConfig | null;
+ llama_cpp_config_summary?: LlamaCppConfigSummary | null;
n_layers?: number | null;
/** Model's MoE expert-layer count (the n_cpu_moe ceiling); 0 if not MoE. */
n_moe_layers?: number;
diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.ts
index 91b6273c01f..41c3d46bdcd 100644
--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts
+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import type { LlamaCppConfig } from "../model-config/llama-cpp-config";
// Server-side mirror of the per-model config. per-model-config.ts lives in browser
// localStorage, so an API auto-switch load came up with none of the user's settings.
// routes/inference.py reads this map and rebuilds the picker's LoadRequest.
@@ -27,6 +28,7 @@ const OVERRIDES_URL = "/api/settings/openai-auto-switch/overrides";
export interface ApiModelOverride {
// biome-ignore lint/style/useNamingConvention: API schema
llama_extra_args?: string[];
+ llama_cpp_config?: LlamaCppConfig;
// biome-ignore lint/style/useNamingConvention: API schema
max_seq_length?: number;
// biome-ignore lint/style/useNamingConvention: API schema
@@ -344,6 +346,7 @@ export function fromApiOverride(
chatTemplateOverride:
override.chat_template_override ?? local.chatTemplateOverride,
llamaExtraArgs: extraArgs,
+ llamaCppConfig: override.llama_cpp_config ?? local.llamaCppConfig,
gpuMemoryMode: override.gpu_memory_mode ?? local.gpuMemoryMode,
gpuLayers: override.gpu_layers ?? local.gpuLayers,
nCpuMoe: override.n_cpu_moe ?? local.nCpuMoe,
@@ -431,6 +434,9 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {
// The one field where absent does NOT mean "app default": the route preserves
// llama_extra_args it is not sent, which kept CLI-set flags alive while this panel had no
// control. So `undefined` stays omitted and a cleared box sends an explicit empty list.
+ if (config.llamaCppConfig !== undefined) {
+ payload.llama_cpp_config = config.llamaCppConfig;
+ }
if (config.llamaExtraArgs !== undefined) {
payload.llama_extra_args = config.llamaExtraArgs ?? [];
}
diff --git a/studio/frontend/src/features/model-picker/components/custom-llama-config-editor.tsx b/studio/frontend/src/features/model-picker/components/custom-llama-config-editor.tsx
new file mode 100644
index 00000000000..21e9769a163
--- /dev/null
+++ b/studio/frontend/src/features/model-picker/components/custom-llama-config-editor.tsx
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { useEffect, useId } from "react";
+import { Button } from "@/components/ui/button";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+import {
+ customConfigSections,
+ MAX_LLAMA_CPP_CONFIG_BYTES,
+ toggledLlamaCppConfig,
+ type LlamaCppConfig,
+} from "../model-config/llama-cpp-config";
+
+// Module scope: switching back to custom restores the last INI even after a remount.
+const lastCustomSource = new Map<
+ string,
+ { ini: string; section: string | null }
+>();
+
+export function CustomLlamaConfigEditor({
+ value,
+ onChange,
+ sourceKey,
+ onLoadableChange,
+}: {
+ value: LlamaCppConfig | undefined;
+ onChange: (value: LlamaCppConfig) => void;
+ sourceKey: string;
+ onLoadableChange: (value: boolean) => void;
+}) {
+ const id = useId();
+ const active = value?.mode === "custom";
+ const ini = active ? value.ini : "";
+ const section = active ? value.section : null;
+ const sections = customConfigSections(ini);
+ const oversized =
+ new TextEncoder().encode(ini).length > MAX_LLAMA_CPP_CONFIG_BYTES;
+ const blank = active && ini.trim().length === 0;
+ const needsSection =
+ active && section === null && sections.some((name) => name !== "default");
+ useEffect(() => {
+ if (active) lastCustomSource.set(sourceKey, { ini, section });
+ }, [active, ini, section, sourceKey]);
+ useEffect(() => {
+ onLoadableChange(!active || !(oversized || blank || needsSection));
+ }, [active, oversized, blank, needsSection, onLoadableChange]);
+ return (
+ <div className="space-y-3">
+ <div className="flex flex-wrap items-center justify-between gap-2">
+ <span className="text-ui-12 font-medium">
+ Custom llama.cpp configuration
+ </span>
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ onClick={() =>
+ onChange(
+ toggledLlamaCppConfig(
+ value,
+ lastCustomSource.get(sourceKey) ?? null,
+ ),
+ )
+ }
+ >
+ {active ? "Use Studio settings" : "Use custom configuration"}
+ </Button>
+ </div>
+ {active && (
+ <>
+ <p className="text-ui-11 text-muted-foreground">
+ llama-server runs with exactly this INI (llama.cpp preset format);
+ Studio adds only the model, host and port. Its sampling values
+ become this model&apos;s chat defaults.
+ </p>
+ <textarea
+ id={id}
+ aria-label="llama.cpp INI configuration"
+ spellCheck={false}
+ className="min-h-40 w-full resize-y rounded-md border border-input bg-transparent px-3 py-2 font-mono text-ui-11"
+ value={ini}
+ onChange={(event) => {
+ const nextIni = event.target.value;
+ onChange({
+ ...value,
+ ini: nextIni,
+ section: customConfigSections(nextIni).includes(
+ value.section ?? "",
+ )
+ ? value.section
+ : null,
+ });
+ }}
+ />
+ {sections.some((name) => name !== "default") && (
+ <div className="space-y-1.5">
+ <label className="text-ui-11" htmlFor={`${id}-section`}>
+ Section
+ </label>
+ <Select
+ value={section ?? ""}
+ onValueChange={(next) => onChange({ ...value, section: next })}
+ >
+ <SelectTrigger id={`${id}-section`}>
+ <SelectValue placeholder="Choose a section" />
+ </SelectTrigger>
+ <SelectContent>
+ {sections.map((name) => (
+ <SelectItem key={name} value={name}>
+ {name}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+ )}
+ {(oversized || blank || needsSection) && (
+ <p role="alert" className="text-ui-11 text-destructive">
+ {oversized
+ ? "Configuration must fit within 64 KiB."
+ : blank
+ ? "Configuration is empty."
+ : "Choose which section to load."}
+ </p>
+ )}
+ </>
+ )}
+ </div>
+ );
+}
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index fb6942eaf75..dc8eba47234 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -1,6 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import { CustomLlamaConfigEditor } from "./custom-llama-config-editor";
+import {
+ MANAGED_LLAMA_CPP_CONFIG,
+ normalizeLlamaCppConfig,
+} from "../model-config/llama-cpp-config";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { InfoHint } from "@/components/ui/info-hint";
@@ -338,6 +343,7 @@ function withoutUnsupportedDiffusionSettings(
config.nBatch == null &&
config.nUbatch == null &&
(config.llamaExtraArgs == null || config.llamaExtraArgs.length === 0) &&
+ config.llamaCppConfig?.mode !== "custom" &&
!hasUnsupportedGpuPick
) {
return config;
@@ -358,6 +364,10 @@ function withoutUnsupportedDiffusionSettings(
// them as though it had, so a box filled before classification flipped would leave the
// model running without what it says.
llamaExtraArgs: null,
+ // The diffusion runner has no llama-server; explicit managed, since omitted inherits the stored custom.
+ ...(config.llamaCppConfig?.mode === "custom"
+ ? { llamaCppConfig: MANAGED_LLAMA_CPP_CONFIG }
+ : {}),
...(hasUnsupportedGpuPick
? {
selectedGpuIds: undefined,
@@ -1310,6 +1320,33 @@ function LoadModeRow({
);
}
+// Outside the managed lock: custom launches still honour disable_vision.
+function VisionRow({
+ config,
+ update,
+}: {
+ config: PerModelConfig;
+ update: (patch: Partial<PerModelConfig>) => void;
+}) {
+ return (
+ <div className={ROW_CLASS}>
+ <div className="flex min-w-0 items-center gap-1.5">
+ <span className={LABEL_CLASS}>Vision</span>
+ <InfoHint>
+ Loads the vision projector so the model can read images. Turning
+ it off frees that VRAM for more layers on the GPU. Text generation
+ is unaffected either way.
+ </InfoHint>
+ </div>
+ <Switch
+ className="panel-switch shrink-0"
+ checked={!config.disableVision}
+ onCheckedChange={(checked) => update({ disableVision: !checked })}
+ />
+ </div>
+ );
+}
+
function GgufAdvancedSettings({
config,
update,
@@ -1325,6 +1362,7 @@ function GgufAdvancedSettings({
moeLayersInputRef,
onExtraArgsLoadableChange,
draftKey,
+ hideVision = false,
}: {
config: PerModelConfig;
update: (patch: Partial<PerModelConfig>) => void;
@@ -1341,6 +1379,7 @@ function GgufAdvancedSettings({
/** Which stored entries the extra-arguments row reads, most specific first. */
onExtraArgsLoadableChange: (loadable: boolean) => void;
draftKey: string;
+ hideVision?: boolean;
}) {
const batchAdviceId = useId();
const ubatchAdviceId = useId();
@@ -1632,21 +1671,7 @@ function GgufAdvancedSettings({
{/* withoutUnsupportedDiffusionSettings forces disableVision back to false on a diffusion model
and the runner never reads it, so the switch would flip back under the pointer. */}
{!isDiffusion && (
- <div className={ROW_CLASS}>
- <div className="flex min-w-0 items-center gap-1.5">
- <span className={LABEL_CLASS}>Vision</span>
- <InfoHint>
- Loads the vision projector so the model can read images. Turning
- it off frees that VRAM for more layers on the GPU. Text generation
- is unaffected either way.
- </InfoHint>
- </div>
- <Switch
- className="panel-switch shrink-0"
- checked={!config.disableVision}
- onCheckedChange={(checked) => update({ disableVision: !checked })}
- />
- </div>
+ hideVision ? null : <VisionRow config={config} update={update} />
)}
{!isDiffusion && (
@@ -2160,6 +2185,7 @@ export function ModelConfigPage({
// refuse. Held by the panel rather than the row, since the row unmounts whenever Advanced
// settings collapse while its tokens stay in the config.
const [extraArgsLoadable, setExtraArgsLoadable] = useState(true);
+ const [customConfigLoadable, setCustomConfigLoadable] = useState(true);
// True until the server-row read below settles: a load started before it lands sends none of the
// row's settings, and with Remember unchecked it forgets the row it never read.
const [extraArgsHydrating, setExtraArgsHydrating] = useState(
@@ -2815,7 +2841,11 @@ export function ModelConfigPage({
// stands down, since its runner allocates on a different plan. The tri-state is read as a
// tri-state, not through resolvedIsDiffusion: a GGUF still being classified may be
// DiffusionGemma, and guessing paints a footprint from the wrong plan that never clears.
+ // A custom INI owns llama.cpp tuning: the managed rows lock and Studio cannot price it.
+ const customActive =
+ config.llamaCppConfig?.mode === "custom" && !resolvedIsDiffusion;
const memoryEstimateRequest =
+ !customActive &&
shouldRequestMemoryEstimate({
isGguf: Boolean(target.isGguf),
isAppleUnifiedMemory,
@@ -3146,6 +3176,14 @@ export function ModelConfigPage({
};
const persistConfig = (next: PerModelConfig) => {
+ // Normalizing drops a blank or oversized custom config, saving managed over a good one.
+ if (
+ remember &&
+ next.llamaCppConfig !== undefined &&
+ normalizeLlamaCppConfig(next.llamaCppConfig) === undefined
+ ) {
+ return { saved: false, defaultConfig: false };
+ }
// Judge what storage keeps: savePerModelConfig normalizes first, so the raw object over-reports.
const normalized = normalizePerModelConfig(next);
const evicted: { modelId: string; ggufVariant: string | null }[] = [];
@@ -3343,7 +3381,11 @@ export function ModelConfigPage({
remember={remember}
hasSavedSettings={savedRemember}
/>
- <div className="space-y-5">
+ <fieldset
+ disabled={customActive}
+ inert={customActive ? true : undefined}
+ className={`min-w-0 space-y-5 ${customActive ? "opacity-50" : ""}`}
+ >
{memoryEstimateRequest != null && (
<MemoryEstimateRow
estimate={memoryEstimate.estimate}
@@ -3467,6 +3509,7 @@ export function ModelConfigPage({
layerCount={stagedDims?.layerCount ?? null}
moeLayerCount={stagedDims?.moeLayerCount ?? null}
isDiffusion={resolvedIsDiffusion}
+ hideVision={customActive}
gpuDevices={gpuDevices}
gpuLayersInputRef={gpuLayersInputRef}
moeLayersInputRef={moeLayersInputRef}
@@ -3509,7 +3552,21 @@ export function ModelConfigPage({
)}
</>
)}
- </div>
+ </fieldset>
+
+ {target.isGguf &&
+ !resolvedIsDiffusion &&
+ (showAdvanced || customActive) && (
+ <div className="mt-5 space-y-5">
+ {customActive && <VisionRow config={config} update={update} />}
+ <CustomLlamaConfigEditor
+ value={config.llamaCppConfig}
+ onChange={(llamaCppConfig) => update({ llamaCppConfig })}
+ sourceKey={draftKey}
+ onLoadableChange={setCustomConfigLoadable}
+ />
+ </div>
+ )}
{/* Stacked in both variants: a row that wraps on demand reflows when the same click that
commits a draft mounts Save settings, moving Load out from under the cursor. */}
@@ -3542,8 +3599,10 @@ export function ModelConfigPage({
sharedVariantUnresolved ||
stagedMetadataPending ||
budgetSettling ||
- (!extraArgsLoadable && !sharedExtraArgsCleared) ||
- sharedExtraArgsRefused ||
+ (customActive && !customConfigLoadable) ||
+ (!customActive &&
+ ((!extraArgsLoadable && !sharedExtraArgsCleared) ||
+ sharedExtraArgsRefused)) ||
extraArgsHydrating ||
(isActiveModel &&
atBaseline &&
@@ -3569,7 +3628,8 @@ export function ModelConfigPage({
stagedMetadataPending ||
budgetSettling ||
(remember &&
- ((!extraArgsLoadable && !sharedExtraArgsCleared) ||
+ ((customActive && !customConfigLoadable) ||
+ (!extraArgsLoadable && !sharedExtraArgsCleared) ||
sharedExtraArgsRefused ||
extraArgsHydrating))
}
@@ -3596,6 +3656,7 @@ export function ModelConfigPage({
// running process's arguments, so a reload after Reset kept the flags the box says are gone.
...DEFAULT_PER_MODEL_CONFIG,
llamaExtraArgs: null,
+ llamaCppConfig: MANAGED_LLAMA_CPP_CONFIG,
});
}}
>
diff --git a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
index 5206a36ac1b..583dff7cfe7 100644
--- a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
+++ b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
@@ -51,6 +51,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
const loadMode = useChatRuntimeStore((s) => s.loadMode);
const ctxCheckpoints = useChatRuntimeStore((s) => s.ctxCheckpoints);
const cacheRam = useChatRuntimeStore((s) => s.cacheRam);
+ const llamaCppConfig = useChatRuntimeStore((s) => s.loadedLlamaCppConfig);
const tensorParallel = useChatRuntimeStore((s) => s.tensorParallel);
const disableVision = useChatRuntimeStore((s) => s.disableVision);
const chatTemplateOverride = useChatRuntimeStore(
@@ -91,6 +92,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
return null;
}
const base: PerModelConfig = {
+ llamaCppConfig: isGguf ? llamaCppConfig ?? undefined : undefined,
customContextLength: customContextLength ?? null,
// A self-sizing backend carries no pin here, exactly as the GGUF path does: this
// is the runtime's resolved length, and reading it back as the user's choice would
@@ -148,6 +150,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {
loadMode,
ctxCheckpoints,
cacheRam,
+ llamaCppConfig,
tensorParallel,
disableVision,
chatTemplateOverride,
diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
index c9c88ad6839..21c0c0b2581 100644
--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
@@ -57,6 +57,11 @@ export function applyPerModelConfigToRuntime(
)
: { ids: null, indexKind: null };
useChatRuntimeStore.setState({
+ // Explicit managed on diffusion: an omitted field inherits the stored custom config.
+ llamaCppConfig:
+ options.isDiffusion && config.llamaCppConfig !== undefined
+ ? { version: 1, mode: "managed" }
+ : config.llamaCppConfig,
customContextLength: config.customContextLength ?? null,
mlxKvQuant: config.mlxKvQuant ?? null,
kvCacheDtype: config.kvCacheDtype ?? null,
@@ -118,6 +123,7 @@ export function currentRuntimePerModelConfig(
): PerModelConfig {
const s = useChatRuntimeStore.getState();
return {
+ llamaCppConfig: s.llamaCppConfig,
customContextLength: s.customContextLength ?? null,
maxSeqLength: options.includeMaxSeqLength
? normalizeMaxSeqLength(s.params.maxSeqLength)
@@ -155,6 +161,10 @@ export function currentRuntimePerModelConfig(
};
}
+function customOnly(config: PerModelConfig["llamaCppConfig"]) {
+ return config?.mode === "custom" ? config : undefined;
+}
+
/** `followGlobal`: only against the running config, which holds the mode a null one resolved to.
* Stored configs and presets keep null distinct from an explicit mode equal to today's global. */
export function perModelConfigsEqual(
@@ -166,6 +176,9 @@ export function perModelConfigsEqual(
? resolvedSpeculativeType
: normalizeSpeculativeType;
return (
+ // Unset and managed load the same engine.
+ JSON.stringify(customOnly(a.llamaCppConfig)) ===
+ JSON.stringify(customOnly(b.llamaCppConfig)) &&
(a.customContextLength ?? null) === (b.customContextLength ?? null) &&
normalizeMaxSeqLength(a.maxSeqLength) ===
normalizeMaxSeqLength(b.maxSeqLength) &&
diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
index c30ef802e63..b48f6718c4a 100644
--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts
+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts
@@ -48,6 +48,7 @@ export function loadedConfigSignature(
return "none";
}
return [
+ JSON.stringify(config.llamaCppConfig) ?? "",
config.customContextLength ?? "",
config.maxSeqLength ?? "",
config.kvCacheDtype ?? "",
diff --git a/studio/frontend/src/features/model-picker/model-config/llama-cpp-config.ts b/studio/frontend/src/features/model-picker/model-config/llama-cpp-config.ts
new file mode 100644
index 00000000000..24cc8805330
--- /dev/null
+++ b/studio/frontend/src/features/model-picker/model-config/llama-cpp-config.ts
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+export type LlamaCppConfig =
+ | { version: 1; mode: "managed" }
+ | { version: 1; mode: "custom"; ini: string; section: string | null };
+
+export interface LlamaCppConfigSummary {
+ mode: "custom";
+ section: string | null;
+ options: Record<string, string | boolean>;
+ request_defaults: Record<string, number>;
+ diagnostics: string[];
+}
+
+export const MANAGED_LLAMA_CPP_CONFIG: LlamaCppConfig = {
+ version: 1,
+ mode: "managed",
+};
+
+export const MAX_LLAMA_CPP_CONFIG_BYTES = 65_536;
+
+/** Shape validation only. The selected server is authoritative for INI semantics. */
+export function normalizeLlamaCppConfig(
+ value: unknown,
+): LlamaCppConfig | undefined {
+ if (!value || typeof value !== "object") return undefined;
+ const source = value as Record<string, unknown>;
+ if (source.version !== 1) return undefined;
+ if (source.mode === "managed") return { version: 1, mode: "managed" };
+ if (
+ source.mode !== "custom" ||
+ typeof source.ini !== "string" ||
+ source.ini.trim().length === 0 ||
+ (source.section !== null && typeof source.section !== "string") ||
+ new TextEncoder().encode(source.ini).length > MAX_LLAMA_CPP_CONFIG_BYTES
+ )
+ return undefined;
+ return {
+ version: 1,
+ mode: "custom",
+ ini: source.ini,
+ section: source.section,
+ };
+}
+
+/** The mode switch: coming back to custom restores the last source instead of a blank one. */
+export function toggledLlamaCppConfig(
+ value: LlamaCppConfig | undefined,
+ lastCustom: { ini: string; section: string | null } | null,
+): LlamaCppConfig {
+ if (value?.mode === "custom") return { version: 1, mode: "managed" };
+ return {
+ version: 1,
+ mode: "custom",
+ ini: lastCustom?.ini ?? "[*]\n",
+ section: lastCustom?.section ?? null,
+ };
+}
+
+/** Suggestions for the selector, never a parser or a validation verdict. */
+// llama.cpp preset grammar: keys above the first header form the "default" section.
+export function customConfigSections(ini: string): string[] {
+ const header = /^\[[ \t]*([^\]\r\n]+)\][ \t]*(?:[#;][^\r\n]*)?\r?$/gm;
+ const firstHeader = ini.search(header);
+ const preamble = firstHeader < 0 ? ini : ini.slice(0, firstHeader);
+ const hasDefault = /^[ \t]*[A-Za-z_]/m.test(preamble);
+ return [
+ ...new Set([
+ ...(hasDefault ? ["default"] : []),
+ ...[...ini.matchAll(header)]
+ .map((match) => match[1].trim())
+ .filter((name) => name !== "*"),
+ ]),
+ ];
+}
+
+export function llamaCppConfigPayload(
+ config: LlamaCppConfig | undefined,
+ options: { isDiffusion?: boolean } = {},
+) {
+ // The diffusion runner has no llama-server; explicit managed keeps the backend from inheriting custom.
+ if (options.isDiffusion)
+ return { llama_cpp_config: MANAGED_LLAMA_CPP_CONFIG };
+ if (config === undefined) return {};
+ return { llama_cpp_config: config };
+}
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index 993249b7cf0..f2c0bd8fd68 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import { normalizeLlamaCppConfig, type LlamaCppConfig } from "./llama-cpp-config";
import type { GpuIndexKind } from "@/hooks/use-gpu-info";
import {
cachedRepoConfigId,
@@ -49,6 +50,7 @@ export interface PerModelConfig {
* alive); `null` means the user cleared the box and must be sent as an explicit `[]`; a non-empty list is what
* to launch with. */
llamaExtraArgs?: string[] | null;
+ llamaCppConfig?: LlamaCppConfig;
// GPU Memory controls (per-model, GGUF-only), optional so older blobs parse. Absent or null
// selectedGpuIds means automatic.
gpuMemoryMode?: "auto" | "manual";
@@ -368,8 +370,9 @@ const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";
// would normalize the unknown field straight back out of the record.
// v2 added nBatch/nUbatch, v3 llamaExtraArgs, v4 disableVision, v5 the llama-server tuning group
// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair,
-// v7 mlxKvQuant
-const STORAGE_SCHEMA_VERSION = 7;
+// v7 mlxKvQuant, and v8 custom llama.cpp configuration.
+const STORAGE_SCHEMA_VERSION = 8;
+const PRE_LLAMA_CPP_CONFIG_SCHEMA_VERSION = 7;
const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;
const PRE_REASONING_BUDGET_SCHEMA_VERSION = 5;
const PRE_SERVER_TUNING_SCHEMA_VERSION = 4;
@@ -426,6 +429,7 @@ const STORED_CONFIG_FIELDS = new Set([
"disableVision",
"chatTemplateOverride",
"llamaExtraArgs",
+ "llamaCppConfig",
"gpuMemoryMode",
"gpuLayers",
"nCpuMoe",
@@ -1061,6 +1065,7 @@ function normalizeV1(partial: RawConfig): PerModelConfig {
? partial.chatTemplateOverride
: null,
llamaExtraArgs: normalizeLlamaExtraArgs(partial.llamaExtraArgs),
+ llamaCppConfig: normalizeLlamaCppConfig(partial.llamaCppConfig),
...normalizeGpuFields(partial),
};
}
@@ -1089,9 +1094,12 @@ function normalize(raw: unknown): PerModelConfig {
* client reconstructs anyway, and stamping every record v4 would put the whole store out of reach.
* The tuning group and the reasoning pair follow the same rule. */
function storedSchemaVersion(normalized: PerModelConfig): number {
- if (normalized.mlxKvQuant != null) {
+ if (normalized.llamaCppConfig !== undefined) {
return STORAGE_SCHEMA_VERSION;
}
+ if (normalized.mlxKvQuant != null) {
+ return PRE_LLAMA_CPP_CONFIG_SCHEMA_VERSION;
+ }
const hasReasoningBudget =
normalized.reasoningBudget !== -1 || normalized.reasoningBudgetMessage !== "";
if (hasReasoningBudget) {
@@ -1287,6 +1295,7 @@ export function isDefaultConfig(config: PerModelConfig): boolean {
// Or a config whose only change is Extra Arguments reads as default, and savePerModelConfig
// deletes the entry it was asked to remember.
(config.llamaExtraArgs == null || config.llamaExtraArgs.length === 0) &&
+ config.llamaCppConfig === undefined &&
gpuFieldsAtDefault(config)
);
}
@@ -1310,6 +1319,9 @@ export function savePerModelConfig(
* without this their server overrides would keep applying with nothing in the UI able to forget them. */
evicted?: { modelId: string; ggufVariant: string | null }[],
): boolean {
+ if (config.llamaCppConfig !== undefined && normalizeLlamaCppConfig(config.llamaCppConfig) === undefined) {
+ return false;
+ }
if (
typeof config.chatTemplateOverride === "string" &&
!isChatTemplateWithinLimit(config.chatTemplateOverride)
diff --git a/studio/frontend/src/features/model-picker/sharing/fields.ts b/studio/frontend/src/features/model-picker/sharing/fields.ts
index d6c3f3b308a..bd38c613bcb 100644
--- a/studio/frontend/src/features/model-picker/sharing/fields.ts
+++ b/studio/frontend/src/features/model-picker/sharing/fields.ts
@@ -41,7 +41,11 @@ const cacheType = nullable(
export type SharedConfigKey = Exclude<
keyof PerModelConfig,
- "chatTemplateOverride" | "tensorSplit" | "maxSeqLength" | "mlxKvQuant"
+ | "chatTemplateOverride"
+ | "tensorSplit"
+ | "maxSeqLength"
+ | "mlxKvQuant"
+ | "llamaCppConfig"
>;
type Field = { label: string; valid: Validator; error?: string };
diff --git a/studio/frontend/tests/llama-cpp-custom-config.test.ts b/studio/frontend/tests/llama-cpp-custom-config.test.ts
new file mode 100644
index 00000000000..b936481b7b7
--- /dev/null
+++ b/studio/frontend/tests/llama-cpp-custom-config.test.ts
@@ -0,0 +1,362 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+ installLocalStorageFake,
+ registerStoreStubResolver,
+ readSrc,
+} from "./helpers/kit.ts";
+
+registerStoreStubResolver();
+const { store } = installLocalStorageFake();
+const {
+ normalizeLlamaCppConfig,
+ customConfigSections,
+ toggledLlamaCppConfig,
+ llamaCppConfigPayload,
+} =
+ await import("../src/features/model-picker/model-config/llama-cpp-config.ts");
+const {
+ DEFAULT_PER_MODEL_CONFIG,
+ PER_MODEL_CONFIG_STORAGE_KEY,
+ savePerModelConfig,
+ resolveInitialConfig,
+} =
+ await import("../src/features/model-picker/model-config/per-model-config.ts");
+const { fromApiOverride, toApiOverride, resolveStoredOverride } =
+ await import("../src/features/model-picker/api/model-overrides.ts");
+const custom = {
+ version: 1,
+ mode: "custom",
+ ini: "[*]\nfit=off\n[my-model]\nctx-size=56000\ntemp=0\n",
+ section: "my-model",
+} as const;
+const managed = { version: 1, mode: "managed" } as const;
+
+test("switching back to custom restores the last source instead of a blank one", () => {
+ assert.deepEqual(toggledLlamaCppConfig(custom, null), managed);
+ assert.deepEqual(
+ toggledLlamaCppConfig(managed, {
+ ini: custom.ini,
+ section: custom.section,
+ }),
+ custom,
+ );
+ assert.deepEqual(toggledLlamaCppConfig(undefined, null), {
+ version: 1,
+ mode: "custom",
+ ini: "[*]\n",
+ section: null,
+ });
+ const editor = readSrc(
+ "features/model-picker/components/custom-llama-config-editor.tsx",
+ );
+ // A refused load remounts the editor, so the remembered source has to live outside it.
+ assert.match(editor, /^const lastCustomSource = new Map</m);
+ assert.match(
+ editor,
+ /toggledLlamaCppConfig\(\s*value,\s*lastCustomSource\.get\(sourceKey\) \?\? null,?\s*\)/,
+ );
+ assert.match(
+ editor,
+ /if \(active\) lastCustomSource\.set\(sourceKey, \{ ini, section \}\)/,
+ );
+});
+
+test("custom source and selection round-trip through storage and API without consuming legacy extras", () => {
+ store.clear();
+ const config = {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ llamaExtraArgs: ["--threads", "3"],
+ llamaCppConfig: custom,
+ };
+ assert.equal(savePerModelConfig("org/model", "Q4", config), true);
+ const read = resolveInitialConfig("org/model", "Q4").config;
+ assert.deepEqual(read.llamaCppConfig, custom);
+ assert.deepEqual(
+ fromApiOverride(toApiOverride(read), DEFAULT_PER_MODEL_CONFIG)
+ .llamaCppConfig,
+ custom,
+ );
+ assert.deepEqual(read.llamaExtraArgs, ["--threads", "3"]);
+ const records = JSON.parse(store.get(PER_MODEL_CONFIG_STORAGE_KEY)!);
+ assert.equal(
+ Object.values(records).some(
+ (row: unknown) => (row as { version: number }).version === 8,
+ ),
+ true,
+ );
+ assert.equal(
+ savePerModelConfig("org/model", "Q4", { ...read, llamaCppConfig: managed }),
+ true,
+ );
+ const reset = resolveInitialConfig("org/model", "Q4").config;
+ assert.deepEqual(reset.llamaCppConfig, managed);
+ assert.deepEqual(reset.llamaExtraArgs, ["--threads", "3"]);
+});
+
+test("a per-quant managed tombstone blocks bare-model custom fallback locally and on the mirror", () => {
+ store.clear();
+ savePerModelConfig("org/model", null, {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ llamaCppConfig: custom,
+ });
+ savePerModelConfig("org/model", "Q4", {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ llamaCppConfig: managed,
+ });
+ assert.deepEqual(
+ resolveInitialConfig("org/model", "Q4").config.llamaCppConfig,
+ managed,
+ );
+ assert.deepEqual(
+ resolveStoredOverride(
+ {
+ "org/model": { llama_cpp_config: custom },
+ "org/model:Q4": { llama_cpp_config: managed },
+ },
+ ["org/model:Q4", "org/model"],
+ )?.llama_cpp_config,
+ managed,
+ );
+});
+
+test("custom configuration advances only its own storage tier", () => {
+ const storedVersion = () => {
+ const records = JSON.parse(store.get(PER_MODEL_CONFIG_STORAGE_KEY) ?? "{}");
+ return (Object.values(records)[0] as { version?: number } | undefined)
+ ?.version;
+ };
+
+ store.clear();
+ assert.equal(
+ savePerModelConfig("org/reasoning", "Q4", {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ reasoningBudget: 256,
+ reasoningBudgetMessage: "Keep the proof short",
+ }),
+ true,
+ );
+ assert.equal(storedVersion(), 6);
+ const reasoning = resolveInitialConfig("org/reasoning", "Q4").config;
+ assert.equal(reasoning.reasoningBudget, 256);
+ assert.equal(reasoning.reasoningBudgetMessage, "Keep the proof short");
+ assert.equal(reasoning.llamaCppConfig, undefined);
+
+ store.clear();
+ assert.equal(
+ savePerModelConfig("org/custom", "Q4", {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ reasoningBudget: 256,
+ reasoningBudgetMessage: "Keep the proof short",
+ llamaCppConfig: custom,
+ }),
+ true,
+ );
+ assert.equal(storedVersion(), 8);
+ const customAndReasoning = resolveInitialConfig("org/custom", "Q4").config;
+ assert.equal(customAndReasoning.reasoningBudget, 256);
+ assert.equal(
+ customAndReasoning.reasoningBudgetMessage,
+ "Keep the proof short",
+ );
+ assert.deepEqual(customAndReasoning.llamaCppConfig, custom);
+
+ store.clear();
+ assert.equal(
+ savePerModelConfig("org/tuning", "Q4", {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ loadMode: "mmap",
+ }),
+ true,
+ );
+ assert.equal(storedVersion(), 5);
+ assert.equal(
+ resolveInitialConfig("org/tuning", "Q4").config.loadMode,
+ "mmap",
+ );
+});
+
+test("absent and explicit reset remain different request values", () => {
+ assert.deepEqual(llamaCppConfigPayload(undefined), {});
+ assert.deepEqual(llamaCppConfigPayload(managed), {
+ llama_cpp_config: managed,
+ });
+ assert.equal(
+ "llama_cpp_config" in toApiOverride(DEFAULT_PER_MODEL_CONFIG),
+ false,
+ );
+ assert.deepEqual(
+ fromApiOverride({}, { ...DEFAULT_PER_MODEL_CONFIG, llamaCppConfig: custom })
+ .llamaCppConfig,
+ custom,
+ );
+});
+
+test("UTF-8 cap rejects an oversized edit without erasing its saved predecessor", () => {
+ store.clear();
+ savePerModelConfig("org/model", "Q4", {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ llamaCppConfig: custom,
+ });
+ const oversized = { ...custom, ini: "é".repeat(32769) };
+ assert.equal(normalizeLlamaCppConfig(oversized), undefined);
+ assert.equal(
+ savePerModelConfig("org/model", "Q4", {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ llamaCppConfig: oversized,
+ }),
+ false,
+ );
+ assert.deepEqual(
+ resolveInitialConfig("org/model", "Q4").config.llamaCppConfig,
+ custom,
+ );
+ assert.equal(
+ normalizeLlamaCppConfig({ ...custom, ini: "é".repeat(32768) })?.mode,
+ "custom",
+ );
+});
+
+test("blank custom source is rejected without erasing its saved predecessor", () => {
+ store.clear();
+ savePerModelConfig("org/model", "Q4", {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ llamaCppConfig: custom,
+ });
+ const blank = { ...custom, ini: " \n\t " };
+ assert.equal(normalizeLlamaCppConfig(blank), undefined);
+ assert.equal(
+ savePerModelConfig("org/model", "Q4", {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ llamaCppConfig: blank,
+ }),
+ false,
+ );
+ assert.deepEqual(
+ resolveInitialConfig("org/model", "Q4").config.llamaCppConfig,
+ custom,
+ );
+});
+
+test("selector suggestions never implicitly select a sole named section", () => {
+ assert.deepEqual(customConfigSections("[*]\n[only]\nctx-size=56000"), [
+ "only",
+ ]);
+ assert.deepEqual(customConfigSections("[ preset]\nctx-size=56000"), [
+ "preset",
+ ]);
+ // llama.cpp files keys above the first header under "default", not under [*].
+ assert.deepEqual(customConfigSections("c=1\n[*]\nngl=-1\n[large]\nc=9"), [
+ "default",
+ "large",
+ ]);
+ assert.deepEqual(customConfigSections("; note\n[large]\nc=9"), ["large"]);
+ assert.deepEqual(customConfigSections("[a]\r\nc=1\r\n[b] ; x\r\nc=2\r\n"), [
+ "a",
+ "b",
+ ]);
+ assert.deepEqual(
+ normalizeLlamaCppConfig({ ...custom, section: null })?.mode,
+ "custom",
+ );
+ // Backend validates null against the source; the frontend does not invent a selection.
+ assert.equal(
+ (
+ llamaCppConfigPayload({ ...custom, section: null })
+ .llama_cpp_config as typeof custom
+ ).section,
+ null,
+ );
+});
+
+test("large per-model sources still obey the aggregate storage budget", () => {
+ store.clear();
+ const evicted: { modelId: string; ggufVariant: string | null }[] = [];
+ for (let index = 0; index < 22; index += 1) {
+ assert.equal(
+ savePerModelConfig(
+ `org/model-${index}`,
+ null,
+ {
+ ...DEFAULT_PER_MODEL_CONFIG,
+ llamaCppConfig: { ...custom, ini: "#".repeat(60_000) },
+ },
+ evicted,
+ ),
+ true,
+ );
+ }
+ assert.ok(evicted.length > 0);
+ assert.ok(
+ new TextEncoder().encode(store.get(PER_MODEL_CONFIG_STORAGE_KEY)!).length <=
+ 1024 * 1024,
+ );
+ assert.equal(
+ resolveInitialConfig("org/model-21", null).config.llamaCppConfig?.mode,
+ "custom",
+ );
+});
+
+test("a diffusion load sends managed in place of a custom config, never omits it", () => {
+ const runtime = readSrc("features/chat/hooks/use-chat-model-runtime.ts");
+ assert.equal(
+ runtime.match(
+ /llamaCppConfigPayload\(loadLlamaCppConfig, \{\s*isDiffusion: targetIsDiffusion/g,
+ )?.length,
+ 2,
+ "validate and load both reset a diffusion target explicitly",
+ );
+ const compare = readSrc("features/chat/shared-composer.tsx");
+ assert.equal(
+ compare.match(
+ /llamaCppConfigPayload\(ownConfig\.llamaCppConfig, \{\s*isDiffusion: resolvedIsDiffusion === true/g,
+ )?.length,
+ 2,
+ "compare validate and load reset a diffusion target too",
+ );
+ // Both GGUF autoload paths record what launched, including the default-model fallback.
+ assert.equal(
+ readSrc("features/chat/api/chat-adapter.ts").split(
+ "...loadedLlamaCppConfigFields(loadResp",
+ ).length - 1,
+ 2,
+ );
+ assert.deepEqual(llamaCppConfigPayload(custom, { isDiffusion: true }), {
+ llama_cpp_config: managed,
+ });
+ assert.deepEqual(llamaCppConfigPayload(custom), { llama_cpp_config: custom });
+ assert.deepEqual(llamaCppConfigPayload(managed, { isDiffusion: true }), {
+ llama_cpp_config: managed,
+ });
+ // Unknown config still resets, or the backend inherits a saved custom source.
+ assert.deepEqual(llamaCppConfigPayload(undefined, { isDiffusion: true }), {
+ llama_cpp_config: managed,
+ });
+});
+
+test("section suggestions trim padding the way the server does", () => {
+ assert.deepEqual(customConfigSections("[*]\n[ fast ]\nc=1\n[fast]\n"), [
+ "fast",
+ ]);
+});
+
+test("a custom config locks the managed rows and keeps the editor outside them", () => {
+ const page = readSrc(
+ "features/model-picker/components/model-config-page.tsx",
+ );
+ const fieldsetEnd = page.indexOf("</fieldset>");
+ assert.ok(page.indexOf("disabled={customActive}") < fieldsetEnd);
+ assert.ok(page.indexOf("<CustomLlamaConfigEditor") > fieldsetEnd);
+ assert.match(page, /!customActive &&\s*shouldRequestMemoryEstimate/);
+ // Custom launches honour disable_vision, so its switch stays outside the lock.
+ assert.ok(page.indexOf("hideVision={customActive}") < fieldsetEnd);
+ // Hidden behind Advanced settings unless custom mode is already on.
+ assert.match(
+ page,
+ /!resolvedIsDiffusion &&\s*\(showAdvanced \|\| customActive\)/,
+ );
+ assert.ok(page.indexOf("{customActive && <VisionRow") > fieldsetEnd);
+});
diff --git a/studio/frontend/tests/llama-cpp-custom-status-managed-fields.test.ts b/studio/frontend/tests/llama-cpp-custom-status-managed-fields.test.ts
new file mode 100644
index 00000000000..0529b938927
--- /dev/null
+++ b/studio/frontend/tests/llama-cpp-custom-status-managed-fields.test.ts
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { register } from "node:module";
+import test from "node:test";
+
+import { installLocalStorageFake } from "./helpers/kit.ts";
+
+installLocalStorageFake();
+register("./store-settings-resolver.mjs", import.meta.url);
+
+const { useChatRuntimeStore } = await import(
+ "../src/features/chat/stores/chat-runtime-store.ts"
+);
+const { applyActiveModelStatusToStore } = await import(
+ "../src/features/chat/lib/apply-inference-status-to-store.ts"
+);
+
+const MODEL = "unsloth/Qwen3-1.7B-GGUF";
+const custom = {
+ version: 1,
+ mode: "custom",
+ ini: "[*]\nctx-size = 3072\ndevice = none\ngpu-layers = 0\n",
+ section: null,
+} as const;
+
+// What /status reports after a custom load: the backend records the INI's placement as
+// manual/0 layers and its ctx-size as the requested context.
+function statusFor(config: unknown) {
+ return {
+ active_model: MODEL,
+ model_identifier: MODEL,
+ is_gguf: true,
+ gpu_memory_mode: "manual",
+ gpu_layers: 0,
+ requested_context_length: 3072,
+ context_length: 3072,
+ requested_llama_cpp_config: config,
+ } as never;
+}
+
+function managedAuto() {
+ useChatRuntimeStore.setState({
+ modelLoading: false,
+ gpuMemoryMode: "auto",
+ loadedGpuMemoryMode: "auto",
+ gpuLayers: -1,
+ loadedGpuLayers: null,
+ customContextLength: null,
+ loadedCustomContextLength: null,
+ params: { ...useChatRuntimeStore.getState().params, checkpoint: MODEL },
+ });
+}
+
+test("a custom load's INI placement does not overwrite the managed controls", () => {
+ managedAuto();
+ applyActiveModelStatusToStore(statusFor(custom), { previousCheckpoint: MODEL });
+ const s = useChatRuntimeStore.getState();
+ assert.equal(s.llamaCppConfig?.mode, "custom");
+ assert.equal(s.gpuMemoryMode, "auto", "Use Studio settings would reload with Manual");
+ assert.equal(s.gpuLayers, -1, "Use Studio settings would reload with --gpu-layers 0");
+ assert.equal(s.customContextLength, null, "Use Studio settings would reload with ctx 3072");
+});
+
+test("a managed load reporting the same placement still hydrates the controls", () => {
+ managedAuto();
+ applyActiveModelStatusToStore(statusFor({ version: 1, mode: "managed" }), {
+ previousCheckpoint: MODEL,
+ });
+ const s = useChatRuntimeStore.getState();
+ assert.equal(s.gpuMemoryMode, "manual");
+ assert.equal(s.gpuLayers, 0);
+});
+
+test("a custom load's response leaves the managed placement alone", async () => {
+ const { managedGpuMemoryFields } = await import(
+ "../src/features/chat/stores/chat-runtime-store.ts"
+ );
+ const resp = { is_gguf: true, gpu_memory_mode: "manual", gpu_layers: 0, n_layers: 28 } as const;
+ assert.deepEqual(
+ managedGpuMemoryFields({ ...resp, requested_llama_cpp_config: custom }),
+ { ggufLayerCount: 28, moeLayerCount: null },
+ );
+ const managed = managedGpuMemoryFields({
+ ...resp,
+ requested_llama_cpp_config: { mode: "managed" },
+ }) as { gpuMemoryMode?: string; gpuLayers?: number };
+ assert.equal(managed.gpuMemoryMode, "manual");
+ assert.equal(managed.gpuLayers, 0);
+});
+
+test("a custom load's KV cache and speculative echo leave the managed controls alone", async () => {
+ const { managedKvCacheFields, managedSpeculativeSettings } = await import(
+ "../src/features/chat/stores/chat-runtime-store.ts"
+ );
+ const resp = { cache_type_kv: "f16", speculative_type: "none", spec_draft_n_max: null };
+ assert.deepEqual(managedKvCacheFields({ ...resp, requested_llama_cpp_config: custom }), {});
+ assert.deepEqual(managedSpeculativeSettings({ ...resp, requested_llama_cpp_config: custom }), {});
+ assert.equal(managedKvCacheFields(resp).kvCacheDtype, "f16");
+
+ useChatRuntimeStore.setState({
+ modelLoading: false,
+ kvCacheDtype: null,
+ loadedKvCacheDtype: null,
+ speculativeType: "auto",
+ loadedSpeculativeType: "auto",
+ params: { ...useChatRuntimeStore.getState().params, checkpoint: MODEL },
+ });
+ applyActiveModelStatusToStore(
+ { ...(statusFor(custom) as object), cache_type_kv: "f16", speculative_type: "none" } as never,
+ { previousCheckpoint: MODEL },
+ );
+ const s = useChatRuntimeStore.getState();
+ assert.equal(s.kvCacheDtype, null);
+ assert.equal(s.speculativeType, "auto");
+});
diff --git a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
index 96de26b75f3..4e8ea726265 100644
--- a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
+++ b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts
@@ -204,12 +204,12 @@ const ROWS: Row[] = [
mlxRequest: 32768,
transformersRequest: 32768,
note:
- "Not a future record: STORAGE_SCHEMA_VERSION is 7, so v4 is a v4-client record " +
+ "Not a future record: STORAGE_SCHEMA_VERSION is 8, so v4 is a v4-client record " +
"and reads normally.",
},
{
- name: "version 8 (genuinely future)",
- raw: { version: 8, customContextLength: 32768 },
+ name: "version 9 (genuinely future)",
+ raw: { version: 9, customContextLength: 32768 },
normalizedPin: null,
rawPin: 32768,
isDefault: true,
@@ -330,6 +330,7 @@ test("a patched pin round-trips through storage on both backends", () => {
test("both pin shapes are stamped version 1, so neither is distinguishable by version", () => {
assert.equal(stampedVersion({ customContextLength: 32768 }), 1);
assert.equal(stampedVersion({ maxSeqLength: 32768 }), 1);
+ // The current client's forwards guard is `version > 8`, and v1 invites any client
// back to v1 to rewrite the record.
assert.equal(
stage({ version: 1, customContextLength: 32768 }).remembered,
@@ -345,6 +346,10 @@ test("both pin shapes are stamped version 1, so neither is distinguishable by ve
);
assert.equal(
stage({ version: 8, customContextLength: 32768 }).remembered,
+ true,
+ );
+ assert.equal(
+ stage({ version: 9, customContextLength: 32768 }).remembered,
false,
);
});
diff --git a/studio/frontend/tests/per-model-params.test.ts b/studio/frontend/tests/per-model-params.test.ts
index 78c562111a0..88792408ea6 100644
--- a/studio/frontend/tests/per-model-params.test.ts
+++ b/studio/frontend/tests/per-model-params.test.ts
@@ -420,3 +420,18 @@ for (const [label, id] of [
assert.equal(replayed.temperature, 0.2);
});
}
+
+test("a custom llama.cpp repeat-penalty seeds the slider; otherwise it is kept", async () => {
+ const { mergeBackendRecommendedInference } =
+ await import("../src/features/chat/presets/preset-policy.ts");
+ const merge = (inference: Record<string, number>) =>
+ mergeBackendRecommendedInference({
+ current: params({ repetitionPenalty: 1.3 }),
+ response: { inference, is_gguf: true, context_length: 4096 },
+ modelId: QWEN,
+ presetSource: "builtin-default",
+ loadedContextLength: 4096,
+ }).repetitionPenalty;
+ assert.equal(merge({ repetition_penalty: 1.1 }), 1.1);
+ assert.equal(merge({ temperature: 0.7 }), 1.3);
+});
diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
index 9844e99ca1c..256d9fc51bf 100644
--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts
@@ -473,6 +473,8 @@ test("every PerModelConfig field is either compared or deliberately excluded", (
// Qualifies selectedGpuIds rather than adding a dimension of its own: it is read, as
// the reconciler's namespace argument, but /status has no field to compare it against.
"selectedGpuIndexKind",
+ // Custom intent always reaches backend preflight and compiled-identity comparison.
+ "llamaCppConfig",
// Compared through standing.splitRatio, which the caller seeds from it.
"tensorSplit",
]);
diff --git a/studio/frontend/tests/server-tuning-settings.test.ts b/studio/frontend/tests/server-tuning-settings.test.ts
index eac1cfdc97b..7e722f70f43 100644
--- a/studio/frontend/tests/server-tuning-settings.test.ts
+++ b/studio/frontend/tests/server-tuning-settings.test.ts
@@ -113,11 +113,19 @@ test("a record only claims the new schema version when it carries one", () => {
// toStoredConfig stamps the OLDEST version that understands every field
// present, so an older client can still rewrite a record it fully knows.
const source = readSrc("features/model-picker/model-config/per-model-config.ts");
- assert.match(source, /const STORAGE_SCHEMA_VERSION = 7;/);
+ assert.match(source, /const STORAGE_SCHEMA_VERSION = 8;/);
+ assert.match(source, /const PRE_LLAMA_CPP_CONFIG_SCHEMA_VERSION = 7;/);
assert.match(source, /const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;/);
assert.match(source, /const PRE_REASONING_BUDGET_SCHEMA_VERSION = 5;/);
assert.match(source, /const PRE_SERVER_TUNING_SCHEMA_VERSION = 4;/);
- assert.match(source, /mlxKvQuant != null\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);
+ assert.match(
+ source,
+ /normalized\.llamaCppConfig !== undefined\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/,
+ );
+ assert.match(
+ source,
+ /mlxKvQuant != null\)\s*\{\s*return PRE_LLAMA_CPP_CONFIG_SCHEMA_VERSION;/,
+ );
assert.match(
source,
/hasReasoningBudget\)\s*\{\s*return PRE_MLX_KV_QUANT_SCHEMA_VERSION;/,
diff --git a/studio/frontend/tests/tensor-parallel-row-gating.test.ts b/studio/frontend/tests/tensor-parallel-row-gating.test.ts
index c04172799d0..9c9c52e86aa 100644
--- a/studio/frontend/tests/tensor-parallel-row-gating.test.ts
+++ b/studio/frontend/tests/tensor-parallel-row-gating.test.ts
@@ -45,5 +45,5 @@ test("the Tensor Parallelism row is gated out for diffusion models", () => {
test("the Vision row it sits beside stays gated too", () => {
// Both rows are unsupported for the same reason; regating one and not the other is
// the state this change exists to end.
- assert.equal(gateAbove("checked={!config.disableVision}"), "!isDiffusion");
+ assert.equal(gateAbove("hideVision ? null : <VisionRow config={config} update={update} />"), "!isDiffusion");
});
diff --git a/studio/frontend/tests/vision-toggle-gating.test.ts b/studio/frontend/tests/vision-toggle-gating.test.ts
index 5e7892c2791..918a4671e0f 100644
--- a/studio/frontend/tests/vision-toggle-gating.test.ts
+++ b/studio/frontend/tests/vision-toggle-gating.test.ts
@@ -91,19 +91,20 @@ test("the Vision row exists only in the GGUF half of Advanced Settings", () => {
1,
"expected exactly one Vision switch in the file",
);
+ assert.ok(bodyOf("VisionRow").includes(wiring));
assert.ok(
- gguf.includes(wiring),
+ gguf.includes("<VisionRow"),
"Vision switch is not in GgufAdvancedSettings",
);
assert.ok(
- !mlx.includes(wiring),
+ !mlx.includes("VisionRow"),
"Vision switch leaked into MlxAdvancedSettings",
);
assert.ok(
!mlx.includes("disableVision"),
"MlxAdvancedSettings reads disableVision, which it cannot act on",
);
- assert.ok(gguf.includes(">Vision</span>"));
+ assert.ok(bodyOf("VisionRow").includes(">Vision</span>"));
// And the GGUF half only renders under target.isGguf, so a non-GGUF target
// shows no Vision switch at all.
@@ -286,7 +287,9 @@ test("the Vision row is gated out for diffusion models", () => {
const lines = CONFIG_PAGE.split("\n");
const visionAt = lines.findIndex((line) =>
- line.includes("checked={!config.disableVision}"),
+ line.includes(
+ "hideVision ? null : <VisionRow config={config} update={update} />",
+ ),
);
assert.notEqual(visionAt, -1, "no Vision switch to gate");
diff --git a/tests/studio/test_chat_autoload_failure_gate.py b/tests/studio/test_chat_autoload_failure_gate.py
index 4c380b24f80..fefc098b71a 100644
--- a/tests/studio/test_chat_autoload_failure_gate.py
+++ b/tests/studio/test_chat_autoload_failure_gate.py
@@ -551,6 +551,11 @@ function reasoningCapsFromLoad(_x: any) { return {}; }
function resolveToolsEnabledOnLoad(_x: any) { return {}; }
function loadedGpuMemoryFields(_x: any) { return {}; }
function resolveLoadedSpeculativeSettings(_x: any) { return {}; }
+function managedGpuMemoryFields(_x: any) { return {}; }
+function managedKvCacheFields(_x: any) { return {}; }
+function managedSpeculativeSettings(_x: any) { return {}; }
+function llamaCppConfigPayload(c: any) { return c === undefined ? {} : { llama_cpp_config: c }; }
+function loadedLlamaCppConfigFields(r: any, sent: any) { const c = r.requested_llama_cpp_config ?? sent; return { llamaCppConfig: c, loadedLlamaCppConfig: c ?? null, llamaCppConfigSummary: r.llama_cpp_config_summary ?? null }; }
function isMultimodalResponse(_x: any) { return false; }
async function listCachedGguf(signal?: any) {
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index 405f6b4d78f..0062e0104ce 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -1552,7 +1552,10 @@ def test_forget_settings_is_not_locked_by_unloadable_extra_args():
"""Forget only deletes, so invalid saved llama args must not lock it: the args gates
apply to a save only."""
gate = " ".join(_save_button_gate().split())
- assert "(remember && ((!extraArgsLoadable && !sharedExtraArgsCleared) ||" in gate, gate
+ assert re.search(
+ r"\(remember && \((?:\([^()]*\) \|\| )?\(!extraArgsLoadable && !sharedExtraArgsCleared\) \|\|",
+ gate,
+ ), gate
assert "sharedExtraArgsRefused || extraArgsHydrating))" in gate, gate