Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: add per-model custom llama.cpp INI configuration (#10783)
Etherl864fec40e8a8
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 7ada48eef08..ea2395c8023 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -56,6 +56,13 @@ from typing import (import httpxfrom core.inference.chat_template_helpers import build_dac_tts_prompt+from core.inference.llama_custom_config import (+ CompiledCustomConfig,+ CustomConfigError,+ CustomConfigSource,+ compile_custom_config,+ parse_config_source,+)from core.inference.context_window import (_COMPACTION_HEADROOM_RATIO,clamp_compaction_headroom_ratio,@@ -625,8 +632,10 @@ class GgufLoadIntent:force_reload: bool = Falsecpu_fallback: bool = False+ llama_cpp_config: Optional[CustomConfigSource] = Nonedef __post_init__(self):+ object.__setattr__(self, "llama_cpp_config", parse_config_source(self.llama_cpp_config))for key in ("tensor_split", "gpu_ids", "extra_args"):value = getattr(self, key)if value is not None:@@ -635,6 +644,11 @@ class GgufLoadIntent:object.__setattr__(self, "gpu_ids", None)+def _is_custom(intent) -> bool:+ source = getattr(intent, "llama_cpp_config", None)+ return source is not None and source.mode == "custom"++class _CpuFallbackRuntime(NamedTuple):tempdir: tempfile.TemporaryDirectorysource_binary: Path@@ -7754,6 +7768,7 @@ class LlamaCppBackend:self._requested_n_ctx: int = 0# Last healthy caller intent for crash recovery. Memory-only and never logged.self._last_load_intent: Optional[GgufLoadIntent] = None+ self._compiled_custom_config: Optional[CompiledCustomConfig] = Noneself._mtp_runtime_fallback_lock = threading.Lock()self._mtp_runtime_fallback_in_progress = False# Background watchdog so an MTP+tensor crash recovers even when no request@@ -8487,6 +8502,8 @@ class LlamaCppBackend:never competed with."""if self._arch_gate_forced_cpu:return True+ if self._compiled_custom_config is not None:+ return self._compiled_custom_config.cpu_only and self._gpu_offload_active is Falsereturn (self._gpu_memory_mode == "manual"and self._gpu_layers == 0@@ -8571,6 +8588,29 @@ class LlamaCppBackend:def last_load_intent(self) -> Optional[GgufLoadIntent]:return self._last_load_intent+ @property+ def requested_llama_cpp_config(self) -> Optional[dict]:+ source = getattr(self._last_load_intent, "llama_cpp_config", None)+ return source.to_wire() if source is not None else None++ @property+ def llama_cpp_config_summary(self) -> Optional[dict]:+ compiled = getattr(self, "_compiled_custom_config", None)+ return compiled.summary() if compiled is not None else None++ def _custom_runtime_matches(self, intent: GgufLoadIntent) -> bool:+ resident = self._last_load_intent+ return (+ not intent.force_reload+ and getattr(self, "_compiled_custom_config", None) is not None+ and resident is not None+ and resident.llama_cpp_config == intent.llama_cpp_config+ and intent.model_identifier == self._model_identifier+ and (intent.hf_variant or None) == (self._hf_variant or None)+ and intent.disable_vision == resident.disable_vision+ and not self._binary_changed_since_launch()+ )+def _runtime_matches_intent(self, intent: GgufLoadIntent, effective_extra_args: Optional[list[str]]) -> bool:@@ -8578,6 +8618,8 @@ class LlamaCppBackend:if intent.force_reload:return False+ if _is_custom(intent) or getattr(self, "_compiled_custom_config", None) is not None:+ return _is_custom(intent) and self._custom_runtime_matches(intent)if self._requested_n_ctx != int(intent.n_ctx):return Falseif not self._is_diffusion and self._requested_n_parallel != max(1, int(intent.n_parallel)):@@ -23313,6 +23355,291 @@ class LlamaCppBackend:self._stdout_thread.start()return True+ def prepare_custom_config(self, intent: GgufLoadIntent) -> CompiledCustomConfig:+ """Compile the INI against the selected llama-server without touching the resident model."""+ if getattr(self, "_llama_update_in_progress", False):+ raise CustomConfigError("llama.cpp is updating; try again in a moment")+ binary = self._exec_path_for_launch(self._find_llama_server_binary())+ if not binary:+ raise LlamaServerNotFoundError(LLAMA_SERVER_NOT_FOUND_DETAIL)+ caps = self.probe_server_capabilities(binary)+ if not caps.get("help_probe_ok"):+ raise CustomConfigError("Could not read the selected llama-server's options")+ return compile_custom_config(+ intent.llama_cpp_config, caps.get("flags") or {}, caps.get("switch_flags") or ()+ )++ def _load_custom_model(+ self,+ intent: GgufLoadIntent,+ *,+ load_cancel_event: Optional[threading.Event] = None,+ ) -> bool:+ """Launch Studio's transport flags plus the compiled INI: no tuning, no fallback."""+ from core.inference.model_ids import public_model_id++ self._cancel_event.clear()+ epoch = self._unload_epoch++ def cancelled():+ return (+ self._cancel_event.is_set()+ or self._unload_epoch != epoch+ or bool(load_cancel_event is not None and load_cancel_event.is_set())+ )++ compiled = self.prepare_custom_config(intent)+ model_path, projector = intent.gguf_path, intent.mmproj_path+ download_cancel = (+ _CombinedCancelEvent(self._cancel_event, load_cancel_event)+ if load_cancel_event+ else self._cancel_event+ )+ if intent.hf_repo and not model_path:+ with _hf_offline_if_unreachable():+ model_path = self._download_gguf(+ hf_repo = _resolve_repo_id_casing(intent.hf_repo),+ hf_variant = intent.hf_variant,+ hf_token = intent.hf_token,+ cancel_event = download_cancel,+ allow_smaller_fallback = False,+ )+ if cancelled():+ return False+ if not model_path or not Path(model_path).is_file():+ raise CustomConfigError("The selected GGUF file is unavailable")+ refusal = self._non_chat_gguf_refusal_for_path(model_path, intent.model_identifier)+ if refusal:+ raise CustomConfigError(refusal)+ # The route could not classify an uncached file; diffusion GGUFs run outside llama-server.+ if self._gguf_path_is_diffusion(model_path, intent.model_identifier):+ raise CustomConfigError(+ "Diffusion GGUFs run on the diffusion runner; use Studio settings"+ )+ # The INI's own projector switch-off wins over Studio's --mmproj (last wins on argv).+ if {"--no-mmproj", "--no-mmproj-auto"} & set(compiled.options):+ projector = None+ # Downloaded even with vision off, as managed loads do: only its header tells an+ # audio-only encoder (kept) from an image tower (dropped below).+ elif intent.is_vision and not projector and intent.hf_repo:+ with _hf_offline_if_unreachable():+ projector = self._download_mmproj(+ hf_repo = intent.hf_repo,+ hf_token = intent.hf_token,+ cancel_event = download_cancel,+ near_path = model_path,+ )+ if projector:+ projector = self._resolve_launch_mmproj_path(+ model_path = model_path, mmproj_path = projector+ )+ if projector and intent.disable_vision and not _mmproj_env_is_audio_only(projector):+ projector = None++ binary = self._exec_path_for_launch(self._find_llama_server_binary())+ caps = self.probe_server_capabilities(binary)+ # The INI is the only configuration source: inherited LLAMA_ARG_* would be a second one.+ env = {+ k: v+ for k, v in self._llama_server_env_for_binary(binary).items()+ if not k.startswith("LLAMA_ARG_")+ }+ scrub_denied_env(env) # LLAMA_API_KEY: the child would refuse Studio's own requests+ # HSA enumeration dies on an arch the build lacks before --device is read: mask like managed loads.+ survivors = self._arch_gate_survivors(binary)+ if survivors:+ self._emit_child_gpu_visibility(+ env, ",".join(str(i) for i in survivors), prefer_rocr = True+ )+ port = self._find_free_port()+ cmd = [binary, "-m", model_path, "--host", "127.0.0.1", "--port", str(port)]+ cmd += ["--alias", public_model_id(intent.model_identifier), "--jinja"]+ if compiled.n_parallel is not None:+ cmd += ["--parallel", str(compiled.n_parallel)]+ if projector:+ cmd += ["--mmproj", projector]+ api_key = None+ if os.getenv("UNSLOTH_DIRECT_STREAM", "0") == "1":+ import secrets+ api_key = secrets.token_urlsafe(32)+ cmd += ["--api-key", api_key]+ cmd += list(compiled.argv)+ resolved = replace(intent, gguf_path = model_path, mmproj_path = projector, verified_gguf = None)++ with self._lock:+ if cancelled():+ return False+ with self._spawn_lock:+ if self._spawn_is_stale():+ return False+ self._begin_load_warnings()+ self._kill_process()+ self._cleanup_cpu_fallback_runtime()+ self._model_identifier = intent.model_identifier+ self._read_gguf_metadata(model_path)+ self._gguf_path = model_path+ self._gguf_load_identity = self._gguf_load_source_identity(model_path, projector)+ self._hf_repo, self._hf_variant = intent.hf_repo, intent.hf_variant+ self._port, self._api_key = port, api_key+ self._launch_binary_revision = self._binary_revision(binary)+ self._slot_save_dir = self._chat_template_override = None+ self._cpu_fallback_reason = self._mmproj_fallback_reason = None+ self._mtp_draft_path = self._mtp_draft_suppressed_path = None+ self._spec_fallback_reason = self._spec_drafter_kind = None+ self._mtp_runtime_fallback_active = self._dflash_retry_needed = False+ self._memory_state = self._memory_direct_io = self._memory_pending_launch = None+ self._is_vision = bool(projector)+ self._disable_vision = self._vision_disabled_by_user = intent.disable_vision+ self._mmproj_has_audio, self._mmproj_accepts_image = False, True+ self._mmproj_projector_type = None+ if projector:+ from utils.models.gguf_metadata import (+ mmproj_capabilities,+ read_mmproj_projector_type,+ )+ self._mmproj_has_audio, self._mmproj_accepts_image = mmproj_capabilities(projector)+ self._mmproj_projector_type = read_mmproj_projector_type(projector)+ self._is_audio = self._audio_probed = self._has_audio_input = False+ self._audio_type = None+ # Every managed knob reads as unset: the INI, not Studio, chose them.+ self._extra_args, self._requested_extra_args = [], []+ self._extra_args_source = (intent.model_identifier, intent.hf_variant)+ self._requested_n_batch = self._requested_n_ubatch = None+ self._requested_load_mode = self._requested_spec_draft_cache_type = None+ self._requested_ctx_checkpoints = self._requested_cache_ram = None+ self._effective_cache_types = (+ compiled.option("-ctk", "--cache-type-k") or "f16",+ compiled.option("-ctv", "--cache-type-v") or "f16",+ )+ self._requested_cache_types = self._effective_cache_types+ self._cache_type_kv = compiled.option("-ctk", "--cache-type-k")+ self._speculative_type = self._requested_spec_mode = self._spec_draft_n_max = None+ self._fit_load_mode_flags, self._vram_fraction_launched = [], None+ self._gpu_memory_mode, self._gpu_layers, self._n_cpu_moe = "auto", -1, 0+ self._tensor_parallel = self._arch_gate_forced_cpu = False+ self._tensor_split = self._gpu_ids = self._requested_gpu_ids = None+ self._auto_tensor_split = self._auto_tensor_split_emitted = None+ self._spill_plan_flags = []+ self._layer_preserves_tensor_intent = self._capability_probe_inconclusive = False+ self._flash_attn_enabled = self._swa_full = self._kv_cache_unified = False+ self._prompt_cache_disabled = "--no-cache-prompt" in compiled.options+ budget = compiled.option("--reasoning-budget")+ self._reasoning_budget = self._requested_reasoning_budget = (+ int(budget) if budget is not None and re.fullmatch(r"-?[0-9]+", budget) else -1+ )+ self._reasoning_budget_message = self._requested_reasoning_budget_message = (+ compiled.option("--reasoning-budget-message") or ""+ )+ try:+ if cancelled() or not self._start_llama_process(+ cmd,+ env,+ child_gpu_physical_ids = None if compiled.cpu_only else tuple(survivors) or None,+ ):+ self._kill_process()+ return False+ if not self._wait_for_health(timeout = 600.0, cancelled = cancelled):+ was_cancelled = cancelled() or getattr(self, "_health_wait_cancelled", False)+ self._kill_process()+ if was_cancelled:+ return False+ # llama-server's own reason (unknown device, bad value) is otherwise only in the log.+ reason = next(+ (+ l.strip()+ for l in reversed(self._stdout_lines[-50:])+ if "error" in l.lower()+ ),+ "check the server log",+ )+ if survivors and "device" in reason.lower():+ reason += f" (only supported GPUs are visible, numbered from 0 to {len(survivors) - 1})"+ raise CustomConfigError(+ f"llama-server did not start with this configuration: {reason}"+ )+ props = self._query_server_props() or {}+ n_ctx = (props.get("default_generation_settings") or {}).get("n_ctx")+ slots = props.get("total_slots")+ if not isinstance(n_ctx, int) or not isinstance(slots, int) or slots < 1:+ raise CustomConfigError("llama-server did not report its context and slots")+ template = props.get("chat_template")+ if isinstance(template, str):+ self._chat_template = template+ flags = detect_reasoning_flags(+ template, intent.model_identifier, log_source = "custom config"+ )+ for name in (+ "supports_reasoning",+ "reasoning_style",+ "reasoning_always_on",+ "reasoning_effort_levels",+ "supports_preserve_thinking",+ "preserve_thinking_default",+ "supports_tools",+ ):+ setattr(self, "_" + name, flags[name])+ self._commit_effective_parallel_slots(slots)+ self._requested_n_parallel = slots+ self._requested_n_ctx = 0+ self._effective_context_length = n_ctx+ self._max_context_length = self._context_length+ # A unified cache is one shared pool (llama-server enables it for np auto):+ # trust its startup line, else the options.+ unified = next(+ (+ m[1] == "true"+ for line in reversed(self._stdout_lines)+ if (m := re.search(r"kv_unified = '(true|false)'", line))+ ),+ compiled.n_parallel is None+ or "--kv-unified" in compiled.options+ or "-kvu" in compiled.options,+ )+ if "--no-kv-unified" in compiled.options or "-no-kvu" in compiled.options:+ unified = False+ self._kv_cache_unified = unified+ per_slot = compiled.option("--kv-unified-per-slot")+ total = n_ctx if unified and not per_slot else n_ctx * slots+ # An explicit -c pins a unified pool; only c=0 sizes it to np x per-slot (server.cpp).+ pinned = compiled.option("-c", "--ctx-size")+ if unified and pinned and re.fullmatch(r"[0-9]+", pinned) and int(pinned) > 0:+ total = min(total, int(pinned))+ self._kv_cache_context_total = total+ self._has_video_input = bool((props.get("modalities") or {}).get("video"))+ self._idle_slot_clearing_active = _idle_slot_clearing_active(+ cmd, supports_cache_ram = bool(caps.get("supports_cache_ram"))+ )+ self._gpu_offload_active = (+ False+ if compiled.cpu_only and not projector+ else classify_gpu_offload_lines(self._stdout_lines)+ )+ self._compiled_custom_config = compiled+ self._last_load_intent = resolved+ if cancelled() or not self._publish_healthy():+ self._kill_process()+ return False+ except Exception:+ self._kill_process()+ raise+ finally:+ if not self._healthy:+ self._compiled_custom_config = None+ self._last_load_intent = None+ # Outside self._lock (non-reentrant; _apply_detected_audio takes it), as managed loads+ # do. The duplicate-load fast path requires the probe.+ try:+ detected = self._detect_audio_type_strict()+ self._audio_probed = True+ except Exception as exc:+ logger.debug("Audio probe failed: %s", exc)+ detected = None+ return (+ self._apply_detected_audio(detected)+ if intent.audio_codec_path is None+ else self._apply_detected_audio(detected, intent.audio_codec_path)+ )+@contextlib.contextmanagerdef _serial_load_scope(self):"""Hold the serial load lock and release pending launch state with it.@@ -23419,6 +23746,9 @@ class LlamaCppBackend:if getattr(self, "_llama_update_in_progress", False):raise RuntimeError("llama.cpp is updating; try again in a moment.")+ if _is_custom(intent):+ return self._load_custom_model(intent, load_cancel_event = load_cancel_event)+intent = self._preserve_cpu_fallback_intent(intent)tensor_parallel = intent.tensor_parallelgpu_memory_mode = intent.gpu_memory_mode@@ -24170,6 +24500,7 @@ class LlamaCppBackend:if _load_cancelled():logger.info("Load cancelled before diffusion server start")return False+ self._compiled_custom_config = Nonereturn self._start_diffusion_server(model_path = model_path,gguf_path = gguf_path,@@ -31605,6 +31936,7 @@ class LlamaCppBackend:# dropped: a respawn replays this snapshot arbitrarily later, so recovery# re-resolves the file instead of trusting a path this request verified.self._last_load_intent = replace(intent, verified_gguf = None)+ self._compiled_custom_config = Noneself._mtp_runtime_fallback_active = _mtp_active_for_launched_serverself._start_mtp_crash_watchdog()@@ -32324,6 +32656,13 @@ class LlamaCppBackend:if self._binary_changed_since_launch():logger.info("llama-server selection changed since launch; forcing a reload")return False+ if _is_custom(intent) or getattr(self, "_compiled_custom_config", None) is not None:+ if not (_is_custom(intent) and self._custom_runtime_matches(intent)):+ return False+ self._last_load_intent = replace(+ intent, gguf_path = intent.gguf_path or self._gguf_path, verified_gguf = None+ )+ return Trueintent = self._preserve_cpu_fallback_intent(intent, source_matches = True)# The stored state is what LAUNCHED, and on a virtualised Metal device that is# the CPU-pinned rewrite, not the request. Apply the same rewrite here@@ -32553,6 +32892,7 @@ class LlamaCppBackend:self._launch_binary_revision = ()self._cpu_fallback_reason = Noneself._last_load_intent = None+ self._compiled_custom_config = Noneself._mtp_runtime_fallback_active = Falseself._hf_variant = Noneself._is_vision = False@@ -34406,6 +34746,8 @@ class LlamaCppBackend:"""# Cheap async-safe gate: only our live MTP+tensor launch, not cancelled,# with a snapshot to replay.+ if getattr(self, "_compiled_custom_config", None) is not None:+ return Falseif self._cancel_event.is_set():return Falseif not self._mtp_runtime_fallback_active:diff --git a/studio/backend/core/inference/llama_custom_config.py b/studio/backend/core/inference/llama_custom_config.pynew file mode 100644index 00000000000..9f463a61c2e--- /dev/null+++ b/studio/backend/core/inference/llama_custom_config.py@@ -0,0 +1,265 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Compile a per-model llama.cpp preset INI into llama-server argv.++Grammar follows llama.cpp common/preset.cpp: keys before any header form the ``default``+section, only ``[*]`` applies to every section, ``;``/``#`` start comments and the last+duplicate key wins.+Option names and arity come from the selected binary's ``--help`` catalog.+"""++from __future__ import annotations++from collections.abc import Mapping+from dataclasses import dataclass, field+import re++from . import llama_server_args as policy++MAX_INI_BYTES = 64 * 1024+_TRUE = frozenset({"on", "enabled", "true", "1"})+_FALSE = frozenset({"off", "disabled", "false", "0"})+# Studio selects the model and projector; preset files commonly carry both.+_IGNORED = frozenset({"-m", "--model", "-mm", "--mmproj"})+_PARALLEL = frozenset({"-np", "--parallel", "--n-parallel"})+# Router metadata in upstream preset files; llama-server itself has no such options.+_ROUTER_ONLY = frozenset({"load-on-startup", "stop-timeout", "dedup-cache-models", "version"})+# Flag -> (request field, Studio chat API bounds in models/inference.py).+_SAMPLING = {+ "--temp": ("temperature", 0.0, 2.0),+ "--top-p": ("top_p", 0.0, 1.0),+ "--top-k": ("top_k", -1, 100),+ "--min-p": ("min_p", 0.0, 1.0),+ "--repeat-penalty": ("repetition_penalty", 1.0, 2.0),+ "--presence-penalty": ("presence_penalty", 0.0, 2.0),+}+++class CustomConfigError(ValueError):+ """A configuration error whose message never echoes INI values."""+++@dataclass(frozen = True)+class CustomConfigSource:+ version: int = 1+ mode: str = "managed"+ ini: str | None = None+ section: str | None = None++ def to_wire(self) -> dict:+ if self.mode != "custom":+ return {"version": 1, "mode": "managed"}+ return {"version": 1, "mode": "custom", "ini": self.ini, "section": self.section}+++def parse_config_source(value) -> CustomConfigSource | None:+ if value is None:+ return None+ if isinstance(value, CustomConfigSource):+ value = value.to_wire()+ if not isinstance(value, Mapping) or value.get("version") != 1:+ raise CustomConfigError("llama_cpp_config must be an object with version 1")+ mode = value.get("mode")+ if mode == "managed" and set(value) <= {"version", "mode"}:+ return CustomConfigSource()+ if mode != "custom" or set(value) - {"version", "mode", "ini", "section"}:+ raise CustomConfigError("llama_cpp_config mode must be managed or custom")+ ini, section = value.get("ini"), value.get("section")+ if not isinstance(ini, str) or not ini.strip():+ raise CustomConfigError("Custom configuration requires INI text")+ if len(ini.encode("utf-8", "surrogatepass")) > MAX_INI_BYTES:+ raise CustomConfigError("INI text exceeds the 64 KiB limit")+ if any((ord(c) < 32 and c not in "\r\n\t") or ord(c) == 127 for c in ini):+ raise CustomConfigError("INI text contains control characters")+ if section is not None and (+ not isinstance(section, str) or not section or len(section) > 512 or "]" in section+ ):+ raise CustomConfigError("Select a valid INI section name")+ return CustomConfigSource(1, "custom", ini, section)+++@dataclass(frozen = True)+class CompiledCustomConfig:+ source: CustomConfigSource+ argv: tuple[str, ...]+ n_parallel: int | None+ options: dict = field(default_factory = dict)+ request_defaults: dict = field(default_factory = dict)+ diagnostics: tuple[str, ...] = ()+ # Every help-block spelling of a valued option, so lookups match whichever alias the INI used.+ aliases: dict = field(default_factory = dict)++ def option(self, *names: str):+ return next(+ (table[n] for table in (self.options, self.aliases) for n in names if n in table),+ None,+ )++ @property+ def cpu_only(self) -> bool:+ # -ngl 0 still offloads ops and the projector; --device none also moves the projector+ # (common/arg.cpp), but -mmdev and a drafter pick their own devices.+ return (+ self.option("-dev", "--device") == "none"+ and self.option("-mmdev", "--mmproj-device") in (None, "none")+ and (+ self.option("-md", "-hfd") is None+ or self.option("-devd", "--device-draft") == "none"+ )+ )++ def summary(self) -> dict:+ return {+ "mode": "custom",+ "section": self.source.section,+ "options": dict(self.options),+ "request_defaults": dict(self.request_defaults),+ "diagnostics": list(self.diagnostics),+ }+++def _sections(ini: str) -> dict[str, dict[str, str]]:+ sections: dict[str, dict[str, str]] = {"default": {}}+ current = "default"+ for number, raw in enumerate(re.split(r"\r\n|\r|\n", ini), 1):+ header = re.fullmatch(r"\[[ \t]*([^\]]+)\][ \t]*(?:[;#].*)?", raw)+ if header:+ current = header[1].strip()+ sections[current] = {}+ continue+ line = re.split(r"[;#]", raw, maxsplit = 1)[0].strip()+ if not line:+ continue+ entry = re.fullmatch(r"([A-Za-z_][A-Za-z0-9_.-]*)\s*=\s*(.*)", line)+ if not entry:+ raise CustomConfigError(f"Invalid INI syntax at line {number}")+ sections.setdefault(current, {})[entry[1]] = entry[2]+ return sections+++def _polarity(block: str) -> tuple[list[str], list[str]]:+ """Positive and negative spellings from a help block's declaration line.++ llama.cpp prints args then args_neg, each run ending in its long form:+ ``-kvo, --kv-offload, -nkvo, --no-kv-offload``."""+ head = re.match(r"(-[\w.-]+(?:, -[\w.-]+)*)", block or "")+ names = head[1].split(", ") if head else []+ cut = next((i + 1 for i, n in enumerate(names) if n.startswith("--")), len(names))+ return names[:cut], names[cut:]+++def _spelling(key: str, flags: Mapping) -> str | None:+ found = next((f for f in (f"--{key}", f"-{key}") if f in flags), None)+ if found is None and re.fullmatch(r"[A-Z][A-Z0-9_]*", key):+ # Presets may name an option by its env variable, which maps to the option itself.+ block = next((d for d in flags.values() if d and f"(env: {key})" in d), None)+ if block is not None:+ positive = [f for f in _polarity(block)[0] if f in flags]+ found = max(positive, key = len, default = None)+ return found+++def compile_custom_config(source, flags: Mapping, switch_flags) -> CompiledCustomConfig:+ """``flags``/``switch_flags`` are the selected binary's probed help catalog."""+ source = parse_config_source(source)+ if source is None or source.mode != "custom":+ raise CustomConfigError("Compilation requires custom mode")+ if "--port" not in flags:+ raise CustomConfigError(+ "The selected llama-server did not report its options; check the install"+ )+ switches = set(switch_flags)+ sections = _sections(source.ini)+ named = [s for s in sections if s != "*" and (s != "default" or sections[s])]+ section = source.section+ if section is None and named:+ if named != ["default"]:+ raise CustomConfigError("Select which INI section to use")+ section = "default"+ if section is not None and section not in named:+ raise CustomConfigError("The selected INI section does not exist")+ # llama.cpp resolves keys to options before cascading: the section overrides [*] per option.+ entries: dict = {}+ for name in ("*", section):+ owners: dict = {}+ for key, value in sections.get(name or "", {}).items():+ if key in _ROUTER_ONLY:+ continue+ flag = _spelling(key, flags)+ if flag is None:+ raise CustomConfigError(+ f"'{key[:80]}' is not an option of the selected llama-server"+ )+ # Two aliases in one section: llama.cpp keeps one arbitrarily, so refuse both.+ identity = flags[flag] or flag+ other = owners.setdefault(identity, key)+ if other != key:+ raise CustomConfigError(f"'{other[:80]}' and '{key[:80]}' set the same option")+ entries[identity] = (key, flag, value)++ argv: list[str] = []+ options: dict = {}+ aliases: dict = {}+ defaults: dict = {}+ diagnostics: list[str] = []+ n_parallel = None+ for identity, (key, flag, value) in entries.items():+ if flag in _IGNORED:+ note = "Studio selects the model and projector; m and mm entries are ignored."+ if note not in diagnostics:+ diagnostics.append(note)+ continue+ if flag in _PARALLEL:+ if (+ not re.fullmatch(r"[0-9]+", value)+ or not policy.PARALLEL_MIN <= int(value) <= policy.PARALLEL_MAX+ ):+ raise CustomConfigError(+ f"'{key}' must be between {policy.PARALLEL_MIN} and {policy.PARALLEL_MAX}"+ )+ n_parallel = int(value)+ continue+ if policy.is_managed_flag(flag):+ raise CustomConfigError(f"'{key[:80]}' is managed by Studio and cannot be set here")+ if flag in switches:+ if value.lower() not in _TRUE | _FALSE:+ raise CustomConfigError(f"'{key[:80]}' needs true or false")+ # Like common/preset.cpp: a negative alias (-nkvo, --no-x) inverts the value, and+ # false on a switch with no negative form is dropped.+ positive, negative = _polarity(identity)+ if flag not in positive + negative:+ positive, negative = [flag], []+ on = (value.lower() in _TRUE) != (flag in negative)+ pick = [f for f in (positive if on else negative) if f in flags]+ if not pick:+ continue+ flag = flag if flag in pick else max(pick, key = len)+ argv.append(flag)+ options[flag] = True+ continue+ if not value:+ raise CustomConfigError(f"'{key[:80]}' needs a value")+ argv += [flag, value]+ options[flag] = value+ aliases.update((f, value) for f, d in flags.items() if d and d == identity)+ sampling = next((s for s in _SAMPLING if s in flags and flags[s] == flags[flag]), None)+ if sampling is not None:+ name, low, high = _SAMPLING[sampling]+ try:+ number = int(value) if sampling == "--top-k" else float(value)+ except ValueError:+ raise CustomConfigError(f"'{key[:80]}' needs a number") from None+ # It becomes the chat default, so it must pass the chat request schema.+ if not low <= number <= high:+ raise CustomConfigError(f"'{key[:80]}' must be between {low} and {high}")+ defaults[name] = number+ if "--no-jinja" in options:+ raise CustomConfigError("jinja = false would break Studio's tool calling")+ try:+ policy.validate_extra_args(argv)+ except ValueError as exc:+ raise CustomConfigError(str(exc)) from None+ return CompiledCustomConfig(+ source, tuple(argv), n_parallel, options, defaults, tuple(diagnostics), aliases+ )diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex 716a9816f33..ac59d7be0a4 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -89,6 +89,7 @@ _DENYLIST_GROUPS: tuple[frozenset[str], ...] = (frozenset({"--path"}),frozenset({"--api-prefix"}),frozenset({"--reuse-port"}),+ frozenset({"--rpc"}),# Auth / TLS: Unsloth terminates auth; upstream --api-key / TLS shadows Unsloth's key and breaks the proxy hopfrozenset({"--api-key"}),frozenset({"--api-key-file"}),@@ -1803,6 +1804,7 @@ DENIED_ENV_VARS: tuple[str, ...] = ("LLAMA_ARG_HOST","LLAMA_ARG_PORT","LLAMA_ARG_REUSE_PORT",+ "LLAMA_ARG_RPC","LLAMA_ARG_N_PARALLEL","LLAMA_ARG_POOLING","LLAMA_ARG_EMBEDDINGS",diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 79783323cb9..6c90ef4735b 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -31,6 +31,7 @@ from core.inference.llama_server_args import (from core.inference.runtime_context import MAX_REQUESTABLE_CONTEXTfrom core.inference.video_families import MAX_VIDEO_NUM_FRAMESfrom picker.schemas import MAX_CHAT_TEMPLATE_BYTES+from models.llama_custom_config import LlamaCppConfigFieldsfrom utils.reasoning_budget import validate_reasoning_budget_message@@ -51,7 +52,7 @@ def resolve_inventory_handle(value: str) -> str:return resolved-class LoadRequest(BaseModel):+class LoadRequest(LlamaCppConfigFields):"""Request to load a model for inference"""model_path: str = Field(..., description = "Model identifier or local path")@@ -509,7 +510,7 @@ class SttLoadRequest(BaseModel):)-class ValidateModelRequest(BaseModel):+class ValidateModelRequest(LlamaCppConfigFields):"""Check whether an identifier resolves to a ModelConfig; does NOT load weights."""model_path: str = Field(..., description = "Model identifier or local path")@@ -794,6 +795,7 @@ class ValidateModelResponse(BaseModel):"""valid: bool = Field(..., description = "Whether the model identifier looks valid")+ llama_cpp_config_summary: Optional[Dict[str, Any]] = Nonemessage: str = Field(..., description = "Human-readable validation message")identifier: Optional[str] = Field(None, description = "Resolved model identifier")resident: bool = Field(@@ -1254,6 +1256,9 @@ class GenerateRequest(BaseModel):class _InferenceRuntimeFields(BaseModel):"""Runtime fields shared by load and status responses."""+ requested_llama_cpp_config: Optional[Dict[str, Any]] = None+ llama_cpp_config_summary: Optional[Dict[str, Any]] = None+is_vision: bool = Field(False, description = "Whether model is a vision model")is_diffusion: bool = Field(False, description = "Whether model is a block-diffusion model (DiffusionGemma)"diff --git a/studio/backend/models/llama_custom_config.py b/studio/backend/models/llama_custom_config.pynew file mode 100644index 00000000000..c36f673f32c--- /dev/null+++ b/studio/backend/models/llama_custom_config.py@@ -0,0 +1,23 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Shared wire fields for a selected model's llama.cpp configuration."""++from typing import Any, Optional++from pydantic import BaseModel, Field, field_validator+++class LlamaCppConfigFields(BaseModel):+ llama_cpp_config: Optional[dict[str, Any]] = Field(+ None,+ description = "Versioned per-model INI configuration, or an explicit managed-mode reset.",+ )++ @field_validator("llama_cpp_config", mode = "before")+ @classmethod+ def validate_llama_cpp_config(cls, value):+ if value is None:+ return None+ from core.inference.llama_custom_config import parse_config_source+ return parse_config_source(value).to_wire()diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 5526d54d93a..d59a9b08bea 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -8078,6 +8078,8 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:# running with no extras would come back carrying the arguments of the load# that just failed. None stays for "nothing was ever set", which is the only# case where inheriting is the right answer.+ requested_llama_cpp_config = getattr(llama_backend, "requested_llama_cpp_config", None),+ llama_cpp_config_summary = getattr(llama_backend, "llama_cpp_config_summary", None),requested_llama_extra_args = (Noneif llama_backend.is_diffusion@@ -8139,9 +8141,15 @@ def _gguf_load_response(is_lora = False,is_gguf = True,is_local_model = is_local_model,- inference = load_inference_config(- inference_identifier or llama_backend.model_identifier or model- ),+ # A custom INI's sampling seeds the chat controls like a model recommendation.+ inference = {+ **load_inference_config(+ inference_identifier or llama_backend.model_identifier or model+ ),+ **(getattr(llama_backend, "llama_cpp_config_summary", None) or {}).get(+ "request_defaults", {}+ ),+ },# Advisory, and None on nearly every load. Recorded by load_model when the# weights outgrow fast memory, so the client can say why generation is slow.# getattr: older/custom backend doubles predate this additive field.@@ -15215,6 +15223,8 @@ async def _prepare_load_placement(request: LoadRequest | ValidateModelRequest,extra_args: Optional[list[str]],) -> _LoadPlacement:+ if _custom_llama_config(request):+ return _LoadPlacement(None, None, False, _classify_diffusion_gguf(config))requested = request.gpu_ids or Noneif not config.is_gguf:return _LoadPlacement(requested, None, False, False)@@ -15599,6 +15609,12 @@ def _guard_chat_load_against_training()return+ if _custom_llama_config(request):+ raise HTTPException(+ status_code = 409,+ detail = "Custom llama.cpp placement cannot be verified beside active training. Stop training before loading this configuration.",+ )+from core.inference.llama_cpp import _diffusion_manual_ngl, _scale_diffusion_required_gbis_gguf = bool(getattr(config, "is_gguf", False))@@ -15822,6 +15838,61 @@ def _guard_chat_load_against_training(raise HTTPException(status_code = 409, detail = detail)+def _custom_llama_config(request) -> bool:+ source = getattr(request, "llama_cpp_config", None)+ return isinstance(source, dict) and source.get("mode") == "custom"+++def _resolve_llama_cpp_config(+ request,+ config = None,+ model_identifier = None,+):+ """Request's own source, else this model row's saved override, else the resident same-model load's."""+ from core.inference.llama_custom_config import parse_config_source+ from utils.openai_auto_switch_settings import resolve_override_for_load++ source = getattr(request, "llama_cpp_config", None)+ identifier = model_identifier or request.model_path+ variant = getattr(request, "gguf_variant", None) or getattr(config, "gguf_variant", None)+ if source is None:+ _, override = resolve_override_for_load(+ identifier, getattr(config, "identifier", None), variant+ )+ source = override.get("llama_cpp_config")+ if source is None:+ resident = get_llama_cpp_backend()+ intent = getattr(resident, "last_load_intent", None)+ if (+ intent is not None+ and _same_loaded_identifier(getattr(intent, "model_identifier", None), identifier)+ and (getattr(intent, "hf_variant", None) or "").casefold() == (variant or "").casefold()+ ):+ source = getattr(intent, "llama_cpp_config", None)+ if source is None:+ return request+ source = parse_config_source(source)+ return request.model_copy(update = {"llama_cpp_config": source.to_wire()})+++async def _preflight_custom_llama_config(request, config):+ """Compile a custom config before any backend or GPU owner can be evicted."""+ if not _custom_llama_config(request):+ return None+ if not config.is_gguf or _classify_diffusion_gguf(config) is True:+ raise HTTPException(+ status_code = 400,+ detail = "Custom llama.cpp configuration needs a GGUF chat model.",+ )+ intent = GgufLoadIntent(+ model_identifier = config.identifier, llama_cpp_config = request.llama_cpp_config+ )+ try:+ return await asyncio.to_thread(get_llama_cpp_backend().prepare_custom_config, intent)+ except ValueError as exc:+ raise HTTPException(status_code = 400, detail = redact_native_paths(str(exc))) from exc++def _resolve_inherited_extra_args(request,config: ModelConfig,@@ -15832,6 +15903,8 @@ def _resolve_inherited_extra_args("""Effective pass-through extras for a GGUF request that omitted the field:the previous same-model load's extras, shadow-stripped, so a settings-Applyreload (which does not round-trip the extras field) keeps them (#5401)."""+ if _custom_llama_config(request):+ return []if getattr(request, "llama_extra_args", None) is not None:return extra_llama_argsif not getattr(config, "is_gguf", False):@@ -17083,17 +17156,27 @@ async def _load_model_impl(gguf_load_stack = ExitStack()token_rejections = gguf_load_stack.enter_context(collecting_hub_token_rejections())try:+ requested_llama_cpp_config = request.llama_cpp_config+# Validate user pass-through args up front so a managed-flag collision- # returns 400 before any model work.- try:- extra_llama_args = validate_extra_args(request.llama_extra_args)- except ValueError as exc:- # Keep the curated validation message (names the flag); just strip paths.- logger.warning("inference.validate_extra_args_failed: %s", exc)- raise HTTPException(- status_code = 400,- detail = redact_native_paths(str(exc)),- )+ # returns 400 before any model work. Custom mode launches without them.+ def _validated_extra_args(req):+ if _custom_llama_config(req):+ return []+ try:+ return validate_extra_args(req.llama_extra_args)+ except ValueError as exc:+ # Keep the curated validation message (names the flag); just strip paths.+ logger.warning("inference.validate_extra_args_failed: %s", exc)+ raise HTTPException(+ status_code = 400,+ detail = redact_native_paths(str(exc)),+ )++ extra_llama_args = _validated_extra_args(request)+ request = _resolve_llama_cpp_config(request)+ if _custom_llama_config(request):+ extra_llama_args = []# Re-narrow []-from-None back to None so the inheritance path below can# tell "caller omitted" from "caller explicit []".extra_llama_args: Optional[list[str]] = (@@ -17277,7 +17360,12 @@ async def _load_model_impl()is_direct_gguf_request = model_identifier.lower().endswith(".gguf")- if llama_backend.is_loaded and (request.gguf_variant or is_direct_gguf_request):+ if (+ llama_backend.is_loaded+ and (request.gguf_variant or is_direct_gguf_request)+ and getattr(request, "llama_cpp_config", None) is None+ and getattr(llama_backend, "requested_llama_cpp_config", None) is None+ ):reused = _reuse_loaded_gguf(_active_gguf_intent(request,@@ -17295,7 +17383,9 @@ async def _load_model_impl(if not llama_backend.holds_no_vram:await asyncio.to_thread(acquire_for_request, CHAT)return reused- if not (request.gguf_variant or is_direct_gguf_request):+ if not (request.gguf_variant or is_direct_gguf_request) and not _custom_llama_config(+ request+ ):_inherit_resident_load_in_4bit(backend, request, model_identifier)if (_same_loaded_identifier(backend.active_model_name, model_identifier)@@ -17413,6 +17503,17 @@ async def _load_model_impl()await asyncio.to_thread(_require_resolved_base_access, config)+ request = _resolve_llama_cpp_config(+ request.model_copy(update = {"llama_cpp_config": requested_llama_cpp_config}),+ config,+ public_model_identifier,+ )+ extra_llama_args = _validated_extra_args(request)+ custom_compiled = await _preflight_custom_llama_config(request, config)+ if custom_compiled is not None:+ _n_parallel = custom_compiled.n_parallel or _n_parallel+ effective_chat_template_override = None+# Resolve inherited extras once before command-dependent preflights.extra_llama_args = _resolve_inherited_extra_args(request,@@ -17652,6 +17753,8 @@ async def _load_model_impl(request.speculative_type,))+ if custom_compiled is not None:+ chat_load_needs_gpu = not custom_compiled.cpu_only# Ahead of the arbiter: acquire_for evicts a resident Images/Video pipeline and the# confirmation below cancels the running generations, both before load_model's own# copy of this check runs. A header-sized read spares them. Fails open into that copy.@@ -17858,13 +17961,18 @@ async def _load_model_impl(# loading; the response reports the backend's actual tensor_parallel# state so the UI toggle reflects the fallback.try:- success = await load_with_tensor_fallback(- _attempt_gguf_load,- requested_tensor = request.tensor_parallel,- extra_args = extra_llama_args,- label = config.identifier,- cancelled = lambda: _gguf_load_cancelled(llama_backend, load_cancel_event),- )+ if custom_compiled is not None:+ success = await _run_gguf_load_attempt(+ llama_backend, load_intent, load_cancel_event+ )+ else:+ success = await load_with_tensor_fallback(+ _attempt_gguf_load,+ requested_tensor = request.tensor_parallel,+ extra_args = extra_llama_args,+ label = config.identifier,+ cancelled = lambda: _gguf_load_cancelled(llama_backend, load_cancel_event),+ )except Exception:# A GGUF load can raise before tearing down the old llama-server (e.g. an# update-in-progress guard fires before _kill_process), leaving the prior@@ -18504,6 +18612,11 @@ async def validate_model(detail = f"Invalid model identifier: {model_log_label}",)+ request = _resolve_llama_cpp_config(+ request, config, _public_model_identifier(request.model_path, model_identifier)+ )+ custom_compiled = await _preflight_custom_llama_config(request, config)+# The caller's own list when it sent one, or the resolver hands back this# fourth argument unchanged and a --ctx-size the load is about to use would# be missing from the estimate that approves it.@@ -18805,6 +18918,9 @@ async def validate_model(return restore_inventory_handles(ValidateModelResponse(valid = True,+ llama_cpp_config_summary = custom_compiled.summary()+ if custom_compiled is not None+ else None,message = " ".join(["Model identifier is valid."] + _hub_access_warnings(token_rejections)),@@ -19502,6 +19618,9 @@ async def estimate_memory(from core.inference.llama_cpp import _args_place_tensors_on_cpufrom core.inference.llama_server_args import _effective_tensor_parallel+ request = _resolve_llama_cpp_config(request)+ if _custom_llama_config(request):+ return EstimateMemoryResponse(available = False, reason = "unsizable")if is_ollama_manifest_ref(request.model_path):# Resolving one writes a .gguf link to disk; that belongs to the load path.return EstimateMemoryResponse(available = False, reason = "unsupported_source")@@ -26305,6 +26424,19 @@ async def delete_openai_container(await client.close()+def _custom_request_defaults(model_id) -> dict:+ backend = get_llama_cpp_backend()+ summary = getattr(backend, "llama_cpp_config_summary", None)+ if not isinstance(summary, dict) or not _same_loaded_identifier(+ getattr(backend, "model_identifier", None), model_id+ ):+ return {}+ defaults = dict(summary.get("request_defaults") or {})+ if "repeat_penalty" in defaults:+ defaults["repetition_penalty"] = defaults.pop("repeat_penalty")+ return defaults++_REASONING_EFFORT_VALUES = {"none", "minimal", "low", "medium", "high", "max", "xhigh"}@@ -26415,7 +26547,9 @@ def _fill_recommended_sampling_openai(payload, model_id) -> None:f: (getattr(payload, f) if f in payload.model_fields_set else None)for f in SAMPLING_FIELD_NAMES}- effective = resolve_effective_sampling(model_id, explicit)+ effective = resolve_effective_sampling(+ model_id, explicit, preset_defaults = _custom_request_defaults(model_id)+ )for field, value in effective.items():setattr(payload, field, value)@@ -26440,7 +26574,9 @@ def _fill_recommended_sampling_completions(body: dict, model_id) -> None:from utils.inference.inference_config import resolve_effective_sampling, SAMPLING_FIELD_NAMESexplicit = {f: body.get(_COMPLETIONS_SAMPLING_BODY_KEY.get(f, f)) for f in SAMPLING_FIELD_NAMES}- effective = resolve_effective_sampling(model_id, explicit, fill_defaults = False)+ effective = resolve_effective_sampling(+ model_id, explicit, fill_defaults = False, preset_defaults = _custom_request_defaults(model_id)+ )for field, value in effective.items():body[_COMPLETIONS_SAMPLING_BODY_KEY.get(field, field)] = value@@ -37434,6 +37570,9 @@ async def anthropic_messages("repetition_penalty": payload.repetition_penalty,"presence_penalty": payload.presence_penalty,},+ preset_defaults = _custom_request_defaults(+ getattr(llama_backend, "model_identifier", None) or model_name+ ),)temperature = _anthropic_sampling["temperature"]top_p = _anthropic_sampling["top_p"]diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.pyindex 8d0cb5306fe..b172fc5c980 100644--- a/studio/backend/routes/settings.py+++ b/studio/backend/routes/settings.py@@ -47,6 +47,7 @@ from core.rag.config import (effective_gguf_repo_for_embedding_model,)from loggers import get_logger+from models.llama_custom_config import LlamaCppConfigFieldsfrom utils.utils import safe_curated_detail, safe_error_detail, log_and_http_errorfrom utils.personalization_settings import (MAX_AVATAR_DATA_URL_BYTES,@@ -917,7 +918,7 @@ MAX_GGUF_VARIANT_KEY_LEN = 4096MAX_GPU_IDS = MAX_GPU_ID + 1-class ModelOverridePayload(BaseModel):+class ModelOverridePayload(LlamaCppConfigFields):"""One model's saved launch config, applied when the API loads that model.Everything past ``model_id`` is optional and omitted means "app default", so a@@ -2303,7 +2304,8 @@ def update_openai_auto_switch_override(_carried_fields = (() if payload.mirrors_server_tuning else _tuning_fields) + (() if payload.mirrors_reasoning_budget else _reasoning_fields)- if _carried_fields and not is_removal:+ _stored_row = None+ if not is_removal and (_carried_fields or payload.llama_cpp_config is None):# The same spellings the extra-args carry-over walks: a cached repo is not an ordinary folded match,# so a save under the repo id would find nothing and retire the alias with its tuning._alias_ids = [payload.model_id]@@ -2320,15 +2322,15 @@ def update_openai_auto_switch_override(# Taken as a unit from the first row that exists, not field by field down the list: a load stops at the# first non-empty row rather than merging, so filling a gap in the winner from a loser would switch# dormant tuning on.- for _alias_id in _alias_ids:- _stored_tuning = get_model_override(_alias_id)- if not _stored_tuning:- continue- for name in _carried_fields:- if _kept_tuning[name] is None:- _kept_tuning[name] = _stored_tuning.get(name)- break+ _stored_row = next((row for row in map(get_model_override, _alias_ids) if row), None)+ if _stored_row:+ for name in _carried_fields:+ if _kept_tuning[name] is None:+ _kept_tuning[name] = _stored_row.get(name)removed_keys: list[str] = []+ kept_custom_config = payload.llama_cpp_config+ if kept_custom_config is None and _stored_row:+ kept_custom_config = _stored_row.get("llama_cpp_config")if payload.remove is True:# An explicit remove wins over any other field. Remove the key a load resolves to, not the literal one sent# (the browser normalizes casing), and every spelling: clearing one of two leaves the survivor as the sole@@ -2409,6 +2411,7 @@ def update_openai_auto_switch_override(set_model_override(target_id,llama_extra_args = extra_args,+ llama_cpp_config = kept_custom_config,keep_empty_extra_args = keep_empty,max_seq_length = max_seq_length,custom_context_length = custom_context_length,diff --git a/studio/backend/tests/test_llama_custom_config.py b/studio/backend/tests/test_llama_custom_config.pynew file mode 100644index 00000000000..2162fa10353--- /dev/null+++ b/studio/backend/tests/test_llama_custom_config.py@@ -0,0 +1,420 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from types import SimpleNamespace++import pytest++from core.inference import llama_cpp as llama+from core.inference.llama_custom_config import (+ CustomConfigError,+ compile_custom_config,+ parse_config_source,+)++_OPTIONS = (+ "--port; -c --ctx-size; -ngl --gpu-layers; -fa --flash-attn; -np --parallel; -m --model; "+ "-mm --mmproj; --temp --temperature; --top-k; --top-p; --warmup --no-warmup; "+ "--mmproj-offload --no-mmproj-offload; --no-mmproj; --mmproj-auto --no-mmproj-auto; --jinja --no-jinja; -dev --device; "+ "-ctk --cache-type-k; --host; --cache-prompt --no-cache-prompt; -mmdev --mmproj-device; "+ "-sp --special; -cb --cont-batching -nocb --no-cont-batching; "+ "-kvo --kv-offload -nkvo --no-kv-offload; "+ "-md --model-draft --spec-draft-model; -hfd --hf-repo-draft --spec-draft-hf; "+ "-devd --device-draft; --kv-unified-per-slot; --reasoning-budget; --reasoning-budget-message"+)+# Like the probe: every spelling of one option maps to that option's help block.+FLAGS = {+ flag: ", ".join(group.split())+ + " (env: LLAMA_ARG_"+ + group.split()[-1].lstrip("-").upper().replace("-", "_")+ + ")"+ for group in _OPTIONS.split("; ")+ for flag in group.split()+}+SWITCHES = {+ "-kvo",+ "--kv-offload",+ "-nkvo",+ "--no-kv-offload",+ "--no-mmproj",+ "--mmproj-auto",+ "--no-mmproj-auto",+ "-sp",+ "--special",+ "-cb",+ "--cont-batching",+ "-nocb",+ "--no-cont-batching",+ "--warmup",+ "--no-warmup",+ "--mmproj-offload",+ "--no-mmproj-offload",+ "--jinja",+ "--no-jinja",+ "--cache-prompt",+ "--no-cache-prompt",+}+++def source(ini, section = None):+ return {"version": 1, "mode": "custom", "ini": ini, "section": section}+++def compile_ini(ini, section = None):+ return compile_custom_config(source(ini, section), FLAGS, SWITCHES)+++def test_preset_grammar_compiles_to_argv_and_defaults():+ compiled = compile_ini(+ "; comment\n[*]\nc = 8192\nngl=-1\n[qwen]\nc=128000 # wins\nfa=on\nwarmup=false\n"+ "mmproj-offload=off\nnp=4\nm=/ignored.gguf\ntemp=1\ntop-k=20",+ "qwen",+ )+ assert compiled.argv == (+ "-c",+ "128000",+ "-ngl",+ "-1",+ "-fa",+ "on",+ "--no-warmup",+ "--no-mmproj-offload",+ "--temp",+ "1",+ "--top-k",+ "20",+ )+ assert compiled.n_parallel == 4+ assert compiled.request_defaults == {"temperature": 1.0, "top_k": 20}+ assert compiled.diagnostics and not compiled.cpu_only+++ "ini,section,message",+ [+ ("[a]\nc=1\n[b]\nc=2", None, "Select which"),+ ("c=1", "missing", "does not exist"),+ ("bogus=1", None, "not an option"),+ ("host=0.0.0.0", None, "managed by Studio"),+ ("np=65", None, "between"),+ ("jinja=false", None, "tool calling"),+ ("warmup=maybe", None, "true or false"),+ ("temp=hot", None, "number"),+ ("not an entry", None, "line 1"),+ ],+)+def test_refusals_name_the_key_not_the_value(ini, section, message):+ with pytest.raises(CustomConfigError, match = message):+ compile_ini(ini, section)++[email protected]("ini", ["c=1024\nctx-size=2048", "warmup=true\nno-warmup=true"])+def test_two_spellings_of_one_option_are_refused(ini):+ with pytest.raises(CustomConfigError, match = "set the same option"):+ compile_ini(ini)+++def test_sampling_aliases_and_router_keys():+ compiled = compile_ini(+ "temperature=0.2\nload-on-startup=true\nstop-timeout=5\ndedup-cache-models=1"+ )+ assert compiled.request_defaults == {"temperature": 0.2}+ assert compiled.argv == ("--temperature", "0.2")++[email protected]("ini", ["top-k=180", "temp=nan", "top-p=inf", "temp=-1"])+def test_defaults_outside_the_chat_schema_are_refused(ini):+ with pytest.raises(CustomConfigError, match = "between"):+ compile_ini(ini)+++ "ini,argv",+ [+ ("special=false", ()),+ ("special=true", ("--special",)),+ ("nocb=false", ("--cont-batching",)),+ ("nocb=true", ("-nocb",)),+ ("cont-batching=off", ("--no-cont-batching",)),+ ("no-warmup=true", ("--no-warmup",)),+ ("nkvo=false", ("--kv-offload",)),+ ("nkvo=true", ("-nkvo",)),+ ("LLAMA_ARG_NO_KV_OFFLOAD=true", ("--kv-offload",)),+ ("LLAMA_ARG_NO_KV_OFFLOAD=false", ("--no-kv-offload",)),+ ],+)+def test_switches_follow_upstream_preset_semantics(ini, argv):+ assert compile_ini(ini).argv == argv+++def test_env_variable_keys_resolve_like_upstream_presets():+ flags = {**FLAGS, "-c": "-c, --ctx-size N (env: LLAMA_ARG_CTX_SIZE)"}+ flags["--ctx-size"] = flags["-c"]+ compiled = compile_custom_config(source("LLAMA_ARG_CTX_SIZE=4096"), flags, SWITCHES)+ assert compiled.argv == ("--ctx-size", "4096")+++def test_unprobed_binary_is_refused():+ with pytest.raises(CustomConfigError, match = "did not report"):+ compile_custom_config(source("c=1"), {}, set())+++ "value",+ [+ {"version": 2, "mode": "custom", "ini": "c=1"},+ {"version": 1, "mode": "custom", "ini": " "},+ {"version": 1, "mode": "custom", "ini": "c=1\x00"},+ {"version": 1, "mode": "custom", "ini": "x" * (64 * 1024 + 1)},+ {"version": 1, "mode": "custom", "ini": "c=1", "section": "a]b"},+ {"version": 1, "mode": "managed", "ini": "c=1"},+ ],+)+def test_wire_validation(value):+ with pytest.raises(CustomConfigError):+ parse_config_source(value)+++ "ini,cpu_only",+ [+ ("dev=none", True),+ ("dev=none\nmmdev=none", True),+ ("dev=none\nmd=d.gguf\ndevd=none", True),+ ("ngl=0", False), # op offload and the projector still use the GPU+ ("dev=none\nmmdev=CUDA0", False),+ ("dev=none\nmd=d.gguf", False),+ ("dev=none\nspec-draft-model=d.gguf", False),+ ("dev=none\nspec-draft-hf=org/d", False),+ ("dev=none\nspec-draft-hf=org/d\ndevice-draft=none", True),+ ],+)+def test_cpu_only_counts_every_companion(ini, cpu_only):+ assert compile_ini(ini).cpu_only is cpu_only+++def test_a_section_overrides_global_through_another_alias():+ compiled = compile_ini("[*]\nc=1024\n[large]\nctx-size=2048", "large")+ assert compiled.argv == ("--ctx-size", "2048")+ assert compiled.option("-c") == "2048"+++def test_unsectioned_keys_are_the_default_section_not_global():+ ini = "c=1000\n[*]\nngl=-1\n[large]\nc=9000"+ assert compile_ini(ini, "large").option("-c") == "9000"+ assert compile_ini("fa=on\n[large]\nc=9000", "large").option("-fa") is None+ assert compile_ini("fa=on\n[large]\nc=9000", "default").option("-fa") == "on"+ assert compile_ini("c=1000\n[*]\nngl=-1").argv == ("-ngl", "-1", "-c", "1000")+ with pytest.raises(CustomConfigError, match = "Select which"):+ compile_ini("fa=on\n[large]\nc=9000")+++def launch(monkeypatch, tmp_path):+ backend = llama.LlamaCppBackend(manages_processes = False)+ model, binary = tmp_path / "selected.gguf", tmp_path / "llama-server"+ model.touch()+ binary.touch()+ caps = {"help_probe_ok": True, "flags": FLAGS, "switch_flags": SWITCHES}+ for name, value in {+ "_find_llama_server_binary": lambda: str(binary),+ "_exec_path_for_launch": lambda p: p,+ "probe_server_capabilities": lambda *a, **k: caps,+ "_binary_revision": lambda p: (str(p), 1),+ "_cuda_sm_gate_error": lambda p: None,+ "_arch_gate_survivors": lambda p: [],+ "_non_chat_gguf_refusal_for_path": lambda *a: None,+ "_gguf_path_is_diffusion": lambda *a: False,+ "_spawn_is_stale": lambda: False,+ "_find_free_port": lambda: 8123,+ "_wait_for_health": lambda **k: True,+ "_detect_audio_type_strict": lambda: None,+ "_llama_server_env_for_binary": lambda p: {+ "PATH": "libs",+ "LLAMA_ARG_CTX_SIZE": "1",+ "LLAMA_API_KEY": "x",+ "KEEP": "1",+ },+ "_query_server_props": lambda: {+ "total_slots": 2,+ "default_generation_settings": {"n_ctx": 28000},+ "modalities": {"vision": False},+ },+ }.items():+ monkeypatch.setattr(backend, name, value)+ monkeypatch.setattr(llama, "_metal_device_is_paravirtual", lambda: False, raising = False)+ monkeypatch.setattr(+ llama.LlamaCppBackend,+ "_read_gguf_metadata",+ lambda self, path: setattr(self, "_context_length", 128000),+ )+ captured = []++ def spawn(cmd, env, **kwargs):+ captured.append((list(cmd), dict(env)))+ backend._process = SimpleNamespace(poll = lambda: None, pid = 1)+ return True++ monkeypatch.setattr(backend, "_start_llama_process", spawn)+ monkeypatch.setattr(+ backend, "_kill_process", lambda *a, **k: setattr(backend, "_process", None)+ )+ intent = llama.GgufLoadIntent(+ model_identifier = "selected",+ gguf_path = str(model),+ n_ctx = 4096,+ n_parallel = 8,+ extra_args = ("--ctx-size", "99999"),+ llama_cpp_config = source("[*]\nnp=2\nc=56000\nngl=-1\nno-warmup=true\ntemp=0.3"),+ )+ return SimpleNamespace(backend = backend, intent = intent, captured = captured)+++def test_custom_launch_runs_only_the_ini(launch):+ assert launch.backend.load_model(launch.intent)+ cmd, env = launch.captured[0]+ assert cmd[cmd.index("--parallel") + 1] == "2"+ assert "99999" not in cmd and "--jinja" in cmd+ assert cmd[cmd.index("-c") + 1] == "56000" and "--no-warmup" in cmd+ assert not any(k.startswith("LLAMA_ARG_") or k == "LLAMA_API_KEY" for k in env)+ assert env["KEEP"] == "1"+ summary = launch.backend.llama_cpp_config_summary+ assert summary["request_defaults"] == {"temperature": 0.3}+ assert launch.backend.extra_args == []+ # The duplicate-load fast path only reuses a probed server.+ assert launch.backend._audio_probed is True+++def test_cpu_only_custom_load_still_masks_unsupported_rocm_cards(launch, monkeypatch):+ from dataclasses import replace++ masked = []+ monkeypatch.setattr(launch.backend, "_arch_gate_survivors", lambda p: [1])+ monkeypatch.setattr(+ launch.backend, "_emit_child_gpu_visibility", lambda env, ids, **k: masked.append(ids)+ )+ cpu = replace(launch.intent, llama_cpp_config = parse_config_source(source("dev=none")))+ assert launch.backend.load_model(cpu)+ assert masked == ["1"]+ # The GPU arbiter leaves image/video work alone for a zero-VRAM resident server.+ assert launch.backend.holds_no_vram+++def test_custom_launch_reports_the_presets_reasoning_budget(launch):+ from dataclasses import replace++ launch.backend._reasoning_budget = launch.backend._requested_reasoning_budget = 512+ launch.backend._reasoning_budget_message = "stale"+ assert launch.backend.load_model(launch.intent)+ assert (launch.backend._reasoning_budget, launch.backend._reasoning_budget_message) == (-1, "")+ ini = parse_config_source(source("reasoning-budget=256\nreasoning-budget-message=done"))+ assert launch.backend.load_model(replace(launch.intent, llama_cpp_config = ini))+ assert launch.backend._requested_reasoning_budget == 256+ assert launch.backend._reasoning_budget_message == "done"+++ "log,ini,total",+ [+ ("", "np=2\nc=56000", 56000), # split cache: 2 slots x 28000+ ("srv init: n_slots = 2, n_ctx_slot = 28000, kv_unified = 'true'", "np=2", 28000),+ ("", "c=56000", 28000), # np auto: llama-server unifies the cache+ # kv-unified-per-slot with an explicit -c: the pinned pool, not slots x per-slot.+ ("kv_unified = 'true'", "np=2\nc=30000\nkv-unified-per-slot=28000", 30000),+ ],+)+def test_published_kv_capacity_follows_the_unified_cache(launch, log, ini, total):+ from dataclasses import replace++ launch.backend._stdout_lines = [log] if log else []+ intent = replace(launch.intent, llama_cpp_config = parse_config_source(source(ini)))+ assert launch.backend.load_model(intent)+ assert launch.backend._kv_cache_context_total == total+++def test_gpu_custom_load_holds_vram(launch):+ assert launch.backend.load_model(launch.intent)+ assert not launch.backend.holds_no_vram+++def test_audio_probe_runs_after_the_load_lock_is_released(launch, monkeypatch):+ held = []+ monkeypatch.setattr(launch.backend, "_detect_audio_type_strict", lambda: "snac")+ monkeypatch.setattr(+ launch.backend,+ "_apply_detected_audio",+ lambda *a: held.append(launch.backend._lock.locked()) or True,+ )+ assert launch.backend.load_model(launch.intent)+ assert held == [False]+++def test_a_downloaded_diffusion_gguf_is_refused_before_launch(launch, monkeypatch):+ monkeypatch.setattr(launch.backend, "_gguf_path_is_diffusion", lambda *a: True)+ with pytest.raises(CustomConfigError, match = "diffusion"):+ launch.backend.load_model(launch.intent)+ assert launch.captured == []++[email protected]("ini", ["no-mmproj=true", "mmproj-auto=false"])+def test_an_ini_that_disables_the_projector_launches_text_only(launch, monkeypatch, tmp_path, ini):+ from dataclasses import replace++ projector = tmp_path / "mmproj.gguf"+ projector.touch()+ monkeypatch.setattr(launch.backend, "_resolve_launch_mmproj_path", lambda **k: str(projector))+ launch.backend._auto_tensor_split_emitted = "3,1"+ intent = replace(+ launch.intent,+ mmproj_path = str(projector),+ llama_cpp_config = parse_config_source(source(ini)),+ )+ assert launch.backend.load_model(intent)+ assert "--mmproj" not in launch.captured[-1][0]+ assert launch.backend._auto_tensor_split_emitted is None++[email protected]("ini,downloads", [("no-mmproj=true", 0), ("c=4096", 1)])+def test_projector_download_follows_the_ini_not_the_vision_toggle(+ launch, monkeypatch, ini, downloads+):+ from dataclasses import replace++ fetched = []+ monkeypatch.setattr(launch.backend, "_download_mmproj", lambda **k: fetched.append(k) or None)+ intent = replace(+ launch.intent,+ is_vision = True,+ disable_vision = True, # an audio-only encoder survives this; its header decides+ hf_repo = "org/model-GGUF",+ llama_cpp_config = parse_config_source(source(ini)),+ )+ assert launch.backend.load_model(intent)+ assert len(fetched) == downloads+++def test_force_reload_never_adopts_a_resident_custom_server(launch):+ from dataclasses import replace++ assert launch.backend.load_model(launch.intent)+ assert launch.backend._custom_runtime_matches(launch.intent)+ assert not launch.backend._custom_runtime_matches(replace(launch.intent, force_reload = True))+++def test_bad_config_never_spawns(launch):+ from dataclasses import replace++ bad = replace(launch.intent, llama_cpp_config = parse_config_source(source("bogus=1")))+ with pytest.raises(CustomConfigError):+ launch.backend.load_model(bad)+ assert launch.captured == []+ assert launch.backend.llama_cpp_config_summary is None+++def test_start_failure_carries_llama_servers_reason(launch, monkeypatch):+ launch.backend._stdout_lines = ["load_model: error: invalid device: ROCm1"]+ monkeypatch.setattr(launch.backend, "_wait_for_health", lambda **k: False)+ monkeypatch.setattr(launch.backend, "_arch_gate_survivors", lambda p: [1])+ monkeypatch.setattr(launch.backend, "_emit_child_gpu_visibility", lambda *a, **k: None)+ with pytest.raises(CustomConfigError, match = "invalid device: ROCm1.*0 to 0"):+ launch.backend.load_model(launch.intent)diff --git a/studio/backend/tests/test_llama_custom_routes.py b/studio/backend/tests/test_llama_custom_routes.pynew file mode 100644index 00000000000..203db0b2e4a--- /dev/null+++ b/studio/backend/tests/test_llama_custom_routes.py@@ -0,0 +1,169 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Custom configuration persistence, preflight and request-default boundaries."""++from types import SimpleNamespace+from pathlib import Path+import sys++import pytest+from fastapi import HTTPException+from pydantic import ValidationError++from models.inference import ChatCompletionRequest, LoadRequest, ValidateModelRequest+from routes import inference as routes+from utils import openai_auto_switch_settings as settings+from utils.inference import inference_config as sampling+from core.inference.llama_custom_config import CustomConfigError++sys.path.insert(0, str(Path(__file__).resolve().parent))+from test_openai_auto_switch import _put, override_store # noqa: E402, F401+++CUSTOM = {"version": 1, "mode": "custom", "ini": "[*]\nnp=1\nc=56000\nfit=off", "section": None}+MANAGED = {"version": 1, "mode": "managed"}+MODEL = "unsloth/example-GGUF:Q4_K_M"+++def test_override_roundtrip_and_managed_reset(override_store):+ _put(MODEL, llama_cpp_config = CUSTOM, llama_extra_args = ["--temp", "0.7"])+ row = settings.get_model_override(MODEL)+ assert row["llama_cpp_config"] == CUSTOM+ assert settings.model_override_load_kwargs(row, is_gguf = True) == {"llama_cpp_config": CUSTOM}+ _put(MODEL, llama_cpp_config = MANAGED)+ row = settings.get_model_override(MODEL)+ assert row["llama_cpp_config"] == MANAGED+ assert row["llama_extra_args"] == ["--temp", "0.7"]+ assert settings.model_override_load_kwargs(row, is_gguf = True)["llama_extra_args"] == [+ "--temp",+ "0.7",+ ]+++def test_older_client_save_preserves_custom_source(override_store):+ _put(MODEL, llama_cpp_config = CUSTOM)+ _put(MODEL, custom_context_length = 8192)+ assert settings.get_model_override(MODEL)["llama_cpp_config"] == CUSTOM+++def test_quant_reset_stops_bare_model_fallback(override_store):+ _put("unsloth/example-GGUF", llama_cpp_config = CUSTOM)+ _put(MODEL, llama_cpp_config = MANAGED)+ _, row = settings.resolve_override_for_load("unsloth/example-GGUF", variant = "Q4_K_M")+ assert row["llama_cpp_config"] == MANAGED++[email protected]("schema", [LoadRequest, ValidateModelRequest])+def test_source_schema_fails_closed(schema):+ with pytest.raises(ValidationError):+ schema(model_path = "model.gguf", llama_cpp_config = {"version": 2, "mode": "custom"})+++def test_corrupt_saved_source_does_not_become_managed():+ with pytest.raises(ValueError):+ settings.normalize_model_override({"llama_cpp_config": {"version": 1, "mode": "custom"}})+++def test_source_inheritance_is_model_scoped_and_reset_is_explicit(monkeypatch):+ monkeypatch.setattr(settings, "resolve_override_for_load", lambda *a, **k: (None, {}))+ resident = SimpleNamespace(+ last_load_intent = SimpleNamespace(+ model_identifier = "/models/Original.gguf", hf_variant = None, llama_cpp_config = CUSTOM+ )+ )+ monkeypatch.setattr(routes, "get_llama_cpp_backend", lambda: resident)+ same = routes._resolve_llama_cpp_config(LoadRequest(model_path = "/models/Original.gguf"))+ other = routes._resolve_llama_cpp_config(LoadRequest(model_path = "/models/Other.gguf"))+ reset = routes._resolve_llama_cpp_config(+ LoadRequest(model_path = "/models/Original.gguf", llama_cpp_config = MANAGED)+ )+ assert same.llama_cpp_config == CUSTOM+ assert other.llama_cpp_config is None+ assert reset.llama_cpp_config == MANAGED+++def test_custom_ignores_legacy_extras():+ request = LoadRequest(+ model_path = "model.gguf", llama_cpp_config = CUSTOM, llama_extra_args = ["--ctx-size", "128000"]+ )+ assert (+ routes._resolve_inherited_extra_args(+ request, SimpleNamespace(is_gguf = True), "model.gguf", request.llama_extra_args+ )+ == []+ )+++def preset_backend(monkeypatch):+ backend = SimpleNamespace(+ model_identifier = "model.gguf",+ llama_cpp_config_summary = {+ "mode": "custom",+ "request_defaults": {"temperature": 0.25, "top_k": 80},+ },+ )+ monkeypatch.setattr(routes, "get_llama_cpp_backend", lambda: backend)+ monkeypatch.setattr(+ sampling, "_recommended_sampling", lambda _: {"temperature": 0.8, "top_k": 20, "top_p": 0.9}+ )+ for field in sampling.SAMPLING_FIELD_NAMES:+ monkeypatch.delenv(sampling._SAMPLING_FIELDS[field][0], raising = False)+ return backend+++def _chat(**fields):+ return ChatCompletionRequest(messages = [{"role": "user", "content": "hi"}], **fields)+++def test_ini_sampling_ranks_between_client_and_recommendation(preset_backend):+ payload = _chat(top_k = 5)+ routes._fill_recommended_sampling_openai(payload, "model.gguf")+ assert (payload.temperature, payload.top_k, payload.top_p) == (0.25, 5, 0.9)+++def test_operator_pin_still_wins(preset_backend, monkeypatch):+ monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", "1.5")+ payload = _chat()+ routes._fill_recommended_sampling_openai(payload, "model.gguf")+ assert payload.temperature == 1.5+++def test_ini_defaults_never_leak_to_another_model(preset_backend):+ payload = _chat()+ routes._fill_recommended_sampling_openai(payload, "other.gguf")+ assert payload.temperature == 0.8+++def test_raw_completions_get_ini_values(preset_backend):+ body = {"prompt": "hi"}+ routes._fill_recommended_sampling_completions(body, "model.gguf")+ assert (body["temperature"], body["top_k"]) == (0.25, 80)+++async def test_preflight_refuses_a_bad_config_before_any_eviction(monkeypatch):+ calls = []++ def prepare(intent):+ calls.append(intent.llama_cpp_config.ini)+ raise CustomConfigError("'bogus' is not an option of the selected llama-server")++ monkeypatch.setattr(+ routes, "get_llama_cpp_backend", lambda: SimpleNamespace(prepare_custom_config = prepare)+ )+ monkeypatch.setattr(routes, "_classify_diffusion_gguf", lambda config: False)+ request = LoadRequest(model_path = "model.gguf", llama_cpp_config = CUSTOM)+ config = SimpleNamespace(is_gguf = True, identifier = "model.gguf")+ with pytest.raises(HTTPException) as caught:+ await routes._preflight_custom_llama_config(request, config)+ assert caught.value.status_code == 400 and "bogus" in caught.value.detail+ assert calls == [CUSTOM["ini"]]+++async def test_preflight_refuses_non_gguf():+ request = LoadRequest(model_path = "org/model", llama_cpp_config = CUSTOM)+ with pytest.raises(HTTPException):+ await routes._preflight_custom_llama_config(request, SimpleNamespace(is_gguf = False))diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.pyindex 4985fa95ebf..ee75f428dfb 100644--- a/studio/backend/tests/test_llama_server_args.py+++ b/studio/backend/tests/test_llama_server_args.py@@ -192,6 +192,7 @@ def test_a_bare_positional_is_rejected():"--path","--api-prefix","--reuse-port",+ "--rpc",# Auth / TLS"--api-key","--api-key-file",@@ -1403,6 +1404,7 @@ def test_every_denied_flag_with_a_twin_in_the_help_is_scrubbed():("LLAMA_ARG_HF_REPO", "--hf-repo"),("LLAMA_ARG_HOST", "--host"),("LLAMA_ARG_PORT", "--port"),+ ("LLAMA_ARG_RPC", "--rpc"),("LLAMA_ARG_N_PARALLEL", "--parallel"),("LLAMA_ARG_SSL_KEY_FILE", "--ssl-key-file"),("LLAMA_ARG_SSL_CERT_FILE", "--ssl-cert-file"),diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.pyindex c91919db76b..4f6f110111b 100644--- a/studio/backend/utils/inference/inference_config.py+++ b/studio/backend/utils/inference/inference_config.py@@ -204,8 +204,9 @@ def resolve_effective_sampling(explicit: Dict[str, Any],*,fill_defaults: bool = True,+ preset_defaults: Optional[Dict[str, Any]] = None,) -> Dict[str, Any]:- """Resolve the effective sampling params for a request. ``explicit`` maps each field in :data:`SAMPLING_FIELD_NAMES` to the client-sent value, or ``None`` when the client omitted it. Precedence, highest first: an operator ``UNSLOTH_SAMPLING_*`` pin, the client's explicit value, the per-model recommendation, then the static schema default. When ``fill_defaults`` is False a field with none of the first three is omitted rather than set to the static default, so a raw proxy body (``/v1/completions``) keeps llama-server's own default for that field."""+ """Resolve the effective sampling params for a request. ``explicit`` maps each field in :data:`SAMPLING_FIELD_NAMES` to the client-sent value, or ``None`` when the client omitted it. Precedence, highest first: an operator ``UNSLOTH_SAMPLING_*`` pin, the client's explicit value, a custom llama.cpp INI's value (``preset_defaults``), the per-model recommendation, then the static schema default. When ``fill_defaults`` is False a field with none of the first four is omitted rather than set to the static default, so a raw proxy body (``/v1/completions``) keeps llama-server's own default for that field."""recommended = _recommended_sampling(model_id or "")effective: Dict[str, Any] = {}for field, (_env, default, _lo, _hi, _int) in _SAMPLING_FIELDS.items():@@ -214,6 +215,8 @@ def resolve_effective_sampling(effective[field] = overrideelif explicit.get(field) is not None:effective[field] = explicit[field]+ elif preset_defaults and field in preset_defaults:+ effective[field] = preset_defaults[field]elif field in recommended:effective[field] = recommended[field]elif fill_defaults:diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.pyindex 576713f8dd7..791c659b0c3 100644--- a/studio/backend/utils/openai_auto_switch_settings.py+++ b/studio/backend/utils/openai_auto_switch_settings.py@@ -427,6 +427,12 @@ def normalize_model_override("""Validate one per-model launch config, dropping anything unusable. Silently drops rather than raising: an override is a convenience mirror of the UI's config, so one stale field (a KV dtype this llama.cpp build lost, a GPU id from another host) must not block persisting the rest or fail the API load that reads it. ``validate_extra_args`` is the caller's job, since it lives in the llama_server_args allow-list module this one must not import. ``keep_empty_extra_args`` keeps an explicit empty list, the difference between "this model has no launch flags" and "nothing is stored for this model": the same thing everywhere except under a fallback, where a quant whose row is gone reads the bare repository row instead and a cleared box would come back holding whatever that legacy row carries."""entry: dict[str, Any] = {}+ if payload.get("llama_cpp_config") is not None:+ from core.inference.llama_custom_config import parse_config_source++ # A broken custom configuration must not silently become a managed load.+ entry["llama_cpp_config"] = parse_config_source(payload["llama_cpp_config"]).to_wire()+extra_args = payload.get("llama_extra_args")if isinstance(extra_args, (list, tuple)) and extra_args:entry["llama_extra_args"] = [str(arg) for arg in extra_args]@@ -576,6 +582,16 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> direturn {}kwargs: dict[str, Any] = {}+ if is_gguf and override.get("llama_cpp_config") is not None:+ from core.inference.llama_custom_config import parse_config_source++ custom = parse_config_source(override["llama_cpp_config"])+ kwargs["llama_cpp_config"] = custom.to_wire()+ if custom.mode == "custom":+ if override.get("disable_vision") is not None:+ kwargs["disable_vision"] = override["disable_vision"]+ return kwargs+max_seq_length = resolve_fit_max_seq_length(override, is_gguf = is_gguf)if max_seq_length is not None:kwargs["max_seq_length"] = max_seq_lengthdiff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex aecec6f0fe9..5d685d84ec5 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -1,6 +1,8 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+// eslint-disable-next-line no-restricted-imports -- The picker barrel imports chat; this payload helper is import-free.+import { llamaCppConfigPayload } from "@/features/model-picker/model-config/llama-cpp-config";import { attachedMediaUnavailableReason } from "../lib/attached-media-gate";import { externalModelLabel } from "../lib/external-model-label";import { mlxRuntimeStateFrom } from "../lib/mlx-runtime-state";@@ -219,9 +221,11 @@ import {type PendingImageEditReference,type RagAutoInject,GPU_LAYERS_AUTO,- loadedGpuMemoryFields,+ managedGpuMemoryFields,reconcilePersistedGpuIds,- resolveLoadedSpeculativeSettings,+ loadedLlamaCppConfigFields,+ managedKvCacheFields,+ managedSpeculativeSettings,resolveSpeculativeSettingsForLoad,persistGpuMemoryModeOnLoad,resolvePreserveThinkingOnLoad,@@ -3497,6 +3501,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{// it disagrees with the launch....serverTuningLoadPayload(config),// Checked with the same arguments the load sends, or a list the backend refuses would pass this gate.+ ...llamaCppConfigPayload(config.llamaCppConfig, { isDiffusion }),...(resolvedExtraArgs !== undefined? { llama_extra_args: resolvedExtraArgs ?? [] }: {}),@@ -3561,6 +3566,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{...serverTuningLoadPayload(config),// Remembered pass-through args: nothing is resident at startup to inherit them from.// Undefined predates the field; a cleared list is an explicit none.+ ...llamaCppConfigPayload(config.llamaCppConfig, { isDiffusion }),...(resolvedExtraArgs !== undefined? { llama_extra_args: resolvedExtraArgs ?? [] }: {}),@@ -3656,8 +3662,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{preserveThinking: resolvePreserveThinkingOnLoad(loadResp),supportsTools: loadResp.supports_tools ?? false,...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false),- kvCacheDtype: loadResp.cache_type_kv ?? null,- loadedKvCacheDtype: loadResp.cache_type_kv ?? null,+ ...managedKvCacheFields(loadResp),...mlxRuntimeStateFrom(loadResp),// Click-time value, not the resolved backend echo (see performLoad).nParallel: committedSlots,@@ -3693,6 +3698,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{...committedServerTuningState(config, loadResp.is_diffusion ?? false),// What this launch is running, for a later rollback: the status applier cannot seed it while// the model-loading lease is held, and a failed switch would restore the wrong args.+ ...loadedLlamaCppConfigFields(loadResp, config.llamaCppConfig),loadedLlamaExtraArgs:loadResp.requested_llama_extra_args !== undefined? (loadResp.requested_llama_extra_args ?? [])@@ -3704,7 +3710,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{// loaded projector, and the next Apply would send it.disableVision: loadResp.disable_vision ?? false,loadedVisionDisabledByUser: loadResp.vision_disabled_by_user ?? false,- ...loadedGpuMemoryFields(loadResp),+ ...managedGpuMemoryFields(loadResp),loadedCustomContextLength: keepCustomCtx,defaultChatTemplate: loadResp.chat_template ?? null,chatTemplateOverride: effectiveChatTemplateOverride,@@ -3715,7 +3721,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{mmprojFallbackReason: loadResp.mmproj_fallback_reason ?? null,loadedIsDiffusion: loadResp.is_diffusion ?? false,activeModelIsLocal: loadResp.is_local_model ?? false,- ...resolveLoadedSpeculativeSettings(loadResp),+ ...managedSpeculativeSettings(loadResp),});} else {useChatRuntimeStore.setState({@@ -3728,8 +3734,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{preserveThinking: resolvePreserveThinkingOnLoad(loadResp),supportsTools: loadResp.supports_tools ?? false,...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false),- kvCacheDtype: loadResp.cache_type_kv ?? null,- loadedKvCacheDtype: loadResp.cache_type_kv ?? null,+ ...managedKvCacheFields(loadResp),...mlxRuntimeStateFrom(loadResp),nParallel: committedSlots,loadedNParallel: committedSlots,@@ -3747,6 +3752,9 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{// Same reason, and the baseline must clear so a rollback to THIS model does not resend a// GGUF's arguments.loadedLlamaExtraArgs: null,+ llamaCppConfig: undefined,+ loadedLlamaCppConfig: null,+ llamaCppConfigSummary: null,tensorParallel: loadResp.tensor_parallel ?? false,loadedTensorParallel: loadResp.tensor_parallel ?? false,loadedDisableVision: loadResp.disable_vision ?? false,@@ -3756,7 +3764,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{loadedVisionDisabledByUser:loadResp.vision_disabled_by_user ?? false,// Non-GGUF response: clears any stale GPU baseline a prior manual-GPU GGUF load left.- ...loadedGpuMemoryFields(loadResp),+ ...managedGpuMemoryFields(loadResp),defaultChatTemplate: loadResp.chat_template ?? null,chatTemplateOverride: effectiveChatTemplateOverride,loadedChatTemplateOverride: effectiveChatTemplateOverride,@@ -3768,7 +3776,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{...loadedContextFields(loadResp),activeNativePathToken: null,activeNativePathExpiresAtMs: null,- ...resolveLoadedSpeculativeSettings(loadResp),+ ...managedSpeculativeSettings(loadResp),loadedIsMultimodal: isMultimodalResponse(loadResp),mmprojFallbackReason: loadResp.mmproj_fallback_reason ?? null,loadedIsDiffusion: loadResp.is_diffusion ?? false,@@ -4071,8 +4079,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{preserveThinking: resolvePreserveThinkingOnLoad(loadResp),supportsTools: loadResp.supports_tools ?? false,...resolveToolsEnabledOnLoad(loadResp.supports_tools ?? false),- kvCacheDtype: loadResp.cache_type_kv ?? null,- loadedKvCacheDtype: loadResp.cache_type_kv ?? null,+ ...managedKvCacheFields(loadResp),...mlxRuntimeStateFrom(loadResp),// The request above omits n_parallel: a staged override would read as applied and be re-sent// by the next Apply.@@ -4096,7 +4103,9 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{disableVision: loadResp.disable_vision ?? false,loadedVisionDisabledByUser:loadResp.vision_disabled_by_user ?? false,- ...loadedGpuMemoryFields(loadResp),+ ...managedGpuMemoryFields(loadResp),+ // The request omits llama_cpp_config, so a saved custom source may be what launched.+ ...loadedLlamaCppConfigFields(loadResp, undefined),// Drives the GPU Memory controls' diffusion gate; set on every load path so the gate cannot read stale.loadedIsDiffusion: loadResp.is_diffusion ?? false,defaultChatTemplate: loadResp.chat_template ?? null,@@ -4104,7 +4113,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{loadedIsMultimodal: isMultimodalResponse(loadResp),mmprojFallbackReason: loadResp.mmproj_fallback_reason ?? null,activeModelIsLocal: loadResp.is_local_model ?? false,- ...resolveLoadedSpeculativeSettings(loadResp),+ ...managedSpeculativeSettings(loadResp),});recordLastLocalModelLoad({id: DEFAULT_CHAT_MODEL_REPO,diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.tsindex ec4d23f1212..a6ced08c1ce 100644--- a/studio/frontend/src/features/chat/api/chat-api.ts+++ b/studio/frontend/src/features/chat/api/chat-api.ts@@ -392,6 +392,9 @@ export async function validateModel(reasoning_budget_message: payload.reasoning_budget_message ?? "",// A --ctx-size or cache override in here changes the estimate, so a preflight that dropped them// would approve a different command from the one that runs.+ ...(payload.llama_cpp_config !== undefined+ ? { llama_cpp_config: payload.llama_cpp_config }+ : {}),...(payload.llama_extra_args !== undefined? // biome-ignore lint/style/useNamingConvention: API schema{ llama_extra_args: payload.llama_extra_args }diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex de0d1f8efe6..ffad8d70c9e 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -1,6 +1,8 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+// eslint-disable-next-line no-restricted-imports -- The picker barrel imports chat; this payload helper is import-free.+import { llamaCppConfigPayload } from "@/features/model-picker/model-config/llama-cpp-config";import {CACHE_MISS_DOWNLOAD_DESCRIPTION,EMPTY_CACHE_MISS_WATCH,@@ -97,7 +99,10 @@ import {import {GPU_LAYERS_AUTO,isLocalModelPath,- loadedGpuMemoryFields,+ loadedLlamaCppConfigFields,+ managedGpuMemoryFields,+ managedKvCacheFields,+ managedSpeculativeSettings,noteLoadedModelReasoningMode,persistGpuMemoryModeOnLoad,pinHoldsLiveEffort,@@ -2095,6 +2100,12 @@ export function useChatModelRuntime() {// them", and the route preserves the stored flags when the field is omitted, so a fallback// would clear flags the user set elsewhere.const loadLlamaExtraArgs = pendingLoadConfig?.llamaExtraArgs;+ const loadLlamaCppConfig =+ pendingLoadConfig?.llamaCppConfig ??+ (currentCheckpoint === modelId &&+ previousVariant === (ggufVariant ?? null)+ ? stateBeforeUnload.llamaCppConfig+ : undefined);let loadNBatch =pendingLoadConfig?.nBatch ?? stateBeforeUnload.nBatch;let loadNUbatch =@@ -2221,6 +2232,9 @@ export function useChatModelRuntime() {// The same list the load below sends: a --ctx-size or cache override changes the memory this// preflight estimates, so omitting it approves a different command and /load then refuses// the target with the real arguments.+ ...llamaCppConfigPayload(loadLlamaCppConfig, {+ isDiffusion: targetIsDiffusion,+ }),...(!targetIsDiffusion && loadLlamaExtraArgs !== undefined? { llama_extra_args: loadLlamaExtraArgs ?? [] }: {}),@@ -2318,10 +2332,11 @@ export function useChatModelRuntime() {// preflight, so a rejected target truncates replies for a model that never loads. Idle,// unload first and free VRAM early.if (!forceCancelActive) {- await unloadModel({ model_path: currentCheckpoint });- // Only a real /unload removes the resident model. The forced path leaves- // it to /load, so cancellation must not treat it as gone.- loadRun.residentModelUnloaded = true;+ if (loadLlamaCppConfig?.mode !== "custom") {+ await unloadModel({ model_path: currentCheckpoint });+ // Custom mode leaves replacement to /load, which preflights before evicting.+ loadRun.residentModelUnloaded = true;+ }}// Set either way: /load can still leave no model resident, and an unneeded rollback hits// already_loaded before the gate.@@ -2495,6 +2510,11 @@ export function useChatModelRuntime() {isGguf && !targetIsDiffusion ? loadReasoningBudgetMessage : "",// Sent only once known, and [] is the explicit "launch with none": the flags are llama-server's,// so neither a transformers load nor a diffusion GGUF carries them.+ ...(isGguf+ ? llamaCppConfigPayload(loadLlamaCppConfig, {+ isDiffusion: targetIsDiffusion,+ })+ : {}),...(isGguf && !targetIsDiffusion && loadLlamaExtraArgs !== undefined? { llama_extra_args: loadLlamaExtraArgs ?? [] }: {}),@@ -2596,11 +2616,7 @@ export function useChatModelRuntime() {}}}- const loadedKv = loadResponse.cache_type_kv ?? null;const loadedTp = loadResponse.tensor_parallel ?? false;- const loadedSpec = normalizeSpeculativeType(- loadResponse.speculative_type,- );const committedSlots =((loadResponse.is_gguf ?? false) && !(loadResponse.is_diffusion ?? false)) ||(loadResponse.is_mlx ?? false)@@ -2686,8 +2702,7 @@ export function useChatModelRuntime() {codeToolsEnabled: stateBeforeUnload.codeToolsEnabled,}: resolveToolsEnabledOnLoad(supportsTools)),- kvCacheDtype: loadedKv,- loadedKvCacheDtype: loadedKv,+ ...managedKvCacheFields(loadResponse),...mlxRuntimeStateFrom(loadResponse),tensorParallel: loadedTp,loadedTensorParallel: loadedTp,@@ -2699,11 +2714,8 @@ export function useChatModelRuntime() {// Set alongside loadedIsMultimodal so the composer can say WHY images are unavailable.loadedVisionDisabledByUser:loadResponse.vision_disabled_by_user ?? false,- ...loadedGpuMemoryFields(loadResponse),- speculativeType: loadedSpec,- loadedSpeculativeType: loadedSpec,- specDraftNMax: loadResponse.spec_draft_n_max ?? null,- loadedSpecDraftNMax: loadResponse.spec_draft_n_max ?? null,+ ...managedGpuMemoryFields(loadResponse),+ ...managedSpeculativeSettings(loadResponse),// Keep the click-time value: the echo is the resolved count, and adopting it would pin a blank// "server default" control.nParallel: committedSlots,@@ -2745,6 +2757,7 @@ export function useChatModelRuntime() {// process's list, so the last thing we knew still holds unless this was a different model. An// explicit empty list is recorded as empty, not null, since omitting the field is what makes /load// inherit. The server's echo comes first, as the only account of what the launch carried.+ ...loadedLlamaCppConfigFields(loadResponse, loadLlamaCppConfig),loadedLlamaExtraArgs:loadResponse.requested_llama_extra_args !== undefined? (loadResponse.requested_llama_extra_args ?? [])@@ -2885,6 +2898,9 @@ export function useChatModelRuntime() {}),// Explicit, unlike the batch pair above: the failed switch left the TARGET resident, so an// omitted field here inherits across models, which the route refuses.+ ...llamaCppConfigPayload(+ rollbackState.loadedLlamaCppConfig ?? undefined,+ ),...(rollbackState.loadedLlamaExtraArgs != null? { llama_extra_args: rollbackState.loadedLlamaExtraArgs }: {}),@@ -2910,6 +2926,12 @@ export function useChatModelRuntime() {rollbackResponse.speculative_type,);useChatRuntimeStore.setState({+ llamaCppConfig:+ rollbackState.loadedLlamaCppConfig ?? undefined,+ loadedLlamaCppConfig: rollbackState.loadedLlamaCppConfig,+ llamaCppConfigSummary:+ rollbackResponse.llama_cpp_config_summary ??+ rollbackState.llamaCppConfigSummary,activeModelIsLocal: rollbackResponse.is_local_model ?? false,activeLoadId: previousActiveLoadId ?? null,activeNativePathToken: previousActiveNativePathToken ?? null,@@ -2956,17 +2978,29 @@ export function useChatModelRuntime() {rollbackState.loadedCtxCheckpoints ?? null,cacheRam: previousServerTuning.cacheRam ?? null,loadedCacheRam: rollbackState.loadedCacheRam ?? null,- loadedSpeculativeType: rollbackSpeculativeType,- loadedSpecDraftNMax:- rollbackResponse.spec_draft_n_max ?? null,- loadedKvCacheDtype: rollbackResponse.cache_type_kv ?? null,+ // A custom echo carries the INI's tuning, not the managed baselines (as managedKvCacheFields).+ ...(rollbackResponse.requested_llama_cpp_config?.mode ===+ "custom"+ ? {+ loadedSpeculativeType:+ rollbackState.loadedSpeculativeType,+ loadedSpecDraftNMax: rollbackState.loadedSpecDraftNMax,+ loadedKvCacheDtype: rollbackState.loadedKvCacheDtype,+ }+ : {+ loadedSpeculativeType: rollbackSpeculativeType,+ loadedSpecDraftNMax:+ rollbackResponse.spec_draft_n_max ?? null,+ loadedKvCacheDtype:+ rollbackResponse.cache_type_kv ?? null,+ }),...mlxRuntimeStateFrom(rollbackResponse),// After the spread, which seeds the control from the echo; the control keeps its intent, like// nParallel above.mlxKvQuant: previousMlxKvQuant,loadedChatTemplateOverride:rollbackState.loadedChatTemplateOverride,- ...loadedGpuMemoryFields(rollbackResponse),+ ...managedGpuMemoryFields(rollbackResponse),tensorParallel: rollbackResponse.tensor_parallel ?? false,loadedTensorParallel:rollbackResponse.tensor_parallel ?? false,diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.tsindex e27d5e7d6d4..cc90062920e 100644--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts@@ -469,7 +469,9 @@ export function applyActiveModelStatusToStore(specDrafterKind: status.spec_drafter_kind ?? null,// Controls follow the server while clean; loaded baselines always describe// the settled resident, including same-model reloads from another client.+ // Never from a custom load: its echo is the INI's tuning....(seedLoadParams &&+ status.requested_llama_cpp_config?.mode !== "custom" &&(status.speculative_type !== undefined ||prevState.loadedSpeculativeType === null ||hydratingExistingModel) && {@@ -481,6 +483,7 @@ export function applyActiveModelStatusToStore(}),}),...(seedLoadParams &&+ status.requested_llama_cpp_config?.mode !== "custom" &&status.spec_draft_n_max !== undefined && {loadedSpecDraftNMax: status.spec_draft_n_max ?? null,...((hydratingExistingModel ||@@ -489,6 +492,7 @@ export function applyActiveModelStatusToStore(}),}),...(seedLoadParams &&+ status.requested_llama_cpp_config?.mode !== "custom" &&status.cache_type_kv !== undefined && {loadedKvCacheDtype: status.cache_type_kv,...((prevState.loadedKvCacheDtype === null ||@@ -622,6 +626,11 @@ export function applyActiveModelStatusToStore(// status, not just the first: another client can reload the SAME model with different// arguments, and a pinned baseline would resurrect arguments that are not running.// seedLoadParams still guards it, so a mid-switch poll cannot overwrite performLoad.+ ...(status.requested_llama_cpp_config !== undefined && seedLoadParams ? {+ llamaCppConfig: status.requested_llama_cpp_config ?? undefined,+ loadedLlamaCppConfig: status.requested_llama_cpp_config ?? null,+ llamaCppConfigSummary: status.llama_cpp_config_summary ?? null,+ } : {}),...resolveLlamaExtraArgsSeed({incoming: status.requested_llama_extra_args,isGguf: status.is_gguf ?? true,@@ -673,8 +682,9 @@ export function applyActiveModelStatusToStore(}),// Re-seed on first hydration, model/variant changes, or a same-model backend// placement change. placementAndContextFields preserves dirty local edits in the last- // case while advancing their loaded baselines.+ // case while advancing their loaded baselines. Never from a custom load (see above)....(seedLoadParams &&+ status.requested_llama_cpp_config?.mode !== "custom" &&(prevState.loadedGpuMemoryMode === null ||hydratingExistingModel ||placementOrContextChanged) &&diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.tsindex b34e0b869db..2af39044f6f 100644--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts@@ -36,6 +36,7 @@ type ResidentRuntime = Pick<| "disable_vision"| "chat_template_override"| "requested_llama_extra_args"+ | "requested_llama_cpp_config"| "gpu_memory_mode"| "gpu_layers"| "n_cpu_moe"@@ -574,6 +575,10 @@ export function residentRuntimeMatchesConfig(if (!config) {return true;}+ if (config.llamaCppConfig?.mode === "custom" || status.requested_llama_cpp_config?.mode === "custom") {+ // Only the server can tell whether a custom config is unchanged (binary, resources).+ return false;+ }const placementPreserved =// A virtualised Metal device pins every GGUF request to the CPU before either comparator runs, so// placement cannot tell two requests apart there.diff --git a/studio/frontend/src/features/chat/presets/preset-policy.ts b/studio/frontend/src/features/chat/presets/preset-policy.tsindex e9af04b1e19..b6537724e77 100644--- a/studio/frontend/src/features/chat/presets/preset-policy.ts+++ b/studio/frontend/src/features/chat/presets/preset-policy.ts@@ -274,6 +274,7 @@ interface BackendInferenceDefaults {top_k?: number;min_p?: number;presence_penalty?: number;+ repetition_penalty?: number;trust_remote_code?: boolean;}@@ -330,6 +331,9 @@ export function mergeBackendRecommendedInference({presencePenalty:toFiniteNumber(inference?.presence_penalty) ??defaultInferenceParams.presencePenalty,+ // Only a custom llama.cpp config reports one; otherwise the slider keeps its value.+ repetitionPenalty:+ toFiniteNumber(inference?.repetition_penalty) ?? next.repetitionPenalty,};}diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsxindex b919c159185..b15ec7ffb43 100644--- a/studio/frontend/src/features/chat/shared-composer.tsx+++ b/studio/frontend/src/features/chat/shared-composer.tsx@@ -1,6 +1,8 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+// eslint-disable-next-line no-restricted-imports -- Keep the import-free payload helper independent of the picker UI.+import { llamaCppConfigPayload } from "@/features/model-picker/model-config/llama-cpp-config";import { useChatArtifactsStore } from "./artifacts/store";import { mlxRuntimeStateFrom } from "./lib/mlx-runtime-state";import { offloadCountsFrom, offloadWarning } from "./lib/partial-offload";@@ -203,10 +205,12 @@ import {shouldPinDiffusionPlacement,} from "./lib/gpu-placement";import {- loadedGpuMemoryFields,+ loadedLlamaCppConfigFields,+ managedGpuMemoryFields,type ReasoningEffort,reconcilePersistedGpuIds,- resolveLoadedSpeculativeSettings,+ managedKvCacheFields,+ managedSpeculativeSettings,resolvePreserveThinkingOnLoad,persistGpuMemoryModeOnLoad,resolveSpeculativeSettingsForLoad,@@ -1728,6 +1732,9 @@ export function SharedComposer({: ownConfig.reasoningBudgetMessage,// Only when this panel has read the stored value: omitted, the load inherits it, which is what// keeps CLI-set flags working.+ ...llamaCppConfigPayload(ownConfig.llamaCppConfig, {+ isDiffusion: resolvedIsDiffusion === true,+ }),...(ownConfig.llamaExtraArgs !== undefined? // biome-ignore lint/style/useNamingConvention: API schema{ llama_extra_args: ownConfig.llamaExtraArgs ?? [] }@@ -1827,6 +1834,9 @@ export function SharedComposer({n_cpu_moe: effectiveNCpuMoe,tensor_split: compareLoadKnobs.splitRatio ?? undefined,gpu_ids: effectiveSelectedGpuIds ?? undefined,+ ...llamaCppConfigPayload(ownConfig.llamaCppConfig, {+ isDiffusion: resolvedIsDiffusion === true,+ }),...(ownConfig.llamaExtraArgs !== undefined? // biome-ignore lint/style/useNamingConvention: API schema{ llama_extra_args: ownConfig.llamaExtraArgs ?? [] }@@ -1923,8 +1933,7 @@ export function SharedComposer({supportsPreserveThinking: resp.supports_preserve_thinking ?? false,preserveThinking: resolvePreserveThinkingOnLoad(resp),supportsTools: resp.supports_tools ?? false,- kvCacheDtype: resp.cache_type_kv ?? null,- loadedKvCacheDtype: resp.cache_type_kv ?? null,+ ...managedKvCacheFields(resp),...mlxRuntimeStateFrom(resp),// Click-time value, not the resolved echo (see the single-model load).nParallel: committedSlots,@@ -1962,6 +1971,7 @@ export function SharedComposer({: clearedServerTuningState()),// What this pane's launch is running, for a later rollback: the status applier is held off for// the whole load, so a switch straight after would snapshot the other model's list.+ ...loadedLlamaCppConfigFields(resp, ownConfig.llamaCppConfig),loadedLlamaExtraArgs:resp.requested_llama_extra_args !== undefined? (resp.requested_llama_extra_args ?? [])@@ -1980,7 +1990,7 @@ export function SharedComposer({loadedCustomContextLength: keepCustomCtx,// Adopt the load response's GPU-memory fields (mode/layers/MoE/split/pick plus loaded baselines)// so the GPU controls round-trip. The context group and native-path token/expiry clear below.- ...loadedGpuMemoryFields(resp),+ ...managedGpuMemoryFields(resp),// Drives the GPU Memory controls' diffusion gate; set alongside the GPU fields on every load path// so the gate cannot read stale.loadedIsDiffusion: resp.is_diffusion ?? false,@@ -1997,7 +2007,7 @@ export function SharedComposer({// lease. Clear any prior picked file's token/expiry so the reload path never sends a stale one.activeNativePathToken: null,activeNativePathExpiresAtMs: null,- ...resolveLoadedSpeculativeSettings(resp),+ ...managedSpeculativeSettings(resp),});if (!targetIsGguf) {// Non-GGUF panes carry their context in params.maxSeqLength.diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.tsindex d1bb7849c2a..b4661f07cf8 100644--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts@@ -1,6 +1,11 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+// eslint-disable-next-line no-restricted-imports -- The picker barrel imports this store; this leaf is import-free.+import type {+ LlamaCppConfig,+ LlamaCppConfigSummary,+} from "@/features/model-picker/model-config/llama-cpp-config";import { authFetch } from "@/features/auth";import type { ImageDisclosure } from "../api/mcp-image";import {@@ -2099,6 +2104,55 @@ export function requestedGpuIdsFromResponse(resp: {: (resp.gpu_ids ?? null);}+type LlamaCppConfigEcho = { requested_llama_cpp_config?: { mode: string } | null };++function isCustomLlamaLoad(resp: LlamaCppConfigEcho): boolean {+ return resp.requested_llama_cpp_config?.mode === "custom";+}++/** The config a load ran with: the server's echo, else what was sent. */+export function loadedLlamaCppConfigFields(+ resp: {+ requested_llama_cpp_config?: LlamaCppConfig | null;+ llama_cpp_config_summary?: LlamaCppConfigSummary | null;+ },+ sent: LlamaCppConfig | undefined,+) {+ const config = resp.requested_llama_cpp_config ?? sent;+ return {+ llamaCppConfig: config,+ loadedLlamaCppConfig: config ?? null,+ llamaCppConfigSummary: resp.llama_cpp_config_summary ?? null,+ };+}++// A custom load echoes its INI's tuning; adopting it would carry that into the next managed load.+export function managedKvCacheFields(+ resp: { cache_type_kv?: string | null } & LlamaCppConfigEcho,+) {+ if (isCustomLlamaLoad(resp)) return {};+ const kv = resp.cache_type_kv ?? null;+ return { kvCacheDtype: kv, loadedKvCacheDtype: kv };+}++export function managedSpeculativeSettings(+ resp: Parameters<typeof resolveLoadedSpeculativeSettings>[0] & LlamaCppConfigEcho,+) {+ return isCustomLlamaLoad(resp) ? {} : resolveLoadedSpeculativeSettings(resp);+}++export function managedGpuMemoryFields(+ resp: Parameters<typeof loadedGpuMemoryFields>[0] & LlamaCppConfigEcho,+) {+ if (isCustomLlamaLoad(resp)) {+ return {+ ggufLayerCount: resp.n_layers ?? null,+ moeLayerCount: resp.n_moe_layers ?? null,+ };+ }+ return loadedGpuMemoryFields(resp);+}+// Store fields derived from a load/status response's GPU-memory settings, shared by every// load path so the manual-knob round-trip cannot drift.export function loadedGpuMemoryFields(resp: {@@ -2442,6 +2496,9 @@ type ChatRuntimeStore = {/** Pass-through args the resident model is running, as far as this client knows. A rollback* resends them: by then the target load has replaced the backend's inheritance source. */loadedLlamaExtraArgs: string[] | null;+ llamaCppConfig?: LlamaCppConfig;+ loadedLlamaCppConfig: LlamaCppConfig | null;+ llamaCppConfigSummary: LlamaCppConfigSummary | null;/** user --ubatch-size override for gguf loads (null = llama.cpp default 512) */nUbatch: number | null;loadedNUbatch: number | null;@@ -4211,6 +4268,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({nBatch: null,loadedNBatch: null,loadedLlamaExtraArgs: null,+ loadedLlamaCppConfig: null,+ llamaCppConfigSummary: null,nUbatch: null,loadedNUbatch: null,specDraftCacheDtype: null,@@ -5159,6 +5218,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({nBatch: null,loadedNBatch: null,loadedLlamaExtraArgs: null,+ loadedLlamaCppConfig: null,+ llamaCppConfigSummary: null,nUbatch: null,loadedNUbatch: null,specDraftCacheDtype: null,diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 119bd98a5aa..8c8344e04f4 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -1,6 +1,11 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+// eslint-disable-next-line no-restricted-imports -- Share the config wire types without expanding the picker UI barrel.+import type {+ LlamaCppConfig,+ LlamaCppConfigSummary,+} from "@/features/model-picker/model-config/llama-cpp-config";import type { TransformersUpgradeInfo } from "@/features/transformers-upgrade";export type CpuFallbackReason = "vulkan_startup_crash";@@ -101,6 +106,7 @@ export interface LoadModelRequest {* flag. Omit/null inherits the stored per-model value; [] launches with none. GGUF only. */// biome-ignore lint/style/useNamingConvention: API schemallama_extra_args?: string[] | null;+ llama_cpp_config?: LlamaCppConfig;/** Split the model across GPUs by tensor (--split-mode tensor) instead of by layer for GGUF models.* Multi-GPU only. */tensor_parallel?: boolean | null;@@ -127,6 +133,8 @@ export interface LoadModelRequest {}export interface ValidateModelResponse {+ requested_llama_cpp_config?: LlamaCppConfig | null;+ llama_cpp_config_summary?: LlamaCppConfigSummary | null;valid: boolean;message: string;identifier?: string | null;@@ -325,6 +333,8 @@ export interface LoadModelResponse {requested_cache_ram?: number | null;/** Pass-through llama-server arguments the running load was invoked with. */requested_llama_extra_args?: string[] | null;+ requested_llama_cpp_config?: LlamaCppConfig | null;+ llama_cpp_config_summary?: LlamaCppConfigSummary | null;}export interface UnloadModelRequest {@@ -441,6 +451,8 @@ export interface InferenceStatusResponse {requested_cache_ram?: number | null;/** Pass-through llama-server arguments the running load was invoked with. */requested_llama_extra_args?: string[] | null;+ requested_llama_cpp_config?: LlamaCppConfig | null;+ llama_cpp_config_summary?: LlamaCppConfigSummary | null;n_layers?: number | null;/** Model's MoE expert-layer count (the n_cpu_moe ceiling); 0 if not MoE. */n_moe_layers?: number;diff --git a/studio/frontend/src/features/model-picker/api/model-overrides.ts b/studio/frontend/src/features/model-picker/api/model-overrides.tsindex 91b6273c01f..41c3d46bdcd 100644--- a/studio/frontend/src/features/model-picker/api/model-overrides.ts+++ b/studio/frontend/src/features/model-picker/api/model-overrides.ts@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import type { LlamaCppConfig } from "../model-config/llama-cpp-config";// Server-side mirror of the per-model config. per-model-config.ts lives in browser// localStorage, so an API auto-switch load came up with none of the user's settings.// routes/inference.py reads this map and rebuilds the picker's LoadRequest.@@ -27,6 +28,7 @@ const OVERRIDES_URL = "/api/settings/openai-auto-switch/overrides";export interface ApiModelOverride {// biome-ignore lint/style/useNamingConvention: API schemallama_extra_args?: string[];+ llama_cpp_config?: LlamaCppConfig;// biome-ignore lint/style/useNamingConvention: API schemamax_seq_length?: number;// biome-ignore lint/style/useNamingConvention: API schema@@ -344,6 +346,7 @@ export function fromApiOverride(chatTemplateOverride:override.chat_template_override ?? local.chatTemplateOverride,llamaExtraArgs: extraArgs,+ llamaCppConfig: override.llama_cpp_config ?? local.llamaCppConfig,gpuMemoryMode: override.gpu_memory_mode ?? local.gpuMemoryMode,gpuLayers: override.gpu_layers ?? local.gpuLayers,nCpuMoe: override.n_cpu_moe ?? local.nCpuMoe,@@ -431,6 +434,9 @@ export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {// The one field where absent does NOT mean "app default": the route preserves// llama_extra_args it is not sent, which kept CLI-set flags alive while this panel had no// control. So `undefined` stays omitted and a cleared box sends an explicit empty list.+ if (config.llamaCppConfig !== undefined) {+ payload.llama_cpp_config = config.llamaCppConfig;+ }if (config.llamaExtraArgs !== undefined) {payload.llama_extra_args = config.llamaExtraArgs ?? [];}diff --git a/studio/frontend/src/features/model-picker/components/custom-llama-config-editor.tsx b/studio/frontend/src/features/model-picker/components/custom-llama-config-editor.tsxnew file mode 100644index 00000000000..21e9769a163--- /dev/null+++ b/studio/frontend/src/features/model-picker/components/custom-llama-config-editor.tsx@@ -0,0 +1,136 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { useEffect, useId } from "react";+import { Button } from "@/components/ui/button";+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from "@/components/ui/select";+import {+ customConfigSections,+ MAX_LLAMA_CPP_CONFIG_BYTES,+ toggledLlamaCppConfig,+ type LlamaCppConfig,+} from "../model-config/llama-cpp-config";++// Module scope: switching back to custom restores the last INI even after a remount.+const lastCustomSource = new Map<+ string,+ { ini: string; section: string | null }+>();++export function CustomLlamaConfigEditor({+ value,+ onChange,+ sourceKey,+ onLoadableChange,+}: {+ value: LlamaCppConfig | undefined;+ onChange: (value: LlamaCppConfig) => void;+ sourceKey: string;+ onLoadableChange: (value: boolean) => void;+}) {+ const id = useId();+ const active = value?.mode === "custom";+ const ini = active ? value.ini : "";+ const section = active ? value.section : null;+ const sections = customConfigSections(ini);+ const oversized =+ new TextEncoder().encode(ini).length > MAX_LLAMA_CPP_CONFIG_BYTES;+ const blank = active && ini.trim().length === 0;+ const needsSection =+ active && section === null && sections.some((name) => name !== "default");+ useEffect(() => {+ if (active) lastCustomSource.set(sourceKey, { ini, section });+ }, [active, ini, section, sourceKey]);+ useEffect(() => {+ onLoadableChange(!active || !(oversized || blank || needsSection));+ }, [active, oversized, blank, needsSection, onLoadableChange]);+ return (+ <div className="space-y-3">+ <div className="flex flex-wrap items-center justify-between gap-2">+ <span className="text-ui-12 font-medium">+ Custom llama.cpp configuration+ </span>+ <Button+ type="button"+ variant="outline"+ size="sm"+ onClick={() =>+ onChange(+ toggledLlamaCppConfig(+ value,+ lastCustomSource.get(sourceKey) ?? null,+ ),+ )+ }+ >+ {active ? "Use Studio settings" : "Use custom configuration"}+ </Button>+ </div>+ {active && (+ <>+ <p className="text-ui-11 text-muted-foreground">+ llama-server runs with exactly this INI (llama.cpp preset format);+ Studio adds only the model, host and port. Its sampling values+ become this model's chat defaults.+ </p>+ <textarea+ id={id}+ aria-label="llama.cpp INI configuration"+ spellCheck={false}+ className="min-h-40 w-full resize-y rounded-md border border-input bg-transparent px-3 py-2 font-mono text-ui-11"+ value={ini}+ onChange={(event) => {+ const nextIni = event.target.value;+ onChange({+ ...value,+ ini: nextIni,+ section: customConfigSections(nextIni).includes(+ value.section ?? "",+ )+ ? value.section+ : null,+ });+ }}+ />+ {sections.some((name) => name !== "default") && (+ <div className="space-y-1.5">+ <label className="text-ui-11" htmlFor={`${id}-section`}>+ Section+ </label>+ <Select+ value={section ?? ""}+ onValueChange={(next) => onChange({ ...value, section: next })}+ >+ <SelectTrigger id={`${id}-section`}>+ <SelectValue placeholder="Choose a section" />+ </SelectTrigger>+ <SelectContent>+ {sections.map((name) => (+ <SelectItem key={name} value={name}>+ {name}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </div>+ )}+ {(oversized || blank || needsSection) && (+ <p role="alert" className="text-ui-11 text-destructive">+ {oversized+ ? "Configuration must fit within 64 KiB."+ : blank+ ? "Configuration is empty."+ : "Choose which section to load."}+ </p>+ )}+ </>+ )}+ </div>+ );+}diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex fb6942eaf75..dc8eba47234 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -1,6 +1,11 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import { CustomLlamaConfigEditor } from "./custom-llama-config-editor";+import {+ MANAGED_LLAMA_CPP_CONFIG,+ normalizeLlamaCppConfig,+} from "../model-config/llama-cpp-config";import { Button } from "@/components/ui/button";import { Checkbox } from "@/components/ui/checkbox";import { InfoHint } from "@/components/ui/info-hint";@@ -338,6 +343,7 @@ function withoutUnsupportedDiffusionSettings(config.nBatch == null &&config.nUbatch == null &&(config.llamaExtraArgs == null || config.llamaExtraArgs.length === 0) &&+ config.llamaCppConfig?.mode !== "custom" &&!hasUnsupportedGpuPick) {return config;@@ -358,6 +364,10 @@ function withoutUnsupportedDiffusionSettings(// them as though it had, so a box filled before classification flipped would leave the// model running without what it says.llamaExtraArgs: null,+ // The diffusion runner has no llama-server; explicit managed, since omitted inherits the stored custom.+ ...(config.llamaCppConfig?.mode === "custom"+ ? { llamaCppConfig: MANAGED_LLAMA_CPP_CONFIG }+ : {}),...(hasUnsupportedGpuPick? {selectedGpuIds: undefined,@@ -1310,6 +1320,33 @@ function LoadModeRow({);}+// Outside the managed lock: custom launches still honour disable_vision.+function VisionRow({+ config,+ update,+}: {+ config: PerModelConfig;+ update: (patch: Partial<PerModelConfig>) => void;+}) {+ return (+ <div className={ROW_CLASS}>+ <div className="flex min-w-0 items-center gap-1.5">+ <span className={LABEL_CLASS}>Vision</span>+ <InfoHint>+ Loads the vision projector so the model can read images. Turning+ it off frees that VRAM for more layers on the GPU. Text generation+ is unaffected either way.+ </InfoHint>+ </div>+ <Switch+ className="panel-switch shrink-0"+ checked={!config.disableVision}+ onCheckedChange={(checked) => update({ disableVision: !checked })}+ />+ </div>+ );+}+function GgufAdvancedSettings({config,update,@@ -1325,6 +1362,7 @@ function GgufAdvancedSettings({moeLayersInputRef,onExtraArgsLoadableChange,draftKey,+ hideVision = false,}: {config: PerModelConfig;update: (patch: Partial<PerModelConfig>) => void;@@ -1341,6 +1379,7 @@ function GgufAdvancedSettings({/** Which stored entries the extra-arguments row reads, most specific first. */onExtraArgsLoadableChange: (loadable: boolean) => void;draftKey: string;+ hideVision?: boolean;}) {const batchAdviceId = useId();const ubatchAdviceId = useId();@@ -1632,21 +1671,7 @@ function GgufAdvancedSettings({{/* withoutUnsupportedDiffusionSettings forces disableVision back to false on a diffusion modeland the runner never reads it, so the switch would flip back under the pointer. */}{!isDiffusion && (- <div className={ROW_CLASS}>- <div className="flex min-w-0 items-center gap-1.5">- <span className={LABEL_CLASS}>Vision</span>- <InfoHint>- Loads the vision projector so the model can read images. Turning- it off frees that VRAM for more layers on the GPU. Text generation- is unaffected either way.- </InfoHint>- </div>- <Switch- className="panel-switch shrink-0"- checked={!config.disableVision}- onCheckedChange={(checked) => update({ disableVision: !checked })}- />- </div>+ hideVision ? null : <VisionRow config={config} update={update} />)}{!isDiffusion && (@@ -2160,6 +2185,7 @@ export function ModelConfigPage({// refuse. Held by the panel rather than the row, since the row unmounts whenever Advanced// settings collapse while its tokens stay in the config.const [extraArgsLoadable, setExtraArgsLoadable] = useState(true);+ const [customConfigLoadable, setCustomConfigLoadable] = useState(true);// True until the server-row read below settles: a load started before it lands sends none of the// row's settings, and with Remember unchecked it forgets the row it never read.const [extraArgsHydrating, setExtraArgsHydrating] = useState(@@ -2815,7 +2841,11 @@ export function ModelConfigPage({// stands down, since its runner allocates on a different plan. The tri-state is read as a// tri-state, not through resolvedIsDiffusion: a GGUF still being classified may be// DiffusionGemma, and guessing paints a footprint from the wrong plan that never clears.+ // A custom INI owns llama.cpp tuning: the managed rows lock and Studio cannot price it.+ const customActive =+ config.llamaCppConfig?.mode === "custom" && !resolvedIsDiffusion;const memoryEstimateRequest =+ !customActive &&shouldRequestMemoryEstimate({isGguf: Boolean(target.isGguf),isAppleUnifiedMemory,@@ -3146,6 +3176,14 @@ export function ModelConfigPage({};const persistConfig = (next: PerModelConfig) => {+ // Normalizing drops a blank or oversized custom config, saving managed over a good one.+ if (+ remember &&+ next.llamaCppConfig !== undefined &&+ normalizeLlamaCppConfig(next.llamaCppConfig) === undefined+ ) {+ return { saved: false, defaultConfig: false };+ }// Judge what storage keeps: savePerModelConfig normalizes first, so the raw object over-reports.const normalized = normalizePerModelConfig(next);const evicted: { modelId: string; ggufVariant: string | null }[] = [];@@ -3343,7 +3381,11 @@ export function ModelConfigPage({remember={remember}hasSavedSettings={savedRemember}/>- <div className="space-y-5">+ <fieldset+ disabled={customActive}+ inert={customActive ? true : undefined}+ className={`min-w-0 space-y-5 ${customActive ? "opacity-50" : ""}`}+ >{memoryEstimateRequest != null && (<MemoryEstimateRowestimate={memoryEstimate.estimate}@@ -3467,6 +3509,7 @@ export function ModelConfigPage({layerCount={stagedDims?.layerCount ?? null}moeLayerCount={stagedDims?.moeLayerCount ?? null}isDiffusion={resolvedIsDiffusion}+ hideVision={customActive}gpuDevices={gpuDevices}gpuLayersInputRef={gpuLayersInputRef}moeLayersInputRef={moeLayersInputRef}@@ -3509,7 +3552,21 @@ export function ModelConfigPage({)}</>)}- </div>+ </fieldset>++ {target.isGguf &&+ !resolvedIsDiffusion &&+ (showAdvanced || customActive) && (+ <div className="mt-5 space-y-5">+ {customActive && <VisionRow config={config} update={update} />}+ <CustomLlamaConfigEditor+ value={config.llamaCppConfig}+ onChange={(llamaCppConfig) => update({ llamaCppConfig })}+ sourceKey={draftKey}+ onLoadableChange={setCustomConfigLoadable}+ />+ </div>+ )}{/* Stacked in both variants: a row that wraps on demand reflows when the same click thatcommits a draft mounts Save settings, moving Load out from under the cursor. */}@@ -3542,8 +3599,10 @@ export function ModelConfigPage({sharedVariantUnresolved ||stagedMetadataPending ||budgetSettling ||- (!extraArgsLoadable && !sharedExtraArgsCleared) ||- sharedExtraArgsRefused ||+ (customActive && !customConfigLoadable) ||+ (!customActive &&+ ((!extraArgsLoadable && !sharedExtraArgsCleared) ||+ sharedExtraArgsRefused)) ||extraArgsHydrating ||(isActiveModel &&atBaseline &&@@ -3569,7 +3628,8 @@ export function ModelConfigPage({stagedMetadataPending ||budgetSettling ||(remember &&- ((!extraArgsLoadable && !sharedExtraArgsCleared) ||+ ((customActive && !customConfigLoadable) ||+ (!extraArgsLoadable && !sharedExtraArgsCleared) ||sharedExtraArgsRefused ||extraArgsHydrating))}@@ -3596,6 +3656,7 @@ export function ModelConfigPage({// running process's arguments, so a reload after Reset kept the flags the box says are gone....DEFAULT_PER_MODEL_CONFIG,llamaExtraArgs: null,+ llamaCppConfig: MANAGED_LLAMA_CPP_CONFIG,});}}>diff --git a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.tsindex 5206a36ac1b..583dff7cfe7 100644--- a/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts+++ b/studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts@@ -51,6 +51,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {const loadMode = useChatRuntimeStore((s) => s.loadMode);const ctxCheckpoints = useChatRuntimeStore((s) => s.ctxCheckpoints);const cacheRam = useChatRuntimeStore((s) => s.cacheRam);+ const llamaCppConfig = useChatRuntimeStore((s) => s.loadedLlamaCppConfig);const tensorParallel = useChatRuntimeStore((s) => s.tensorParallel);const disableVision = useChatRuntimeStore((s) => s.disableVision);const chatTemplateOverride = useChatRuntimeStore(@@ -91,6 +92,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {return null;}const base: PerModelConfig = {+ llamaCppConfig: isGguf ? llamaCppConfig ?? undefined : undefined,customContextLength: customContextLength ?? null,// A self-sizing backend carries no pin here, exactly as the GGUF path does: this// is the runtime's resolved length, and reading it back as the user's choice would@@ -148,6 +150,7 @@ export function useActiveModelConfig(): ActiveModelConfigState {loadMode,ctxCheckpoints,cacheRam,+ llamaCppConfig,tensorParallel,disableVision,chatTemplateOverride,diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.tsindex c9c88ad6839..21c0c0b2581 100644--- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts@@ -57,6 +57,11 @@ export function applyPerModelConfigToRuntime(): { ids: null, indexKind: null };useChatRuntimeStore.setState({+ // Explicit managed on diffusion: an omitted field inherits the stored custom config.+ llamaCppConfig:+ options.isDiffusion && config.llamaCppConfig !== undefined+ ? { version: 1, mode: "managed" }+ : config.llamaCppConfig,customContextLength: config.customContextLength ?? null,mlxKvQuant: config.mlxKvQuant ?? null,kvCacheDtype: config.kvCacheDtype ?? null,@@ -118,6 +123,7 @@ export function currentRuntimePerModelConfig(): PerModelConfig {const s = useChatRuntimeStore.getState();return {+ llamaCppConfig: s.llamaCppConfig,customContextLength: s.customContextLength ?? null,maxSeqLength: options.includeMaxSeqLength? normalizeMaxSeqLength(s.params.maxSeqLength)@@ -155,6 +161,10 @@ export function currentRuntimePerModelConfig(};}+function customOnly(config: PerModelConfig["llamaCppConfig"]) {+ return config?.mode === "custom" ? config : undefined;+}+/** `followGlobal`: only against the running config, which holds the mode a null one resolved to.* Stored configs and presets keep null distinct from an explicit mode equal to today's global. */export function perModelConfigsEqual(@@ -166,6 +176,9 @@ export function perModelConfigsEqual(? resolvedSpeculativeType: normalizeSpeculativeType;return (+ // Unset and managed load the same engine.+ JSON.stringify(customOnly(a.llamaCppConfig)) ===+ JSON.stringify(customOnly(b.llamaCppConfig)) &&(a.customContextLength ?? null) === (b.customContextLength ?? null) &&normalizeMaxSeqLength(a.maxSeqLength) ===normalizeMaxSeqLength(b.maxSeqLength) &&diff --git a/studio/frontend/src/features/model-picker/model-config/config-signature.ts b/studio/frontend/src/features/model-picker/model-config/config-signature.tsindex c30ef802e63..b48f6718c4a 100644--- a/studio/frontend/src/features/model-picker/model-config/config-signature.ts+++ b/studio/frontend/src/features/model-picker/model-config/config-signature.ts@@ -48,6 +48,7 @@ export function loadedConfigSignature(return "none";}return [+ JSON.stringify(config.llamaCppConfig) ?? "",config.customContextLength ?? "",config.maxSeqLength ?? "",config.kvCacheDtype ?? "",diff --git a/studio/frontend/src/features/model-picker/model-config/llama-cpp-config.ts b/studio/frontend/src/features/model-picker/model-config/llama-cpp-config.tsnew file mode 100644index 00000000000..24cc8805330--- /dev/null+++ b/studio/frontend/src/features/model-picker/model-config/llama-cpp-config.ts@@ -0,0 +1,87 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++export type LlamaCppConfig =+ | { version: 1; mode: "managed" }+ | { version: 1; mode: "custom"; ini: string; section: string | null };++export interface LlamaCppConfigSummary {+ mode: "custom";+ section: string | null;+ options: Record<string, string | boolean>;+ request_defaults: Record<string, number>;+ diagnostics: string[];+}++export const MANAGED_LLAMA_CPP_CONFIG: LlamaCppConfig = {+ version: 1,+ mode: "managed",+};++export const MAX_LLAMA_CPP_CONFIG_BYTES = 65_536;++/** Shape validation only. The selected server is authoritative for INI semantics. */+export function normalizeLlamaCppConfig(+ value: unknown,+): LlamaCppConfig | undefined {+ if (!value || typeof value !== "object") return undefined;+ const source = value as Record<string, unknown>;+ if (source.version !== 1) return undefined;+ if (source.mode === "managed") return { version: 1, mode: "managed" };+ if (+ source.mode !== "custom" ||+ typeof source.ini !== "string" ||+ source.ini.trim().length === 0 ||+ (source.section !== null && typeof source.section !== "string") ||+ new TextEncoder().encode(source.ini).length > MAX_LLAMA_CPP_CONFIG_BYTES+ )+ return undefined;+ return {+ version: 1,+ mode: "custom",+ ini: source.ini,+ section: source.section,+ };+}++/** The mode switch: coming back to custom restores the last source instead of a blank one. */+export function toggledLlamaCppConfig(+ value: LlamaCppConfig | undefined,+ lastCustom: { ini: string; section: string | null } | null,+): LlamaCppConfig {+ if (value?.mode === "custom") return { version: 1, mode: "managed" };+ return {+ version: 1,+ mode: "custom",+ ini: lastCustom?.ini ?? "[*]\n",+ section: lastCustom?.section ?? null,+ };+}++/** Suggestions for the selector, never a parser or a validation verdict. */+// llama.cpp preset grammar: keys above the first header form the "default" section.+export function customConfigSections(ini: string): string[] {+ const header = /^\[[ \t]*([^\]\r\n]+)\][ \t]*(?:[#;][^\r\n]*)?\r?$/gm;+ const firstHeader = ini.search(header);+ const preamble = firstHeader < 0 ? ini : ini.slice(0, firstHeader);+ const hasDefault = /^[ \t]*[A-Za-z_]/m.test(preamble);+ return [+ ...new Set([+ ...(hasDefault ? ["default"] : []),+ ...[...ini.matchAll(header)]+ .map((match) => match[1].trim())+ .filter((name) => name !== "*"),+ ]),+ ];+}++export function llamaCppConfigPayload(+ config: LlamaCppConfig | undefined,+ options: { isDiffusion?: boolean } = {},+) {+ // The diffusion runner has no llama-server; explicit managed keeps the backend from inheriting custom.+ if (options.isDiffusion)+ return { llama_cpp_config: MANAGED_LLAMA_CPP_CONFIG };+ if (config === undefined) return {};+ return { llama_cpp_config: config };+}diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.tsindex 993249b7cf0..f2c0bd8fd68 100644--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import { normalizeLlamaCppConfig, type LlamaCppConfig } from "./llama-cpp-config";import type { GpuIndexKind } from "@/hooks/use-gpu-info";import {cachedRepoConfigId,@@ -49,6 +50,7 @@ export interface PerModelConfig {* alive); `null` means the user cleared the box and must be sent as an explicit `[]`; a non-empty list is what* to launch with. */llamaExtraArgs?: string[] | null;+ llamaCppConfig?: LlamaCppConfig;// GPU Memory controls (per-model, GGUF-only), optional so older blobs parse. Absent or null// selectedGpuIds means automatic.gpuMemoryMode?: "auto" | "manual";@@ -368,8 +370,9 @@ const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";// would normalize the unknown field straight back out of the record.// v2 added nBatch/nUbatch, v3 llamaExtraArgs, v4 disableVision, v5 the llama-server tuning group// (loadMode / specDraftCacheDtype / ctxCheckpoints / cacheRam), v6 the reasoning budget pair,-// v7 mlxKvQuant-const STORAGE_SCHEMA_VERSION = 7;+// v7 mlxKvQuant, and v8 custom llama.cpp configuration.+const STORAGE_SCHEMA_VERSION = 8;+const PRE_LLAMA_CPP_CONFIG_SCHEMA_VERSION = 7;const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;const PRE_REASONING_BUDGET_SCHEMA_VERSION = 5;const PRE_SERVER_TUNING_SCHEMA_VERSION = 4;@@ -426,6 +429,7 @@ const STORED_CONFIG_FIELDS = new Set(["disableVision","chatTemplateOverride","llamaExtraArgs",+ "llamaCppConfig","gpuMemoryMode","gpuLayers","nCpuMoe",@@ -1061,6 +1065,7 @@ function normalizeV1(partial: RawConfig): PerModelConfig {? partial.chatTemplateOverride: null,llamaExtraArgs: normalizeLlamaExtraArgs(partial.llamaExtraArgs),+ llamaCppConfig: normalizeLlamaCppConfig(partial.llamaCppConfig),...normalizeGpuFields(partial),};}@@ -1089,9 +1094,12 @@ function normalize(raw: unknown): PerModelConfig {* client reconstructs anyway, and stamping every record v4 would put the whole store out of reach.* The tuning group and the reasoning pair follow the same rule. */function storedSchemaVersion(normalized: PerModelConfig): number {- if (normalized.mlxKvQuant != null) {+ if (normalized.llamaCppConfig !== undefined) {return STORAGE_SCHEMA_VERSION;}+ if (normalized.mlxKvQuant != null) {+ return PRE_LLAMA_CPP_CONFIG_SCHEMA_VERSION;+ }const hasReasoningBudget =normalized.reasoningBudget !== -1 || normalized.reasoningBudgetMessage !== "";if (hasReasoningBudget) {@@ -1287,6 +1295,7 @@ export function isDefaultConfig(config: PerModelConfig): boolean {// Or a config whose only change is Extra Arguments reads as default, and savePerModelConfig// deletes the entry it was asked to remember.(config.llamaExtraArgs == null || config.llamaExtraArgs.length === 0) &&+ config.llamaCppConfig === undefined &&gpuFieldsAtDefault(config));}@@ -1310,6 +1319,9 @@ export function savePerModelConfig(* without this their server overrides would keep applying with nothing in the UI able to forget them. */evicted?: { modelId: string; ggufVariant: string | null }[],): boolean {+ if (config.llamaCppConfig !== undefined && normalizeLlamaCppConfig(config.llamaCppConfig) === undefined) {+ return false;+ }if (typeof config.chatTemplateOverride === "string" &&!isChatTemplateWithinLimit(config.chatTemplateOverride)diff --git a/studio/frontend/src/features/model-picker/sharing/fields.ts b/studio/frontend/src/features/model-picker/sharing/fields.tsindex d6c3f3b308a..bd38c613bcb 100644--- a/studio/frontend/src/features/model-picker/sharing/fields.ts+++ b/studio/frontend/src/features/model-picker/sharing/fields.ts@@ -41,7 +41,11 @@ const cacheType = nullable(export type SharedConfigKey = Exclude<keyof PerModelConfig,- "chatTemplateOverride" | "tensorSplit" | "maxSeqLength" | "mlxKvQuant"+ | "chatTemplateOverride"+ | "tensorSplit"+ | "maxSeqLength"+ | "mlxKvQuant"+ | "llamaCppConfig">;type Field = { label: string; valid: Validator; error?: string };diff --git a/studio/frontend/tests/llama-cpp-custom-config.test.ts b/studio/frontend/tests/llama-cpp-custom-config.test.tsnew file mode 100644index 00000000000..b936481b7b7--- /dev/null+++ b/studio/frontend/tests/llama-cpp-custom-config.test.ts@@ -0,0 +1,362 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";+import {+ installLocalStorageFake,+ registerStoreStubResolver,+ readSrc,+} from "./helpers/kit.ts";++registerStoreStubResolver();+const { store } = installLocalStorageFake();+const {+ normalizeLlamaCppConfig,+ customConfigSections,+ toggledLlamaCppConfig,+ llamaCppConfigPayload,+} =+ await import("../src/features/model-picker/model-config/llama-cpp-config.ts");+const {+ DEFAULT_PER_MODEL_CONFIG,+ PER_MODEL_CONFIG_STORAGE_KEY,+ savePerModelConfig,+ resolveInitialConfig,+} =+ await import("../src/features/model-picker/model-config/per-model-config.ts");+const { fromApiOverride, toApiOverride, resolveStoredOverride } =+ await import("../src/features/model-picker/api/model-overrides.ts");+const custom = {+ version: 1,+ mode: "custom",+ ini: "[*]\nfit=off\n[my-model]\nctx-size=56000\ntemp=0\n",+ section: "my-model",+} as const;+const managed = { version: 1, mode: "managed" } as const;++test("switching back to custom restores the last source instead of a blank one", () => {+ assert.deepEqual(toggledLlamaCppConfig(custom, null), managed);+ assert.deepEqual(+ toggledLlamaCppConfig(managed, {+ ini: custom.ini,+ section: custom.section,+ }),+ custom,+ );+ assert.deepEqual(toggledLlamaCppConfig(undefined, null), {+ version: 1,+ mode: "custom",+ ini: "[*]\n",+ section: null,+ });+ const editor = readSrc(+ "features/model-picker/components/custom-llama-config-editor.tsx",+ );+ // A refused load remounts the editor, so the remembered source has to live outside it.+ assert.match(editor, /^const lastCustomSource = new Map</m);+ assert.match(+ editor,+ /toggledLlamaCppConfig\(\s*value,\s*lastCustomSource\.get\(sourceKey\) \?\? null,?\s*\)/,+ );+ assert.match(+ editor,+ /if \(active\) lastCustomSource\.set\(sourceKey, \{ ini, section \}\)/,+ );+});++test("custom source and selection round-trip through storage and API without consuming legacy extras", () => {+ store.clear();+ const config = {+ ...DEFAULT_PER_MODEL_CONFIG,+ llamaExtraArgs: ["--threads", "3"],+ llamaCppConfig: custom,+ };+ assert.equal(savePerModelConfig("org/model", "Q4", config), true);+ const read = resolveInitialConfig("org/model", "Q4").config;+ assert.deepEqual(read.llamaCppConfig, custom);+ assert.deepEqual(+ fromApiOverride(toApiOverride(read), DEFAULT_PER_MODEL_CONFIG)+ .llamaCppConfig,+ custom,+ );+ assert.deepEqual(read.llamaExtraArgs, ["--threads", "3"]);+ const records = JSON.parse(store.get(PER_MODEL_CONFIG_STORAGE_KEY)!);+ assert.equal(+ Object.values(records).some(+ (row: unknown) => (row as { version: number }).version === 8,+ ),+ true,+ );+ assert.equal(+ savePerModelConfig("org/model", "Q4", { ...read, llamaCppConfig: managed }),+ true,+ );+ const reset = resolveInitialConfig("org/model", "Q4").config;+ assert.deepEqual(reset.llamaCppConfig, managed);+ assert.deepEqual(reset.llamaExtraArgs, ["--threads", "3"]);+});++test("a per-quant managed tombstone blocks bare-model custom fallback locally and on the mirror", () => {+ store.clear();+ savePerModelConfig("org/model", null, {+ ...DEFAULT_PER_MODEL_CONFIG,+ llamaCppConfig: custom,+ });+ savePerModelConfig("org/model", "Q4", {+ ...DEFAULT_PER_MODEL_CONFIG,+ llamaCppConfig: managed,+ });+ assert.deepEqual(+ resolveInitialConfig("org/model", "Q4").config.llamaCppConfig,+ managed,+ );+ assert.deepEqual(+ resolveStoredOverride(+ {+ "org/model": { llama_cpp_config: custom },+ "org/model:Q4": { llama_cpp_config: managed },+ },+ ["org/model:Q4", "org/model"],+ )?.llama_cpp_config,+ managed,+ );+});++test("custom configuration advances only its own storage tier", () => {+ const storedVersion = () => {+ const records = JSON.parse(store.get(PER_MODEL_CONFIG_STORAGE_KEY) ?? "{}");+ return (Object.values(records)[0] as { version?: number } | undefined)+ ?.version;+ };++ store.clear();+ assert.equal(+ savePerModelConfig("org/reasoning", "Q4", {+ ...DEFAULT_PER_MODEL_CONFIG,+ reasoningBudget: 256,+ reasoningBudgetMessage: "Keep the proof short",+ }),+ true,+ );+ assert.equal(storedVersion(), 6);+ const reasoning = resolveInitialConfig("org/reasoning", "Q4").config;+ assert.equal(reasoning.reasoningBudget, 256);+ assert.equal(reasoning.reasoningBudgetMessage, "Keep the proof short");+ assert.equal(reasoning.llamaCppConfig, undefined);++ store.clear();+ assert.equal(+ savePerModelConfig("org/custom", "Q4", {+ ...DEFAULT_PER_MODEL_CONFIG,+ reasoningBudget: 256,+ reasoningBudgetMessage: "Keep the proof short",+ llamaCppConfig: custom,+ }),+ true,+ );+ assert.equal(storedVersion(), 8);+ const customAndReasoning = resolveInitialConfig("org/custom", "Q4").config;+ assert.equal(customAndReasoning.reasoningBudget, 256);+ assert.equal(+ customAndReasoning.reasoningBudgetMessage,+ "Keep the proof short",+ );+ assert.deepEqual(customAndReasoning.llamaCppConfig, custom);++ store.clear();+ assert.equal(+ savePerModelConfig("org/tuning", "Q4", {+ ...DEFAULT_PER_MODEL_CONFIG,+ loadMode: "mmap",+ }),+ true,+ );+ assert.equal(storedVersion(), 5);+ assert.equal(+ resolveInitialConfig("org/tuning", "Q4").config.loadMode,+ "mmap",+ );+});++test("absent and explicit reset remain different request values", () => {+ assert.deepEqual(llamaCppConfigPayload(undefined), {});+ assert.deepEqual(llamaCppConfigPayload(managed), {+ llama_cpp_config: managed,+ });+ assert.equal(+ "llama_cpp_config" in toApiOverride(DEFAULT_PER_MODEL_CONFIG),+ false,+ );+ assert.deepEqual(+ fromApiOverride({}, { ...DEFAULT_PER_MODEL_CONFIG, llamaCppConfig: custom })+ .llamaCppConfig,+ custom,+ );+});++test("UTF-8 cap rejects an oversized edit without erasing its saved predecessor", () => {+ store.clear();+ savePerModelConfig("org/model", "Q4", {+ ...DEFAULT_PER_MODEL_CONFIG,+ llamaCppConfig: custom,+ });+ const oversized = { ...custom, ini: "é".repeat(32769) };+ assert.equal(normalizeLlamaCppConfig(oversized), undefined);+ assert.equal(+ savePerModelConfig("org/model", "Q4", {+ ...DEFAULT_PER_MODEL_CONFIG,+ llamaCppConfig: oversized,+ }),+ false,+ );+ assert.deepEqual(+ resolveInitialConfig("org/model", "Q4").config.llamaCppConfig,+ custom,+ );+ assert.equal(+ normalizeLlamaCppConfig({ ...custom, ini: "é".repeat(32768) })?.mode,+ "custom",+ );+});++test("blank custom source is rejected without erasing its saved predecessor", () => {+ store.clear();+ savePerModelConfig("org/model", "Q4", {+ ...DEFAULT_PER_MODEL_CONFIG,+ llamaCppConfig: custom,+ });+ const blank = { ...custom, ini: " \n\t " };+ assert.equal(normalizeLlamaCppConfig(blank), undefined);+ assert.equal(+ savePerModelConfig("org/model", "Q4", {+ ...DEFAULT_PER_MODEL_CONFIG,+ llamaCppConfig: blank,+ }),+ false,+ );+ assert.deepEqual(+ resolveInitialConfig("org/model", "Q4").config.llamaCppConfig,+ custom,+ );+});++test("selector suggestions never implicitly select a sole named section", () => {+ assert.deepEqual(customConfigSections("[*]\n[only]\nctx-size=56000"), [+ "only",+ ]);+ assert.deepEqual(customConfigSections("[ preset]\nctx-size=56000"), [+ "preset",+ ]);+ // llama.cpp files keys above the first header under "default", not under [*].+ assert.deepEqual(customConfigSections("c=1\n[*]\nngl=-1\n[large]\nc=9"), [+ "default",+ "large",+ ]);+ assert.deepEqual(customConfigSections("; note\n[large]\nc=9"), ["large"]);+ assert.deepEqual(customConfigSections("[a]\r\nc=1\r\n[b] ; x\r\nc=2\r\n"), [+ "a",+ "b",+ ]);+ assert.deepEqual(+ normalizeLlamaCppConfig({ ...custom, section: null })?.mode,+ "custom",+ );+ // Backend validates null against the source; the frontend does not invent a selection.+ assert.equal(+ (+ llamaCppConfigPayload({ ...custom, section: null })+ .llama_cpp_config as typeof custom+ ).section,+ null,+ );+});++test("large per-model sources still obey the aggregate storage budget", () => {+ store.clear();+ const evicted: { modelId: string; ggufVariant: string | null }[] = [];+ for (let index = 0; index < 22; index += 1) {+ assert.equal(+ savePerModelConfig(+ `org/model-${index}`,+ null,+ {+ ...DEFAULT_PER_MODEL_CONFIG,+ llamaCppConfig: { ...custom, ini: "#".repeat(60_000) },+ },+ evicted,+ ),+ true,+ );+ }+ assert.ok(evicted.length > 0);+ assert.ok(+ new TextEncoder().encode(store.get(PER_MODEL_CONFIG_STORAGE_KEY)!).length <=+ 1024 * 1024,+ );+ assert.equal(+ resolveInitialConfig("org/model-21", null).config.llamaCppConfig?.mode,+ "custom",+ );+});++test("a diffusion load sends managed in place of a custom config, never omits it", () => {+ const runtime = readSrc("features/chat/hooks/use-chat-model-runtime.ts");+ assert.equal(+ runtime.match(+ /llamaCppConfigPayload\(loadLlamaCppConfig, \{\s*isDiffusion: targetIsDiffusion/g,+ )?.length,+ 2,+ "validate and load both reset a diffusion target explicitly",+ );+ const compare = readSrc("features/chat/shared-composer.tsx");+ assert.equal(+ compare.match(+ /llamaCppConfigPayload\(ownConfig\.llamaCppConfig, \{\s*isDiffusion: resolvedIsDiffusion === true/g,+ )?.length,+ 2,+ "compare validate and load reset a diffusion target too",+ );+ // Both GGUF autoload paths record what launched, including the default-model fallback.+ assert.equal(+ readSrc("features/chat/api/chat-adapter.ts").split(+ "...loadedLlamaCppConfigFields(loadResp",+ ).length - 1,+ 2,+ );+ assert.deepEqual(llamaCppConfigPayload(custom, { isDiffusion: true }), {+ llama_cpp_config: managed,+ });+ assert.deepEqual(llamaCppConfigPayload(custom), { llama_cpp_config: custom });+ assert.deepEqual(llamaCppConfigPayload(managed, { isDiffusion: true }), {+ llama_cpp_config: managed,+ });+ // Unknown config still resets, or the backend inherits a saved custom source.+ assert.deepEqual(llamaCppConfigPayload(undefined, { isDiffusion: true }), {+ llama_cpp_config: managed,+ });+});++test("section suggestions trim padding the way the server does", () => {+ assert.deepEqual(customConfigSections("[*]\n[ fast ]\nc=1\n[fast]\n"), [+ "fast",+ ]);+});++test("a custom config locks the managed rows and keeps the editor outside them", () => {+ const page = readSrc(+ "features/model-picker/components/model-config-page.tsx",+ );+ const fieldsetEnd = page.indexOf("</fieldset>");+ assert.ok(page.indexOf("disabled={customActive}") < fieldsetEnd);+ assert.ok(page.indexOf("<CustomLlamaConfigEditor") > fieldsetEnd);+ assert.match(page, /!customActive &&\s*shouldRequestMemoryEstimate/);+ // Custom launches honour disable_vision, so its switch stays outside the lock.+ assert.ok(page.indexOf("hideVision={customActive}") < fieldsetEnd);+ // Hidden behind Advanced settings unless custom mode is already on.+ assert.match(+ page,+ /!resolvedIsDiffusion &&\s*\(showAdvanced \|\| customActive\)/,+ );+ assert.ok(page.indexOf("{customActive && <VisionRow") > fieldsetEnd);+});diff --git a/studio/frontend/tests/llama-cpp-custom-status-managed-fields.test.ts b/studio/frontend/tests/llama-cpp-custom-status-managed-fields.test.tsnew file mode 100644index 00000000000..0529b938927--- /dev/null+++ b/studio/frontend/tests/llama-cpp-custom-status-managed-fields.test.ts@@ -0,0 +1,117 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import { register } from "node:module";+import test from "node:test";++import { installLocalStorageFake } from "./helpers/kit.ts";++installLocalStorageFake();+register("./store-settings-resolver.mjs", import.meta.url);++const { useChatRuntimeStore } = await import(+ "../src/features/chat/stores/chat-runtime-store.ts"+);+const { applyActiveModelStatusToStore } = await import(+ "../src/features/chat/lib/apply-inference-status-to-store.ts"+);++const MODEL = "unsloth/Qwen3-1.7B-GGUF";+const custom = {+ version: 1,+ mode: "custom",+ ini: "[*]\nctx-size = 3072\ndevice = none\ngpu-layers = 0\n",+ section: null,+} as const;++// What /status reports after a custom load: the backend records the INI's placement as+// manual/0 layers and its ctx-size as the requested context.+function statusFor(config: unknown) {+ return {+ active_model: MODEL,+ model_identifier: MODEL,+ is_gguf: true,+ gpu_memory_mode: "manual",+ gpu_layers: 0,+ requested_context_length: 3072,+ context_length: 3072,+ requested_llama_cpp_config: config,+ } as never;+}++function managedAuto() {+ useChatRuntimeStore.setState({+ modelLoading: false,+ gpuMemoryMode: "auto",+ loadedGpuMemoryMode: "auto",+ gpuLayers: -1,+ loadedGpuLayers: null,+ customContextLength: null,+ loadedCustomContextLength: null,+ params: { ...useChatRuntimeStore.getState().params, checkpoint: MODEL },+ });+}++test("a custom load's INI placement does not overwrite the managed controls", () => {+ managedAuto();+ applyActiveModelStatusToStore(statusFor(custom), { previousCheckpoint: MODEL });+ const s = useChatRuntimeStore.getState();+ assert.equal(s.llamaCppConfig?.mode, "custom");+ assert.equal(s.gpuMemoryMode, "auto", "Use Studio settings would reload with Manual");+ assert.equal(s.gpuLayers, -1, "Use Studio settings would reload with --gpu-layers 0");+ assert.equal(s.customContextLength, null, "Use Studio settings would reload with ctx 3072");+});++test("a managed load reporting the same placement still hydrates the controls", () => {+ managedAuto();+ applyActiveModelStatusToStore(statusFor({ version: 1, mode: "managed" }), {+ previousCheckpoint: MODEL,+ });+ const s = useChatRuntimeStore.getState();+ assert.equal(s.gpuMemoryMode, "manual");+ assert.equal(s.gpuLayers, 0);+});++test("a custom load's response leaves the managed placement alone", async () => {+ const { managedGpuMemoryFields } = await import(+ "../src/features/chat/stores/chat-runtime-store.ts"+ );+ const resp = { is_gguf: true, gpu_memory_mode: "manual", gpu_layers: 0, n_layers: 28 } as const;+ assert.deepEqual(+ managedGpuMemoryFields({ ...resp, requested_llama_cpp_config: custom }),+ { ggufLayerCount: 28, moeLayerCount: null },+ );+ const managed = managedGpuMemoryFields({+ ...resp,+ requested_llama_cpp_config: { mode: "managed" },+ }) as { gpuMemoryMode?: string; gpuLayers?: number };+ assert.equal(managed.gpuMemoryMode, "manual");+ assert.equal(managed.gpuLayers, 0);+});++test("a custom load's KV cache and speculative echo leave the managed controls alone", async () => {+ const { managedKvCacheFields, managedSpeculativeSettings } = await import(+ "../src/features/chat/stores/chat-runtime-store.ts"+ );+ const resp = { cache_type_kv: "f16", speculative_type: "none", spec_draft_n_max: null };+ assert.deepEqual(managedKvCacheFields({ ...resp, requested_llama_cpp_config: custom }), {});+ assert.deepEqual(managedSpeculativeSettings({ ...resp, requested_llama_cpp_config: custom }), {});+ assert.equal(managedKvCacheFields(resp).kvCacheDtype, "f16");++ useChatRuntimeStore.setState({+ modelLoading: false,+ kvCacheDtype: null,+ loadedKvCacheDtype: null,+ speculativeType: "auto",+ loadedSpeculativeType: "auto",+ params: { ...useChatRuntimeStore.getState().params, checkpoint: MODEL },+ });+ applyActiveModelStatusToStore(+ { ...(statusFor(custom) as object), cache_type_kv: "f16", speculative_type: "none" } as never,+ { previousCheckpoint: MODEL },+ );+ const s = useChatRuntimeStore.getState();+ assert.equal(s.kvCacheDtype, null);+ assert.equal(s.speculativeType, "auto");+});diff --git a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts b/studio/frontend/tests/mlx-context-pin-record-compat.test.tsindex 96de26b75f3..4e8ea726265 100644--- a/studio/frontend/tests/mlx-context-pin-record-compat.test.ts+++ b/studio/frontend/tests/mlx-context-pin-record-compat.test.ts@@ -204,12 +204,12 @@ const ROWS: Row[] = [mlxRequest: 32768,transformersRequest: 32768,note:- "Not a future record: STORAGE_SCHEMA_VERSION is 7, so v4 is a v4-client record " ++ "Not a future record: STORAGE_SCHEMA_VERSION is 8, so v4 is a v4-client record " +"and reads normally.",},{- name: "version 8 (genuinely future)",- raw: { version: 8, customContextLength: 32768 },+ name: "version 9 (genuinely future)",+ raw: { version: 9, customContextLength: 32768 },normalizedPin: null,rawPin: 32768,isDefault: true,@@ -330,6 +330,7 @@ test("a patched pin round-trips through storage on both backends", () => {test("both pin shapes are stamped version 1, so neither is distinguishable by version", () => {assert.equal(stampedVersion({ customContextLength: 32768 }), 1);assert.equal(stampedVersion({ maxSeqLength: 32768 }), 1);+ // The current client's forwards guard is `version > 8`, and v1 invites any client// back to v1 to rewrite the record.assert.equal(stage({ version: 1, customContextLength: 32768 }).remembered,@@ -345,6 +346,10 @@ test("both pin shapes are stamped version 1, so neither is distinguishable by ve);assert.equal(stage({ version: 8, customContextLength: 32768 }).remembered,+ true,+ );+ assert.equal(+ stage({ version: 9, customContextLength: 32768 }).remembered,false,);});diff --git a/studio/frontend/tests/per-model-params.test.ts b/studio/frontend/tests/per-model-params.test.tsindex 78c562111a0..88792408ea6 100644--- a/studio/frontend/tests/per-model-params.test.ts+++ b/studio/frontend/tests/per-model-params.test.ts@@ -420,3 +420,18 @@ for (const [label, id] of [assert.equal(replayed.temperature, 0.2);});}++test("a custom llama.cpp repeat-penalty seeds the slider; otherwise it is kept", async () => {+ const { mergeBackendRecommendedInference } =+ await import("../src/features/chat/presets/preset-policy.ts");+ const merge = (inference: Record<string, number>) =>+ mergeBackendRecommendedInference({+ current: params({ repetitionPenalty: 1.3 }),+ response: { inference, is_gguf: true, context_length: 4096 },+ modelId: QWEN,+ presetSource: "builtin-default",+ loadedContextLength: 4096,+ }).repetitionPenalty;+ assert.equal(merge({ repetition_penalty: 1.1 }), 1.1);+ assert.equal(merge({ temperature: 0.7 }), 1.3);+});diff --git a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.tsindex 9844e99ca1c..256d9fc51bf 100644--- a/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts+++ b/studio/frontend/tests/resident-config-match-accelerator-matrix.test.ts@@ -473,6 +473,8 @@ test("every PerModelConfig field is either compared or deliberately excluded", (// Qualifies selectedGpuIds rather than adding a dimension of its own: it is read, as// the reconciler's namespace argument, but /status has no field to compare it against."selectedGpuIndexKind",+ // Custom intent always reaches backend preflight and compiled-identity comparison.+ "llamaCppConfig",// Compared through standing.splitRatio, which the caller seeds from it."tensorSplit",]);diff --git a/studio/frontend/tests/server-tuning-settings.test.ts b/studio/frontend/tests/server-tuning-settings.test.tsindex eac1cfdc97b..7e722f70f43 100644--- a/studio/frontend/tests/server-tuning-settings.test.ts+++ b/studio/frontend/tests/server-tuning-settings.test.ts@@ -113,11 +113,19 @@ test("a record only claims the new schema version when it carries one", () => {// toStoredConfig stamps the OLDEST version that understands every field// present, so an older client can still rewrite a record it fully knows.const source = readSrc("features/model-picker/model-config/per-model-config.ts");- assert.match(source, /const STORAGE_SCHEMA_VERSION = 7;/);+ assert.match(source, /const STORAGE_SCHEMA_VERSION = 8;/);+ assert.match(source, /const PRE_LLAMA_CPP_CONFIG_SCHEMA_VERSION = 7;/);assert.match(source, /const PRE_MLX_KV_QUANT_SCHEMA_VERSION = 6;/);assert.match(source, /const PRE_REASONING_BUDGET_SCHEMA_VERSION = 5;/);assert.match(source, /const PRE_SERVER_TUNING_SCHEMA_VERSION = 4;/);- assert.match(source, /mlxKvQuant != null\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/);+ assert.match(+ source,+ /normalized\.llamaCppConfig !== undefined\)\s*\{\s*return STORAGE_SCHEMA_VERSION;/,+ );+ assert.match(+ source,+ /mlxKvQuant != null\)\s*\{\s*return PRE_LLAMA_CPP_CONFIG_SCHEMA_VERSION;/,+ );assert.match(source,/hasReasoningBudget\)\s*\{\s*return PRE_MLX_KV_QUANT_SCHEMA_VERSION;/,diff --git a/studio/frontend/tests/tensor-parallel-row-gating.test.ts b/studio/frontend/tests/tensor-parallel-row-gating.test.tsindex c04172799d0..9c9c52e86aa 100644--- a/studio/frontend/tests/tensor-parallel-row-gating.test.ts+++ b/studio/frontend/tests/tensor-parallel-row-gating.test.ts@@ -45,5 +45,5 @@ test("the Tensor Parallelism row is gated out for diffusion models", () => {test("the Vision row it sits beside stays gated too", () => {// Both rows are unsupported for the same reason; regating one and not the other is// the state this change exists to end.- assert.equal(gateAbove("checked={!config.disableVision}"), "!isDiffusion");+ assert.equal(gateAbove("hideVision ? null : <VisionRow config={config} update={update} />"), "!isDiffusion");});diff --git a/studio/frontend/tests/vision-toggle-gating.test.ts b/studio/frontend/tests/vision-toggle-gating.test.tsindex 5e7892c2791..918a4671e0f 100644--- a/studio/frontend/tests/vision-toggle-gating.test.ts+++ b/studio/frontend/tests/vision-toggle-gating.test.ts@@ -91,19 +91,20 @@ test("the Vision row exists only in the GGUF half of Advanced Settings", () => {1,"expected exactly one Vision switch in the file",);+ assert.ok(bodyOf("VisionRow").includes(wiring));assert.ok(- gguf.includes(wiring),+ gguf.includes("<VisionRow"),"Vision switch is not in GgufAdvancedSettings",);assert.ok(- !mlx.includes(wiring),+ !mlx.includes("VisionRow"),"Vision switch leaked into MlxAdvancedSettings",);assert.ok(!mlx.includes("disableVision"),"MlxAdvancedSettings reads disableVision, which it cannot act on",);- assert.ok(gguf.includes(">Vision</span>"));+ assert.ok(bodyOf("VisionRow").includes(">Vision</span>"));// And the GGUF half only renders under target.isGguf, so a non-GGUF target// shows no Vision switch at all.@@ -286,7 +287,9 @@ test("the Vision row is gated out for diffusion models", () => {const lines = CONFIG_PAGE.split("\n");const visionAt = lines.findIndex((line) =>- line.includes("checked={!config.disableVision}"),+ line.includes(+ "hideVision ? null : <VisionRow config={config} update={update} />",+ ),);assert.notEqual(visionAt, -1, "no Vision switch to gate");diff --git a/tests/studio/test_chat_autoload_failure_gate.py b/tests/studio/test_chat_autoload_failure_gate.pyindex 4c380b24f80..fefc098b71a 100644--- a/tests/studio/test_chat_autoload_failure_gate.py+++ b/tests/studio/test_chat_autoload_failure_gate.py@@ -551,6 +551,11 @@ function reasoningCapsFromLoad(_x: any) { return {}; }function resolveToolsEnabledOnLoad(_x: any) { return {}; }function loadedGpuMemoryFields(_x: any) { return {}; }function resolveLoadedSpeculativeSettings(_x: any) { return {}; }+function managedGpuMemoryFields(_x: any) { return {}; }+function managedKvCacheFields(_x: any) { return {}; }+function managedSpeculativeSettings(_x: any) { return {}; }+function llamaCppConfigPayload(c: any) { return c === undefined ? {} : { llama_cpp_config: c }; }+function loadedLlamaCppConfigFields(r: any, sent: any) { const c = r.requested_llama_cpp_config ?? sent; return { llamaCppConfig: c, loadedLlamaCppConfig: c ?? null, llamaCppConfigSummary: r.llama_cpp_config_summary ?? null }; }function isMultimodalResponse(_x: any) { return false; }async function listCachedGguf(signal?: any) {diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.pyindex 405f6b4d78f..0062e0104ce 100644--- a/tests/studio/test_model_picker_contracts.py+++ b/tests/studio/test_model_picker_contracts.py@@ -1552,7 +1552,10 @@ def test_forget_settings_is_not_locked_by_unloadable_extra_args():"""Forget only deletes, so invalid saved llama args must not lock it: the args gatesapply to a save only."""gate = " ".join(_save_button_gate().split())- assert "(remember && ((!extraArgsLoadable && !sharedExtraArgsCleared) ||" in gate, gate+ assert re.search(+ r"\(remember && \((?:\([^()]*\) \|\| )?\(!extraArgsLoadable && !sharedExtraArgsCleared\) \|\|",+ gate,+ ), gateassert "sharedExtraArgsRefused || extraArgsHydrating))" in gate, gate