Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: respell user ngram-mod extra args for older llama.cpp builds (#14880)

Daniel Hanb04b740de880
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index d36ba0f61ce..efa6069160b 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -155,6 +155,7 @@ from core.inference.llama_server_args import (
strip_context_only,
strip_shadowing_flags,
strip_split_mode_only,
+ translate_ngram_mod_args,
)
@@ -29127,6 +29128,14 @@ class LlamaCppBackend:
"Dropped user device placement flags from extra args: "
"explicit gpu_ids owns placement."
)
+ _emit_extra_args = translate_ngram_mod_args(
+ _emit_extra_args,
+ _caps.get("ngram_mod_flavor"),
+ chain_with_mtp = self._speculative_type == "draft-mtp"
+ or _extra_args_requests_mtp(
+ _emit_extra_args, env = _child_spec_env(_emit_extra_args)
+ ),
+ )
cmd.extend(str(a) for a in _emit_extra_args)
logger.info(
f"Appending user extra args to llama-server: {list(_emit_extra_args)}"
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index 473666ada01..f46324b967d 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -1890,3 +1890,38 @@ def memory_state_satisfies_settings(
return False
return mlock or not mlock_applicable
return not policy_active
+
+
+_NGRAM_MOD_LEGACY_FLAGS = {
+ "--spec-ngram-mod-n-match": "--spec-ngram-size-n",
+ "--spec-ngram-mod-n-min": "--draft-min",
+ "--spec-ngram-mod-n-max": "--draft-max",
+}
+
+
+def translate_ngram_mod_args(
+ args: Iterable[str],
+ flavor: Optional[str],
+ *,
+ chain_with_mtp: bool = False,
+) -> list[str]:
+ """Respell ``--spec-ngram-mod-n-*`` for a legacy build (else it refuses to start); with MTP drop min/max (MTP owns the range)."""
+ args = [str(a) for a in args]
+ if flavor != "legacy":
+ return args
+ out: list[str] = []
+ i = 0
+ while i < len(args):
+ _flag, eq, value = args[i].partition("=")
+ legacy = _NGRAM_MOD_LEGACY_FLAGS.get(_flag_name(args[i]) or "")
+ if legacy is None or (not eq and i + 1 >= len(args)):
+ out.append(args[i])
+ i += 1
+ continue
+ if not eq:
+ value = args[i + 1]
+ i += 1 if eq else 2
+ if chain_with_mtp and legacy != "--spec-ngram-size-n":
+ continue
+ out.extend([legacy, value])
+ return out
diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py
index abb6a72e910..9d40d8e42e8 100644
--- a/studio/backend/tests/test_llama_server_args.py
+++ b/studio/backend/tests/test_llama_server_args.py
@@ -1448,6 +1448,72 @@ def test_custom_projector_is_allowed(flag):
assert validate_extra_args(args) == args
+def test_ngram_mod_extras_are_respelled_for_a_legacy_build():
+ from core.inference.llama_server_args import translate_ngram_mod_args
+
+ knobs = [
+ "--spec-ngram-mod-n-match",
+ "16",
+ "--spec-ngram-mod-n-min=8",
+ "--spec-ngram-mod-n-max",
+ "64",
+ "--top-k",
+ "20",
+ ]
+ assert translate_ngram_mod_args(knobs, "new") == knobs
+ assert translate_ngram_mod_args(knobs, None) == knobs
+ assert translate_ngram_mod_args(knobs, "legacy") == [
+ "--spec-ngram-size-n",
+ "16",
+ "--draft-min",
+ "8",
+ "--draft-max",
+ "64",
+ "--top-k",
+ "20",
+ ]
+ assert translate_ngram_mod_args(knobs, "legacy", chain_with_mtp = True) == [
+ "--spec-ngram-size-n",
+ "16",
+ "--top-k",
+ "20",
+ ]
+ # A trailing flag with no value is left for llama-server to reject.
+ assert translate_ngram_mod_args(["--spec-ngram-mod-n-max"], "legacy") == [
+ "--spec-ngram-mod-n-max"
+ ]
+
+
+def test_the_launch_respells_user_ngram_mod_extras_last():
+ """Source check: reaching the emit point needs a real binary."""
+ import inspect
+
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ src = inspect.getsource(LlamaCppBackend.load_model)
+ emit = src[src.index("_emit_extra_args = self._strip_mmproj_args(") :]
+ emit = emit[: emit.index("cmd.extend(str(a) for a in _emit_extra_args)")]
+ assert "translate_ngram_mod_args(" in emit
+ assert '_caps.get("ngram_mod_flavor")' in emit
+ assert emit.index("_strip_device_extra_args") < emit.index("translate_ngram_mod_args(")
+ assert "_extra_args_requests_mtp(" in emit[emit.index("translate_ngram_mod_args(") :]
+
+
+def test_ngram_mod_underscore_spelling_is_respelled_too():
+ from core.inference.llama_server_args import translate_ngram_mod_args
+ assert translate_ngram_mod_args(
+ ["--spec_ngram_mod_n_min", "8", "--spec_ngram_mod_n_match=4"], "legacy"
+ ) == ["--draft-min", "8", "--spec-ngram-size-n", "4"]
+
+
+def test_user_spec_type_with_mtp_counts_as_chained():
+ from core.inference.llama_cpp import _child_spec_env, _extra_args_requests_mtp
+
+ extras = ["--spec-type", "ngram-mod,draft-mtp", "--spec-ngram-mod-n-min", "48"]
+ assert _extra_args_requests_mtp(extras, env = _child_spec_env(extras))
+ assert not _extra_args_requests_mtp(["--spec-ngram-mod-n-min", "48"], env = {})
+
+
def test_owner_only_path_flags_names_each_file_option_once():
args = [
"--ctx-size",