Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: accept auto and all in custom GPU-layer arguments
Daniel Hanc3459cdbc15d
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex b9716d65129..6708b4d641c 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -851,13 +851,20 @@ def resolve_reasoning_budget_message_with_env(def parse_gpu_layers_override(args: Optional[Iterable[str]]) -> Optional[int]:"""Last user-supplied -ngl value from extras, read before Manual mode strips the flags."""- raw_value = _last_flag_value(args, _GPU_LAYER_FLAGS)+ raw_value = _last_flag_value(args, _GPU_LAYER_FLAGS, preserve_raw = True)if raw_value is None:return None+ if raw_value == "auto":+ return -1+ if raw_value == "all":+ # Manual mode needs a fixed count; negative values enable fitting.+ return 2**31 - 1try:value = int(raw_value)except ValueError as exc:- raise ValueError("llama-server GPU layers flag requires an integer value") from exc+ raise ValueError(+ "llama-server GPU layers flag requires an integer value, auto, or all"+ ) from excif value < -1:raise ValueError("llama-server GPU layers flag requires an integer value of at least -1")return valuediff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.pyindex 9b3dc2f28a1..42ddca40ca3 100644--- a/studio/backend/routes/settings.py+++ b/studio/backend/routes/settings.py@@ -1093,7 +1093,7 @@ class ModelOverridePayload(BaseModel):chat_template_override: Optional[str] = Nonegpu_memory_mode: Optional[Literal["auto", "manual"]] = None# -1 is Auto (llama.cpp --fit sizes the offload); the normalizer treats it as unset.- gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024)+ gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 2**31 - 1)n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024)tensor_split: Optional[list[float]] = Field(default = None, min_length = 2, max_length = MAX_GPU_IDS)gpu_ids: Optional[list[int]] = Field(default = None, max_length = MAX_GPU_IDS)diff --git a/studio/backend/tests/test_llama_extra_args_compatibility.py b/studio/backend/tests/test_llama_extra_args_compatibility.pyindex 30fa4b0773f..1a51d541439 100644--- a/studio/backend/tests/test_llama_extra_args_compatibility.py+++ b/studio/backend/tests/test_llama_extra_args_compatibility.py@@ -221,6 +221,32 @@ def test_the_override_save_carries_over_without_refusing(monkeypatch):assert "--numa" in kept[email protected]("flag", ["-ngl", "--gpu-layers", "--n-gpu-layers"])[email protected]("value", ["auto", "all"])+def test_override_save_and_reload_keep_gpu_layer_keywords(monkeypatch, flag, value):+ import routes.settings as settings_route+ import utils.openai_auto_switch_settings as settings++ saved = {}+ monkeypatch.setattr(settings, "get_model_override", lambda _id: {})+ monkeypatch.setattr(+ settings, "set_model_override", lambda model_id, **kwargs: saved.update({model_id: kwargs})+ )+ monkeypatch.setattr(settings_route, "set_model_override", settings.set_model_override)+ monkeypatch.setattr(settings_route, "resolve_model_override_keys", lambda _id: ["local/x"])+ monkeypatch.setattr(settings_route, "cached_repo_alias_keys", lambda _id: [])+ args = [flag, value]+ payload = settings_route.ModelOverridePayload(model_id = "local/x", llama_extra_args = args)++ settings_route.update_openai_auto_switch_override(payload, current_subject = "t")++ assert saved["local/x"]["llama_extra_args"] == args+ assert (+ settings.model_override_load_kwargs(saved["local/x"], is_gguf = True)["llama_extra_args"]+ == args+ )++def test_the_auto_switch_path_sanitizes_a_legacy_override(monkeypatch):from utils.openai_auto_switch_settings import model_override_load_kwargsdiff --git a/studio/backend/tests/test_llama_extra_args_end_to_end.py b/studio/backend/tests/test_llama_extra_args_end_to_end.pyindex 80566c3d677..0d9cb3975d8 100644--- a/studio/backend/tests/test_llama_extra_args_end_to_end.py+++ b/studio/backend/tests/test_llama_extra_args_end_to_end.py@@ -51,6 +51,31 @@ def test_an_extra_arg_reaches_the_child(tmp_path):assert cmd[cmd.index("--top-k") + 1] == "20"[email protected]("value", ["auto", "all"])+def test_gpu_layer_keywords_reach_the_auto_launcher_verbatim(tmp_path, value):+ args = ["-ngl", value]+ cmd = _cmd(tmp_path, extra_args = args)++ assert cmd[-len(args) :] == args++[email protected]("value, layers", [("auto", -1), ("all", 2147483647)])+def test_promoted_gpu_layer_keywords_keep_manual_fit_intent(tmp_path, value, layers):+ from core.inference.llama_server_args import parse_gpu_layers_override+ cmd = _cmd(+ tmp_path,+ gpu_memory_mode = "manual",+ gpu_layers = parse_gpu_layers_override(["-ngl", value]),+ )++ if layers < 0:+ assert "--gpu-layers" not in cmd and "-ngl" not in cmd+ assert cmd[cmd.index("--fit") + 1] == "on"+ else:+ assert cmd[cmd.index("--gpu-layers") + 1] == str(layers)+ assert cmd[cmd.index("--fit") + 1] == "off"++def test_extra_args_come_after_unsloths_own(tmp_path):cmd = _cmd(tmp_path, extra_args = ["--top-k", "20"])managed = [i for i, token in enumerate(cmd) if token in {"--model", "-m", "--port"}]diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.pyindex c5ed950257b..a3bf4865bc4 100644--- a/studio/backend/tests/test_llama_server_args.py+++ b/studio/backend/tests/test_llama_server_args.py@@ -486,6 +486,28 @@ def test_parse_gpu_layers_override(args, expected):assert parse_gpu_layers_override(args) == expected[email protected]("flag", ["-ngl", "--gpu-layers", "--n-gpu-layers"])[email protected]("value,expected", [("auto", -1), ("all", 2**31 - 1)])+def test_gpu_layer_keywords_are_preserved_and_resolved(flag, value, expected):+ args = [flag, value]+ assert validate_extra_args(args) == args+ assert parse_gpu_layers_override(args) == expected+++ "args,expected",+ [+ (["-ngl", "all", "--gpu-layers", "auto"], -1),+ (["-ngl", "auto", "--n-gpu-layers", "all"], 2**31 - 1),+ (["-ngl", "all", "--gpu-layers", "0"], 0),+ (["-ngl", "12", "--gpu-layers", "all"], 2**31 - 1),+ ],+)+def test_gpu_layer_keywords_respect_last_override(args, expected):+ assert validate_extra_args(args) == args+ assert parse_gpu_layers_override(args) == expected++@pytest.mark.parametrize("args",[diff --git a/studio/backend/tests/test_model_override_schema_compatibility.py b/studio/backend/tests/test_model_override_schema_compatibility.pyindex 4d48ce669b4..781404e4df0 100644--- a/studio/backend/tests/test_model_override_schema_compatibility.py+++ b/studio/backend/tests/test_model_override_schema_compatibility.py@@ -40,6 +40,13 @@ TUNING_PAYLOAD = dict()+def test_manual_full_offload_count_survives_save_and_reload(override_store):+ _put(MODEL, gpu_memory_mode = "manual", gpu_layers = 2**31 - 1)+ saved = settings.get_model_override(MODEL)+ assert saved["gpu_layers"] == 2**31 - 1+ assert settings.model_override_load_kwargs(saved, is_gguf = True)["gpu_layers"] == 2**31 - 1++def test_a_row_written_before_the_tuning_group_still_loads(override_store):legacy_row = {"llama_extra_args": ["--numa", "distribute"],diff --git a/studio/backend/tests/test_validate_diffusion_extra_args.py b/studio/backend/tests/test_validate_diffusion_extra_args.pyindex 4e0acdeeb23..6c363c97e3e 100644--- a/studio/backend/tests/test_validate_diffusion_extra_args.py+++ b/studio/backend/tests/test_validate_diffusion_extra_args.py@@ -10,6 +10,8 @@ from pathlib import Pathfrom types import SimpleNamespacefrom unittest.mock import patch+import pytest+from models.inference import ValidateModelRequest@@ -163,6 +165,7 @@ class TestValidateTranslatesManualNgl(unittest.TestCase):gpu_layers,extra_args,diffusion_kind = True,+ gpu_memory_mode = "manual",):seen: list = []@@ -172,7 +175,7 @@ class TestValidateTranslatesManualNgl(unittest.TestCase):request = ValidateModelRequest(model_path = "someone/diffusion-gguf",llama_extra_args = extra_args,- gpu_memory_mode = "manual",+ gpu_memory_mode = gpu_memory_mode,gpu_layers = gpu_layers,)config = SimpleNamespace(@@ -248,6 +251,62 @@ class TestValidateTranslatesManualNgl(unittest.TestCase):self.assertEqual(seen, [(request.gpu_layers, ["-ngl", "20"])])[email protected]("flag", ["-ngl", "--gpu-layers", "--n-gpu-layers"])[email protected]("value, layers", [("auto", -1), ("all", 2147483647)])[email protected]("mode", ["auto", "manual"])+def test_validate_preserves_gpu_layer_keyword_intent(flag, value, layers, mode):+ route = _load_route_module("inf_route_ngl_keywords")+ args = [flag, value]+ seen = TestValidateTranslatesManualNgl()._validate(+ route,+ gpu_layers = 0,+ extra_args = args,+ diffusion_kind = False,+ gpu_memory_mode = mode,+ )+ assert seen == ([(layers, [])] if mode == "manual" else [(0, args)])++[email protected]("value, layers", [("auto", -1), ("all", 2147483647)])[email protected]("mode", ["auto", "manual"])+def test_load_normalizes_gpu_layer_keywords_before_model_work(monkeypatch, value, layers, mode):+ from fastapi import HTTPException+ from starlette.requests import Request+ from models.inference import LoadRequest+ import routes.inference as route++ seen = []++ def capture(request, **kwargs):+ seen.append((request.gpu_layers, request.llama_extra_args))+ raise HTTPException(status_code = 418, detail = "stopped before model resolution")++ async def no_ollama_lease(*args, **kwargs):+ return None++ monkeypatch.setattr(route, "_resolve_model_identifier_for_request", capture)+ monkeypatch.setattr(route, "_lease_ollama_model_ref", no_ollama_lease)+ monkeypatch.setattr(route, "_as_ollama_manifest_request", lambda request: request)+ monkeypatch.setattr(route, "_as_local_scan_folder_request", lambda request, owner: request)+ monkeypatch.setattr(route, "_repo_is_in_the_hub_cache", lambda model: None)+ monkeypatch.setattr(route, "_hub_cache_footprint", lambda model: None)+ monkeypatch.setattr(route, "_lora_base_already_in_the_hub_cache", lambda model: None)+ monkeypatch.setattr(route, "_note_lora_base_fetched_with_a_request_token", lambda *a, **k: None)+ args = ["-ngl", value]+ request = LoadRequest(+ model_path = "placeholder.gguf",+ gpu_memory_mode = mode,+ gpu_layers = 0,+ llama_extra_args = args,+ )++ with pytest.raises(HTTPException) as caught:+ asyncio.run(route._load_model_impl(request, Request({"type": "http", "headers": []}), "t"))++ assert caught.value.status_code == 418+ assert seen == ([(layers, [])] if mode == "manual" else [(0, args)])++class TestValidateTranslatesManualTensorSplit(unittest.TestCase):"""/validate must promote -ts to tensor_split as /load does, or it judges a different split."""diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.pyindex 4dbc5699f70..77ec325e8ab 100644--- a/studio/backend/utils/openai_auto_switch_settings.py+++ b/studio/backend/utils/openai_auto_switch_settings.py@@ -583,7 +583,7 @@ def normalize_model_override(if _clean_str(payload.get("gpu_memory_mode"), VALID_GPU_MEMORY_MODES) == "manual":entry["gpu_memory_mode"] = "manual"- gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 1024)+ gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 2**31 - 1)if gpu_layers is not None:entry["gpu_layers"] = gpu_layersdiff --git a/studio/frontend/src/features/chat/lib/llama-extra-args-normalize.ts b/studio/frontend/src/features/chat/lib/llama-extra-args-normalize.tsindex b3663595bda..11bb6360264 100644--- a/studio/frontend/src/features/chat/lib/llama-extra-args-normalize.ts+++ b/studio/frontend/src/features/chat/lib/llama-extra-args-normalize.ts@@ -78,6 +78,12 @@ export function parseGpuLayersOverride(if (raw === null) {return { kind: "absent" };}+ if (raw === "auto") {+ return { kind: "value", layers: -1 };+ }+ if (raw === "all") {+ return { kind: "value", layers: 2147483647 };+ }const value = Number.parseInt(raw, 10);return Number.isInteger(value) && value >= -1 && String(value) === raw.trim()? { kind: "value", layers: value }diff --git a/studio/frontend/src/features/model-picker/model-config/llama-extra-args.ts b/studio/frontend/src/features/model-picker/model-config/llama-extra-args.tsindex 2999151eaff..bc38ba2a8de 100644--- a/studio/frontend/src/features/model-picker/model-config/llama-extra-args.ts+++ b/studio/frontend/src/features/model-picker/model-config/llama-extra-args.ts@@ -218,6 +218,7 @@ function dropUnusableValues(tokens: readonly string[]): string[] {(RATIO_VALUE_FLAGS.has(flag) &&ratioValueProblem(flag, value, false) !== null) ||(INTEGER_VALUE_FLAGS.has(flag) &&+ !isGpuLayersKeyword(flag, value) &&(!INTEGER.test(value.trim()) ||(minimum !== undefined && Number(value.trim()) < minimum)));if (!unusable) {@@ -587,20 +588,24 @@ const VALUE_REQUIRED_FLAGS = new Set([const GPU_LAYERS_FLAGS = new Set(["--gpu-layers", "--n-gpu-layers", "-ngl"]);-/** Mirrors parse_gpu_layers_override only for: is the resolved count non-negative. */-function lastIntegerFlagValue(- tokens: readonly string[],- flags: ReadonlySet<string>,-): number | null {+function isGpuLayersKeyword(flag: string, value: string | undefined): boolean {+ return GPU_LAYERS_FLAGS.has(flag) && (value === "auto" || value === "all");+}++function lastGpuLayersValue(tokens: readonly string[]): number | null {let found: number | null = null;for (const [index, token] of tokens.entries()) {const flag = extraArgFlagName(token);- if (flag === null || !flags.has(flag)) {+ if (flag === null || !GPU_LAYERS_FLAGS.has(flag)) {continue;}const attached = valueIsAttached(token, flag);const raw = attached ? token.split("=")[1] : tokens[index + 1];- if (raw !== undefined && INTEGER.test(raw.trim())) {+ if (raw === "auto") {+ found = -1;+ } else if (raw === "all") {+ found = 2147483647;+ } else if (raw !== undefined && INTEGER.test(raw.trim())) {found = Number(raw.trim());}}@@ -738,7 +743,7 @@ export function diagnoseExtraArgs(const out: ExtraArgsDiagnostic[] = [];const { tokens, unterminatedQuote, quotedIndices } = parseExtraArgs(input);// Mirrors _should_strip_tensor_split: an extras -ngl is promoted first.- const nglOverride = lastIntegerFlagValue(tokens, GPU_LAYERS_FLAGS);+ const nglOverride = lastGpuLayersValue(tokens);const resolvedGpuLayers = nglOverride ?? context.gpuLayers ?? -1;const reserializesSplit = manualGpuMemory && resolvedGpuLayers >= 0;// A token in value position is a value whatever it starts with; llama.cpp takes the next argv blindly.@@ -883,7 +888,7 @@ export function diagnoseExtraArgs(: `${flag} needs a value after it.`;} else if (RATIO_VALUE_FLAGS.has(flag)) {message = ratioValueProblem(flag, value, reserializesSplit);- } else if (!numeric) {+ } else if (!numeric || isGpuLayersKeyword(flag, value)) {message = null;} else if (!INTEGER.test(value.trim())) {message = `${flag} takes a number, and "${value}" is not one.`;diff --git a/studio/frontend/src/features/model-picker/sharing/extra-args.ts b/studio/frontend/src/features/model-picker/sharing/extra-args.tsindex 624fa9d372b..1f458279060 100644--- a/studio/frontend/src/features/model-picker/sharing/extra-args.ts+++ b/studio/frontend/src/features/model-picker/sharing/extra-args.ts@@ -40,7 +40,8 @@ const threads = number(-1, 1024, true);const context: ValueRule = (value) =>value === "0" || number(CONTEXT_LENGTH_MIN, 2_147_483_647)(value);const batch: ValueRule = (value) => number(N_BATCH_MIN, N_BATCH_MAX)(value);-const layers = number(-1, 2_147_483_647);+const layers: ValueRule = (value) =>+ value === "auto" || value === "all" || number(-1, 2_147_483_647)(value);const cache: ValueRule = (value) =>value === "f16" || KV_CACHE_DTYPES.some((dtype) => dtype === value);const ratio = number(0, 1);diff --git a/studio/frontend/tests/llama-extra-args-diagnostics.test.ts b/studio/frontend/tests/llama-extra-args-diagnostics.test.tsindex d7bd9db560e..35be89ca426 100644--- a/studio/frontend/tests/llama-extra-args-diagnostics.test.ts+++ b/studio/frontend/tests/llama-extra-args-diagnostics.test.ts@@ -224,6 +224,40 @@ test("a flag that takes a number rejects a value that is not one", () => {);});+test("GPU layer keywords load and survive stored-argument sanitizing", () => {+ for (const flag of ["-ngl", "--gpu-layers", "--n-gpu-layers"]) {+ for (const value of ["auto", "all"]) {+ const tokens = [flag, value, "--top-k", "20"];+ for (const manualGpuMemory of [false, true]) {+ const diagnostics = diagnoseExtraArgs(tokens.join(" "), CATALOG, {+ manualGpuMemory,+ });+ assert.equal(extraArgsAreLoadable(diagnostics), true, JSON.stringify(diagnostics));+ }+ assert.deepEqual(sanitizeStoredExtraArgs(tokens, CATALOG.managed), tokens);+ }+ for (const value of ["many", "AUTO", "ALL", "-2"]) {+ const tokens = [flag, value];+ assert.equal(extraArgsAreLoadable(diagnoseExtraArgs(tokens.join(" "), CATALOG)), false);+ assert.deepEqual(sanitizeStoredExtraArgs(tokens, CATALOG.managed), []);+ }+ }+ assert.equal(extraArgsAreLoadable(diagnoseExtraArgs("--batch-size all", CATALOG)), false);+});++test("GPU layer keywords preserve the last override's manual ratio behavior", () => {+ const ratio = "-ts 1.1754943508222874e-38,1";+ const error = (layers: string, gpuLayers: number) =>+ diagnoseExtraArgs(`${layers} ${ratio}`, CATALOG, {+ manualGpuMemory: true,+ gpuLayers,+ }).find((d) => d.level === "error")?.message ?? null;+ assert.equal(error("-ngl auto", 49), null);+ assert.equal(error("-ngl all --gpu-layers auto", 49), null);+ assert.match(error("-ngl all", -1) ?? "", /0 or at least/);+ assert.match(error("-ngl auto --gpu-layers all", -1) ?? "", /0 or at least/);+});+test("a control character is an error, as it is at the backend", () => {const diagnostics = diagnoseExtraArgs("--grammar a\u001b[0mb", CATALOG);assert.equal(diagnostics[0].level, "error");diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.tsindex c8f066dd6a9..990f3b0eef4 100644--- a/studio/frontend/tests/resident-config-match.test.ts+++ b/studio/frontend/tests/resident-config-match.test.ts@@ -885,6 +885,17 @@ test("the arch-gate excuse reads the resolved split, not the toggle", () => {);});+test("manual GPU layer keywords adopt a matching resident model", () => {+ for (const flag of ["-ngl", "--gpu-layers", "--n-gpu-layers"]) {+ for (const [value, layers] of [["auto", -1], ["all", 2147483647]] as const) {+ assert.equal(matches(+ { ...DEFAULTS, is_gguf: true, gpu_memory_mode: "manual", gpu_layers: layers },+ { ...BLANK, gpuMemoryMode: "manual", llamaExtraArgs: [flag, value] },+ ), true, `${flag} ${value}`);+ }+ }+});+test("a malformed manual layer override declines rather than normalizing away", () => {// parse_gpu_layers_override raises on these, so they must not fold into no override.const running = {diff --git a/studio/frontend/tests/share-run-configs.security.test.ts b/studio/frontend/tests/share-run-configs.security.test.tsindex cf880db8903..742fbc8bd23 100644--- a/studio/frontend/tests/share-run-configs.security.test.ts+++ b/studio/frontend/tests/share-run-configs.security.test.ts@@ -47,6 +47,14 @@ const params = (key: string, value: string) =>const args = (list: unknown) => query("llamaExtraArgs", list);const valid = (config: object) => ({ kind: "valid", value: { config } });+test("GPU layer keywords round-trip through shared settings", () => {+ for (const flag of ["-ngl", "--gpu-layers", "--n-gpu-layers"]) {+ for (const value of ["auto", "all"]) {+ roundTrips({ config: { llamaExtraArgs: [flag, value] } });+ }+ }+});+function rejects(query: string) {for (const prefix of prefixes) {const result = parseRunConfigLink(`${prefix}${query}`);