Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: accept auto and all in custom GPU-layer arguments

Daniel Hanc3459cdbc15d
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index b9716d65129..6708b4d641c 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -851,13 +851,20 @@ def resolve_reasoning_budget_message_with_env(
def parse_gpu_layers_override(args: Optional[Iterable[str]]) -> Optional[int]:
"""Last user-supplied -ngl value from extras, read before Manual mode strips the flags."""
- raw_value = _last_flag_value(args, _GPU_LAYER_FLAGS)
+ raw_value = _last_flag_value(args, _GPU_LAYER_FLAGS, preserve_raw = True)
if raw_value is None:
return None
+ if raw_value == "auto":
+ return -1
+ if raw_value == "all":
+ # Manual mode needs a fixed count; negative values enable fitting.
+ return 2**31 - 1
try:
value = int(raw_value)
except ValueError as exc:
- raise ValueError("llama-server GPU layers flag requires an integer value") from exc
+ raise ValueError(
+ "llama-server GPU layers flag requires an integer value, auto, or all"
+ ) from exc
if value < -1:
raise ValueError("llama-server GPU layers flag requires an integer value of at least -1")
return value
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 9b3dc2f28a1..42ddca40ca3 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -1093,7 +1093,7 @@ class ModelOverridePayload(BaseModel):
chat_template_override: Optional[str] = None
gpu_memory_mode: Optional[Literal["auto", "manual"]] = None
# -1 is Auto (llama.cpp --fit sizes the offload); the normalizer treats it as unset.
- gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024)
+ gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 2**31 - 1)
n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024)
tensor_split: Optional[list[float]] = Field(default = None, min_length = 2, max_length = MAX_GPU_IDS)
gpu_ids: Optional[list[int]] = Field(default = None, max_length = MAX_GPU_IDS)
diff --git a/studio/backend/tests/test_llama_extra_args_compatibility.py b/studio/backend/tests/test_llama_extra_args_compatibility.py
index 30fa4b0773f..1a51d541439 100644
--- a/studio/backend/tests/test_llama_extra_args_compatibility.py
+++ b/studio/backend/tests/test_llama_extra_args_compatibility.py
@@ -221,6 +221,32 @@ def test_the_override_save_carries_over_without_refusing(monkeypatch):
assert "--numa" in kept
[email protected]("flag", ["-ngl", "--gpu-layers", "--n-gpu-layers"])
[email protected]("value", ["auto", "all"])
+def test_override_save_and_reload_keep_gpu_layer_keywords(monkeypatch, flag, value):
+ import routes.settings as settings_route
+ import utils.openai_auto_switch_settings as settings
+
+ saved = {}
+ monkeypatch.setattr(settings, "get_model_override", lambda _id: {})
+ monkeypatch.setattr(
+ settings, "set_model_override", lambda model_id, **kwargs: saved.update({model_id: kwargs})
+ )
+ monkeypatch.setattr(settings_route, "set_model_override", settings.set_model_override)
+ monkeypatch.setattr(settings_route, "resolve_model_override_keys", lambda _id: ["local/x"])
+ monkeypatch.setattr(settings_route, "cached_repo_alias_keys", lambda _id: [])
+ args = [flag, value]
+ payload = settings_route.ModelOverridePayload(model_id = "local/x", llama_extra_args = args)
+
+ settings_route.update_openai_auto_switch_override(payload, current_subject = "t")
+
+ assert saved["local/x"]["llama_extra_args"] == args
+ assert (
+ settings.model_override_load_kwargs(saved["local/x"], is_gguf = True)["llama_extra_args"]
+ == args
+ )
+
+
def test_the_auto_switch_path_sanitizes_a_legacy_override(monkeypatch):
from utils.openai_auto_switch_settings import model_override_load_kwargs
diff --git a/studio/backend/tests/test_llama_extra_args_end_to_end.py b/studio/backend/tests/test_llama_extra_args_end_to_end.py
index 80566c3d677..0d9cb3975d8 100644
--- a/studio/backend/tests/test_llama_extra_args_end_to_end.py
+++ b/studio/backend/tests/test_llama_extra_args_end_to_end.py
@@ -51,6 +51,31 @@ def test_an_extra_arg_reaches_the_child(tmp_path):
assert cmd[cmd.index("--top-k") + 1] == "20"
[email protected]("value", ["auto", "all"])
+def test_gpu_layer_keywords_reach_the_auto_launcher_verbatim(tmp_path, value):
+ args = ["-ngl", value]
+ cmd = _cmd(tmp_path, extra_args = args)
+
+ assert cmd[-len(args) :] == args
+
+
[email protected]("value, layers", [("auto", -1), ("all", 2147483647)])
+def test_promoted_gpu_layer_keywords_keep_manual_fit_intent(tmp_path, value, layers):
+ from core.inference.llama_server_args import parse_gpu_layers_override
+ cmd = _cmd(
+ tmp_path,
+ gpu_memory_mode = "manual",
+ gpu_layers = parse_gpu_layers_override(["-ngl", value]),
+ )
+
+ if layers < 0:
+ assert "--gpu-layers" not in cmd and "-ngl" not in cmd
+ assert cmd[cmd.index("--fit") + 1] == "on"
+ else:
+ assert cmd[cmd.index("--gpu-layers") + 1] == str(layers)
+ assert cmd[cmd.index("--fit") + 1] == "off"
+
+
def test_extra_args_come_after_unsloths_own(tmp_path):
cmd = _cmd(tmp_path, extra_args = ["--top-k", "20"])
managed = [i for i, token in enumerate(cmd) if token in {"--model", "-m", "--port"}]
diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py
index c5ed950257b..a3bf4865bc4 100644
--- a/studio/backend/tests/test_llama_server_args.py
+++ b/studio/backend/tests/test_llama_server_args.py
@@ -486,6 +486,28 @@ def test_parse_gpu_layers_override(args, expected):
assert parse_gpu_layers_override(args) == expected
[email protected]("flag", ["-ngl", "--gpu-layers", "--n-gpu-layers"])
[email protected]("value,expected", [("auto", -1), ("all", 2**31 - 1)])
+def test_gpu_layer_keywords_are_preserved_and_resolved(flag, value, expected):
+ args = [flag, value]
+ assert validate_extra_args(args) == args
+ assert parse_gpu_layers_override(args) == expected
+
+
+ "args,expected",
+ [
+ (["-ngl", "all", "--gpu-layers", "auto"], -1),
+ (["-ngl", "auto", "--n-gpu-layers", "all"], 2**31 - 1),
+ (["-ngl", "all", "--gpu-layers", "0"], 0),
+ (["-ngl", "12", "--gpu-layers", "all"], 2**31 - 1),
+ ],
+)
+def test_gpu_layer_keywords_respect_last_override(args, expected):
+ assert validate_extra_args(args) == args
+ assert parse_gpu_layers_override(args) == expected
+
+
@pytest.mark.parametrize(
"args",
[
diff --git a/studio/backend/tests/test_model_override_schema_compatibility.py b/studio/backend/tests/test_model_override_schema_compatibility.py
index 4d48ce669b4..781404e4df0 100644
--- a/studio/backend/tests/test_model_override_schema_compatibility.py
+++ b/studio/backend/tests/test_model_override_schema_compatibility.py
@@ -40,6 +40,13 @@ TUNING_PAYLOAD = dict(
)
+def test_manual_full_offload_count_survives_save_and_reload(override_store):
+ _put(MODEL, gpu_memory_mode = "manual", gpu_layers = 2**31 - 1)
+ saved = settings.get_model_override(MODEL)
+ assert saved["gpu_layers"] == 2**31 - 1
+ assert settings.model_override_load_kwargs(saved, is_gguf = True)["gpu_layers"] == 2**31 - 1
+
+
def test_a_row_written_before_the_tuning_group_still_loads(override_store):
legacy_row = {
"llama_extra_args": ["--numa", "distribute"],
diff --git a/studio/backend/tests/test_validate_diffusion_extra_args.py b/studio/backend/tests/test_validate_diffusion_extra_args.py
index 4e0acdeeb23..6c363c97e3e 100644
--- a/studio/backend/tests/test_validate_diffusion_extra_args.py
+++ b/studio/backend/tests/test_validate_diffusion_extra_args.py
@@ -10,6 +10,8 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
+import pytest
+
from models.inference import ValidateModelRequest
@@ -163,6 +165,7 @@ class TestValidateTranslatesManualNgl(unittest.TestCase):
gpu_layers,
extra_args,
diffusion_kind = True,
+ gpu_memory_mode = "manual",
):
seen: list = []
@@ -172,7 +175,7 @@ class TestValidateTranslatesManualNgl(unittest.TestCase):
request = ValidateModelRequest(
model_path = "someone/diffusion-gguf",
llama_extra_args = extra_args,
- gpu_memory_mode = "manual",
+ gpu_memory_mode = gpu_memory_mode,
gpu_layers = gpu_layers,
)
config = SimpleNamespace(
@@ -248,6 +251,62 @@ class TestValidateTranslatesManualNgl(unittest.TestCase):
self.assertEqual(seen, [(request.gpu_layers, ["-ngl", "20"])])
[email protected]("flag", ["-ngl", "--gpu-layers", "--n-gpu-layers"])
[email protected]("value, layers", [("auto", -1), ("all", 2147483647)])
[email protected]("mode", ["auto", "manual"])
+def test_validate_preserves_gpu_layer_keyword_intent(flag, value, layers, mode):
+ route = _load_route_module("inf_route_ngl_keywords")
+ args = [flag, value]
+ seen = TestValidateTranslatesManualNgl()._validate(
+ route,
+ gpu_layers = 0,
+ extra_args = args,
+ diffusion_kind = False,
+ gpu_memory_mode = mode,
+ )
+ assert seen == ([(layers, [])] if mode == "manual" else [(0, args)])
+
+
[email protected]("value, layers", [("auto", -1), ("all", 2147483647)])
[email protected]("mode", ["auto", "manual"])
+def test_load_normalizes_gpu_layer_keywords_before_model_work(monkeypatch, value, layers, mode):
+ from fastapi import HTTPException
+ from starlette.requests import Request
+ from models.inference import LoadRequest
+ import routes.inference as route
+
+ seen = []
+
+ def capture(request, **kwargs):
+ seen.append((request.gpu_layers, request.llama_extra_args))
+ raise HTTPException(status_code = 418, detail = "stopped before model resolution")
+
+ async def no_ollama_lease(*args, **kwargs):
+ return None
+
+ monkeypatch.setattr(route, "_resolve_model_identifier_for_request", capture)
+ monkeypatch.setattr(route, "_lease_ollama_model_ref", no_ollama_lease)
+ monkeypatch.setattr(route, "_as_ollama_manifest_request", lambda request: request)
+ monkeypatch.setattr(route, "_as_local_scan_folder_request", lambda request, owner: request)
+ monkeypatch.setattr(route, "_repo_is_in_the_hub_cache", lambda model: None)
+ monkeypatch.setattr(route, "_hub_cache_footprint", lambda model: None)
+ monkeypatch.setattr(route, "_lora_base_already_in_the_hub_cache", lambda model: None)
+ monkeypatch.setattr(route, "_note_lora_base_fetched_with_a_request_token", lambda *a, **k: None)
+ args = ["-ngl", value]
+ request = LoadRequest(
+ model_path = "placeholder.gguf",
+ gpu_memory_mode = mode,
+ gpu_layers = 0,
+ llama_extra_args = args,
+ )
+
+ with pytest.raises(HTTPException) as caught:
+ asyncio.run(route._load_model_impl(request, Request({"type": "http", "headers": []}), "t"))
+
+ assert caught.value.status_code == 418
+ assert seen == ([(layers, [])] if mode == "manual" else [(0, args)])
+
+
class TestValidateTranslatesManualTensorSplit(unittest.TestCase):
"""/validate must promote -ts to tensor_split as /load does, or it judges a different split."""
diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py
index 4dbc5699f70..77ec325e8ab 100644
--- a/studio/backend/utils/openai_auto_switch_settings.py
+++ b/studio/backend/utils/openai_auto_switch_settings.py
@@ -583,7 +583,7 @@ def normalize_model_override(
if _clean_str(payload.get("gpu_memory_mode"), VALID_GPU_MEMORY_MODES) == "manual":
entry["gpu_memory_mode"] = "manual"
- gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 1024)
+ gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 2**31 - 1)
if gpu_layers is not None:
entry["gpu_layers"] = gpu_layers
diff --git a/studio/frontend/src/features/chat/lib/llama-extra-args-normalize.ts b/studio/frontend/src/features/chat/lib/llama-extra-args-normalize.ts
index b3663595bda..11bb6360264 100644
--- a/studio/frontend/src/features/chat/lib/llama-extra-args-normalize.ts
+++ b/studio/frontend/src/features/chat/lib/llama-extra-args-normalize.ts
@@ -78,6 +78,12 @@ export function parseGpuLayersOverride(
if (raw === null) {
return { kind: "absent" };
}
+ if (raw === "auto") {
+ return { kind: "value", layers: -1 };
+ }
+ if (raw === "all") {
+ return { kind: "value", layers: 2147483647 };
+ }
const value = Number.parseInt(raw, 10);
return Number.isInteger(value) && value >= -1 && String(value) === raw.trim()
? { kind: "value", layers: value }
diff --git a/studio/frontend/src/features/model-picker/model-config/llama-extra-args.ts b/studio/frontend/src/features/model-picker/model-config/llama-extra-args.ts
index 2999151eaff..bc38ba2a8de 100644
--- a/studio/frontend/src/features/model-picker/model-config/llama-extra-args.ts
+++ b/studio/frontend/src/features/model-picker/model-config/llama-extra-args.ts
@@ -218,6 +218,7 @@ function dropUnusableValues(tokens: readonly string[]): string[] {
(RATIO_VALUE_FLAGS.has(flag) &&
ratioValueProblem(flag, value, false) !== null) ||
(INTEGER_VALUE_FLAGS.has(flag) &&
+ !isGpuLayersKeyword(flag, value) &&
(!INTEGER.test(value.trim()) ||
(minimum !== undefined && Number(value.trim()) < minimum)));
if (!unusable) {
@@ -587,20 +588,24 @@ const VALUE_REQUIRED_FLAGS = new Set([
const GPU_LAYERS_FLAGS = new Set(["--gpu-layers", "--n-gpu-layers", "-ngl"]);
-/** Mirrors parse_gpu_layers_override only for: is the resolved count non-negative. */
-function lastIntegerFlagValue(
- tokens: readonly string[],
- flags: ReadonlySet<string>,
-): number | null {
+function isGpuLayersKeyword(flag: string, value: string | undefined): boolean {
+ return GPU_LAYERS_FLAGS.has(flag) && (value === "auto" || value === "all");
+}
+
+function lastGpuLayersValue(tokens: readonly string[]): number | null {
let found: number | null = null;
for (const [index, token] of tokens.entries()) {
const flag = extraArgFlagName(token);
- if (flag === null || !flags.has(flag)) {
+ if (flag === null || !GPU_LAYERS_FLAGS.has(flag)) {
continue;
}
const attached = valueIsAttached(token, flag);
const raw = attached ? token.split("=")[1] : tokens[index + 1];
- if (raw !== undefined && INTEGER.test(raw.trim())) {
+ if (raw === "auto") {
+ found = -1;
+ } else if (raw === "all") {
+ found = 2147483647;
+ } else if (raw !== undefined && INTEGER.test(raw.trim())) {
found = Number(raw.trim());
}
}
@@ -738,7 +743,7 @@ export function diagnoseExtraArgs(
const out: ExtraArgsDiagnostic[] = [];
const { tokens, unterminatedQuote, quotedIndices } = parseExtraArgs(input);
// Mirrors _should_strip_tensor_split: an extras -ngl is promoted first.
- const nglOverride = lastIntegerFlagValue(tokens, GPU_LAYERS_FLAGS);
+ const nglOverride = lastGpuLayersValue(tokens);
const resolvedGpuLayers = nglOverride ?? context.gpuLayers ?? -1;
const reserializesSplit = manualGpuMemory && resolvedGpuLayers >= 0;
// A token in value position is a value whatever it starts with; llama.cpp takes the next argv blindly.
@@ -883,7 +888,7 @@ export function diagnoseExtraArgs(
: `${flag} needs a value after it.`;
} else if (RATIO_VALUE_FLAGS.has(flag)) {
message = ratioValueProblem(flag, value, reserializesSplit);
- } else if (!numeric) {
+ } else if (!numeric || isGpuLayersKeyword(flag, value)) {
message = null;
} else if (!INTEGER.test(value.trim())) {
message = `${flag} takes a number, and "${value}" is not one.`;
diff --git a/studio/frontend/src/features/model-picker/sharing/extra-args.ts b/studio/frontend/src/features/model-picker/sharing/extra-args.ts
index 624fa9d372b..1f458279060 100644
--- a/studio/frontend/src/features/model-picker/sharing/extra-args.ts
+++ b/studio/frontend/src/features/model-picker/sharing/extra-args.ts
@@ -40,7 +40,8 @@ const threads = number(-1, 1024, true);
const context: ValueRule = (value) =>
value === "0" || number(CONTEXT_LENGTH_MIN, 2_147_483_647)(value);
const batch: ValueRule = (value) => number(N_BATCH_MIN, N_BATCH_MAX)(value);
-const layers = number(-1, 2_147_483_647);
+const layers: ValueRule = (value) =>
+ value === "auto" || value === "all" || number(-1, 2_147_483_647)(value);
const cache: ValueRule = (value) =>
value === "f16" || KV_CACHE_DTYPES.some((dtype) => dtype === value);
const ratio = number(0, 1);
diff --git a/studio/frontend/tests/llama-extra-args-diagnostics.test.ts b/studio/frontend/tests/llama-extra-args-diagnostics.test.ts
index d7bd9db560e..35be89ca426 100644
--- a/studio/frontend/tests/llama-extra-args-diagnostics.test.ts
+++ b/studio/frontend/tests/llama-extra-args-diagnostics.test.ts
@@ -224,6 +224,40 @@ test("a flag that takes a number rejects a value that is not one", () => {
);
});
+test("GPU layer keywords load and survive stored-argument sanitizing", () => {
+ for (const flag of ["-ngl", "--gpu-layers", "--n-gpu-layers"]) {
+ for (const value of ["auto", "all"]) {
+ const tokens = [flag, value, "--top-k", "20"];
+ for (const manualGpuMemory of [false, true]) {
+ const diagnostics = diagnoseExtraArgs(tokens.join(" "), CATALOG, {
+ manualGpuMemory,
+ });
+ assert.equal(extraArgsAreLoadable(diagnostics), true, JSON.stringify(diagnostics));
+ }
+ assert.deepEqual(sanitizeStoredExtraArgs(tokens, CATALOG.managed), tokens);
+ }
+ for (const value of ["many", "AUTO", "ALL", "-2"]) {
+ const tokens = [flag, value];
+ assert.equal(extraArgsAreLoadable(diagnoseExtraArgs(tokens.join(" "), CATALOG)), false);
+ assert.deepEqual(sanitizeStoredExtraArgs(tokens, CATALOG.managed), []);
+ }
+ }
+ assert.equal(extraArgsAreLoadable(diagnoseExtraArgs("--batch-size all", CATALOG)), false);
+});
+
+test("GPU layer keywords preserve the last override's manual ratio behavior", () => {
+ const ratio = "-ts 1.1754943508222874e-38,1";
+ const error = (layers: string, gpuLayers: number) =>
+ diagnoseExtraArgs(`${layers} ${ratio}`, CATALOG, {
+ manualGpuMemory: true,
+ gpuLayers,
+ }).find((d) => d.level === "error")?.message ?? null;
+ assert.equal(error("-ngl auto", 49), null);
+ assert.equal(error("-ngl all --gpu-layers auto", 49), null);
+ assert.match(error("-ngl all", -1) ?? "", /0 or at least/);
+ assert.match(error("-ngl auto --gpu-layers all", -1) ?? "", /0 or at least/);
+});
+
test("a control character is an error, as it is at the backend", () => {
const diagnostics = diagnoseExtraArgs("--grammar a\u001b[0mb", CATALOG);
assert.equal(diagnostics[0].level, "error");
diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.ts
index c8f066dd6a9..990f3b0eef4 100644
--- a/studio/frontend/tests/resident-config-match.test.ts
+++ b/studio/frontend/tests/resident-config-match.test.ts
@@ -885,6 +885,17 @@ test("the arch-gate excuse reads the resolved split, not the toggle", () => {
);
});
+test("manual GPU layer keywords adopt a matching resident model", () => {
+ for (const flag of ["-ngl", "--gpu-layers", "--n-gpu-layers"]) {
+ for (const [value, layers] of [["auto", -1], ["all", 2147483647]] as const) {
+ assert.equal(matches(
+ { ...DEFAULTS, is_gguf: true, gpu_memory_mode: "manual", gpu_layers: layers },
+ { ...BLANK, gpuMemoryMode: "manual", llamaExtraArgs: [flag, value] },
+ ), true, `${flag} ${value}`);
+ }
+ }
+});
+
test("a malformed manual layer override declines rather than normalizing away", () => {
// parse_gpu_layers_override raises on these, so they must not fold into no override.
const running = {
diff --git a/studio/frontend/tests/share-run-configs.security.test.ts b/studio/frontend/tests/share-run-configs.security.test.ts
index cf880db8903..742fbc8bd23 100644
--- a/studio/frontend/tests/share-run-configs.security.test.ts
+++ b/studio/frontend/tests/share-run-configs.security.test.ts
@@ -47,6 +47,14 @@ const params = (key: string, value: string) =>
const args = (list: unknown) => query("llamaExtraArgs", list);
const valid = (config: object) => ({ kind: "valid", value: { config } });
+test("GPU layer keywords round-trip through shared settings", () => {
+ for (const flag of ["-ngl", "--gpu-layers", "--n-gpu-layers"]) {
+ for (const value of ["auto", "all"]) {
+ roundTrips({ config: { llamaExtraArgs: [flag, value] } });
+ }
+ }
+});
+
function rejects(query: string) {
for (const prefix of prefixes) {
const result = parseRunConfigLink(`${prefix}${query}`);