Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""/validate drops pass-through args for diffusion GGUFs only after classifying, like /load does."""
import asyncio
import importlib.util
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from models.inference import ValidateModelRequest
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
def _load_route_module(name: str):
spec = importlib.util.spec_from_file_location(name, _BACKEND_ROOT / "routes/inference.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
async def _noop_gpu_ids(_config, gpu_ids, **_kwargs):
return gpu_ids, False
class TestValidateDropsDiffusionExtraArgs(unittest.TestCase):
def _validate(self, route, *, diffusion_kind):
seen: list = []
def _capture(*_args, **kwargs):
seen.append(kwargs.get("llama_extra_args"))
request = ValidateModelRequest(
model_path = "someone/diffusion-gguf",
llama_extra_args = ["--ctx-size", "8192"],
)
config = SimpleNamespace(
identifier = "someone/diffusion-gguf",
display_name = "diffusion-gguf",
is_gguf = True,
is_lora = False,
is_vision = False,
gguf_file = None,
)
with (
patch.object(
route,
"_resolve_model_identifier_for_request",
return_value = ("someone/diffusion-gguf", "someone/diffusion-gguf", False),
),
patch.object(route.ModelConfig, "from_identifier", return_value = config),
patch.object(
route,
"_resolve_inherited_extra_args",
return_value = ["--ctx-size", "8192"],
),
patch.object(route, "_classify_diffusion_gguf", return_value = diffusion_kind),
patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
patch.object(route, "_effective_load_in_4bit", return_value = True),
patch.object(route, "_guard_chat_load_against_training", new = _capture),
):
asyncio.run(route.validate_model(request, current_subject = "test-user"))
return seen
def test_a_diffusion_gguf_is_estimated_without_them(self):
route = _load_route_module("inf_route_diffusion_extra_args_1")
self.assertEqual(self._validate(route, diffusion_kind = True), [[]])
def test_an_ordinary_gguf_still_estimates_with_them(self):
# The drop is narrow: a --ctx-size on an ordinary GGUF must reach the load estimate.
route = _load_route_module("inf_route_diffusion_extra_args_2")
self.assertEqual(
self._validate(route, diffusion_kind = False),
[["--ctx-size", "8192"]],
)
def test_an_inconclusive_gguf_keeps_them(self):
# None is "nothing to read yet", not "diffusion". Dropping on it would strip a
# working override from an ordinary model whose header has not arrived.
route = _load_route_module("inf_route_diffusion_extra_args_3")
self.assertEqual(
self._validate(route, diffusion_kind = None),
[["--ctx-size", "8192"]],
)
class TestValidateJudgesTheListBeforeRewritingIt(unittest.TestCase):
"""Validate the extras before stripping -ngl from them, or a spelling /load refuses gets approved."""
def _validate(
self,
route,
*,
extra_args,
manual = True,
):
request = ValidateModelRequest(
model_path = "someone/gguf",
llama_extra_args = extra_args,
**({"gpu_memory_mode": "manual", "gpu_layers": 0} if manual else {}),
)
config = SimpleNamespace(
identifier = "someone/gguf",
display_name = "gguf",
is_gguf = True,
is_lora = False,
is_vision = False,
gguf_file = None,
)
with (
patch.object(
route,
"_resolve_model_identifier_for_request",
return_value = ("someone/gguf", "someone/gguf", False),
),
patch.object(route.ModelConfig, "from_identifier", return_value = config),
patch.object(route, "_resolve_inherited_extra_args", return_value = list(extra_args)),
patch.object(route, "_classify_diffusion_gguf", return_value = False),
patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
patch.object(route, "_effective_load_in_4bit", return_value = True),
patch.object(route, "_guard_chat_load_against_training", new = lambda *a, **k: None),
):
return asyncio.run(route.validate_model(request, current_subject = "test-user"))
def test_an_attached_offload_spelling_is_refused_not_translated(self):
# llama.cpp looks up the whole token, so "--gpu-layers=20" is unknown to it.
from fastapi import HTTPException
route = _load_route_module("inf_route_validate_order_1")
with self.assertRaises(HTTPException) as caught:
self._validate(route, extra_args = ["--gpu-layers=20"])
self.assertEqual(caught.exception.status_code, 400)
self.assertIn("two separate arguments", str(caught.exception.detail))
def test_a_malformed_layer_count_is_a_refusal_not_a_crash(self):
# parse_gpu_layers_override raises on non-integers, so it must sit inside the 400 try.
from fastapi import HTTPException
route = _load_route_module("inf_route_validate_order_2")
with self.assertRaises(HTTPException) as caught:
self._validate(route, extra_args = ["-ngl", "bad"])
self.assertEqual(caught.exception.status_code, 400)
def test_a_well_formed_list_still_passes(self):
route = _load_route_module("inf_route_validate_order_3")
response = self._validate(route, extra_args = ["-ngl", "20"])
self.assertTrue(getattr(response, "valid", True))
class TestValidateTranslatesManualNgl(unittest.TestCase):
"""/validate must promote -ngl to gpu_layers as /load does, or it approves a different load."""
def _validate(
self,
route,
*,
gpu_layers,
extra_args,
diffusion_kind = True,
gpu_memory_mode = "manual",
):
seen: list = []
def _capture(_config, request, **kwargs):
seen.append((request.gpu_layers, kwargs.get("llama_extra_args")))
request = ValidateModelRequest(
model_path = "someone/diffusion-gguf",
llama_extra_args = extra_args,
gpu_memory_mode = gpu_memory_mode,
gpu_layers = gpu_layers,
)
config = SimpleNamespace(
identifier = "someone/diffusion-gguf",
display_name = "diffusion-gguf",
is_gguf = True,
is_lora = False,
is_vision = False,
gguf_file = None,
)
with (
patch.object(
route,
"_resolve_model_identifier_for_request",
return_value = ("someone/diffusion-gguf", "someone/diffusion-gguf", False),
),
patch.object(route.ModelConfig, "from_identifier", return_value = config),
patch.object(route, "_resolve_inherited_extra_args", return_value = list(extra_args)),
patch.object(route, "_classify_diffusion_gguf", return_value = diffusion_kind),
patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
patch.object(route, "_effective_load_in_4bit", return_value = True),
patch.object(route, "_guard_chat_load_against_training", new = _capture),
):
asyncio.run(route.validate_model(request, current_subject = "test-user"))
return seen
def test_an_explicit_layer_count_reaches_the_guard(self):
route = _load_route_module("inf_route_manual_ngl_1")
seen = self._validate(route, gpu_layers = 0, extra_args = ["-ngl", "20"])
self.assertEqual(seen, [(20, [])])
def test_a_zero_layer_override_is_read_the_same_way(self):
# A 0-layer override is judged as a CPU-only load, not refused for VRAM it never uses.
route = _load_route_module("inf_route_manual_ngl_2")
seen = self._validate(route, gpu_layers = 20, extra_args = ["-ngl", "0"])
self.assertEqual(seen, [(0, [])])
def test_auto_mode_leaves_the_flag_alone(self):
# Only manual mode owns these; in Auto the flag passes through to the loader.
route = _load_route_module("inf_route_manual_ngl_3")
seen: list = []
def _capture(_config, request, **kwargs):
seen.append((request.gpu_layers, kwargs.get("llama_extra_args")))
request = ValidateModelRequest(
model_path = "someone/gguf",
llama_extra_args = ["-ngl", "20"],
)
config = SimpleNamespace(
identifier = "someone/gguf",
display_name = "gguf",
is_gguf = True,
is_lora = False,
is_vision = False,
gguf_file = None,
)
with (
patch.object(
route,
"_resolve_model_identifier_for_request",
return_value = ("someone/gguf", "someone/gguf", False),
),
patch.object(route.ModelConfig, "from_identifier", return_value = config),
patch.object(route, "_resolve_inherited_extra_args", return_value = ["-ngl", "20"]),
patch.object(route, "_classify_diffusion_gguf", return_value = False),
patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
patch.object(route, "_effective_load_in_4bit", return_value = True),
patch.object(route, "_guard_chat_load_against_training", new = _capture),
):
asyncio.run(route.validate_model(request, current_subject = "test-user"))
self.assertEqual(seen, [(request.gpu_layers, ["-ngl", "20"])])
@pytest.mark.parametrize("flag", ["-ngl", "--gpu-layers", "--n-gpu-layers"])
@pytest.mark.parametrize("value, layers", [("auto", -1), ("all", 2147483647)])
@pytest.mark.parametrize("mode", ["auto", "manual"])
def test_validate_preserves_gpu_layer_keyword_intent(flag, value, layers, mode):
route = _load_route_module("inf_route_ngl_keywords")
args = [flag, value]
seen = TestValidateTranslatesManualNgl()._validate(
route,
gpu_layers = 0,
extra_args = args,
diffusion_kind = False,
gpu_memory_mode = mode,
)
assert seen == ([(layers, [])] if mode == "manual" else [(0, args)])
@pytest.mark.parametrize("value, layers", [("auto", -1), ("all", 2147483647)])
@pytest.mark.parametrize("mode", ["auto", "manual"])
def test_load_normalizes_gpu_layer_keywords_before_model_work(monkeypatch, value, layers, mode):
from fastapi import HTTPException
from starlette.requests import Request
from models.inference import LoadRequest
import routes.inference as route
seen = []
def capture(request, **kwargs):
seen.append((request.gpu_layers, request.llama_extra_args))
raise HTTPException(status_code = 418, detail = "stopped before model resolution")
async def no_ollama_lease(*args, **kwargs):
return None
monkeypatch.setattr(route, "_resolve_model_identifier_for_request", capture)
monkeypatch.setattr(route, "_lease_ollama_model_ref", no_ollama_lease)
monkeypatch.setattr(route, "_as_ollama_manifest_request", lambda request: request)
monkeypatch.setattr(route, "_as_local_scan_folder_request", lambda request, owner: request)
monkeypatch.setattr(route, "_repo_is_in_the_hub_cache", lambda model: None)
monkeypatch.setattr(route, "_hub_cache_footprint", lambda model: None)
monkeypatch.setattr(route, "_lora_base_already_in_the_hub_cache", lambda model: None)
monkeypatch.setattr(route, "_note_lora_base_fetched_with_a_request_token", lambda *a, **k: None)
args = ["-ngl", value]
request = LoadRequest(
model_path = "placeholder.gguf",
gpu_memory_mode = mode,
gpu_layers = 0,
llama_extra_args = args,
)
with pytest.raises(HTTPException) as caught:
asyncio.run(route._load_model_impl(request, Request({"type": "http", "headers": []}), "t"))
assert caught.value.status_code == 418
assert seen == ([(layers, [])] if mode == "manual" else [(0, args)])
class TestValidateTranslatesManualTensorSplit(unittest.TestCase):
"""/validate must promote -ts to tensor_split as /load does, or it judges a different split."""
def _validate(
self,
route,
*,
gpu_layers,
extra_args,
tensor_split = None,
):
seen: list = []
def _capture(_config, request, **kwargs):
seen.append(
(
request.gpu_layers,
request.tensor_split,
kwargs.get("llama_extra_args"),
)
)
request = ValidateModelRequest(
model_path = "someone/moe-gguf",
llama_extra_args = extra_args,
gpu_memory_mode = "manual",
gpu_layers = gpu_layers,
tensor_split = tensor_split,
)
config = SimpleNamespace(
identifier = "someone/moe-gguf",
display_name = "moe-gguf",
is_gguf = True,
is_lora = False,
is_vision = False,
gguf_file = None,
)
with (
patch.object(
route,
"_resolve_model_identifier_for_request",
return_value = ("someone/moe-gguf", "someone/moe-gguf", False),
),
patch.object(route.ModelConfig, "from_identifier", return_value = config),
patch.object(route, "_resolve_inherited_extra_args", return_value = list(extra_args)),
patch.object(route, "_classify_diffusion_gguf", return_value = False),
patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
patch.object(route, "_effective_load_in_4bit", return_value = True),
patch.object(route, "_guard_chat_load_against_training", new = _capture),
):
asyncio.run(route.validate_model(request, current_subject = "test-user"))
return seen
def test_an_explicit_tensor_split_reaches_the_guard(self):
route = _load_route_module("inf_route_manual_ts_1")
seen = self._validate(
route,
gpu_layers = 49,
extra_args = ["-ts", "2.2,1", "-sm", "layer"],
)
self.assertEqual(seen, [(49, [2.2, 1.0], ["-sm", "layer"])])
def test_extras_override_a_first_class_tensor_split(self):
route = _load_route_module("inf_route_manual_ts_2")
seen = self._validate(
route,
gpu_layers = 49,
tensor_split = [1, 1],
extra_args = ["--tensor-split", "3,1"],
)
self.assertEqual(seen, [(49, [3.0, 1.0], [])])
if __name__ == "__main__":
unittest.main()
class TestValidateRefusesWhatLoadWouldRefuse(unittest.TestCase):
"""The picker unloads the current model once /validate approves, so /validate must refuse what
/load would: a refusal keeps the model, a failed switch does not."""
def _validate(
self,
route,
*,
extra_args,
n_parallel = None,
diffusion_kind = False,
):
request = ValidateModelRequest(
model_path = "someone/gguf",
llama_extra_args = extra_args,
n_parallel = n_parallel,
)
config = SimpleNamespace(
identifier = "someone/gguf",
display_name = "gguf",
is_gguf = True,
is_lora = False,
is_vision = False,
gguf_file = None,
)
with (
patch.object(
route,
"_resolve_model_identifier_for_request",
return_value = ("someone/gguf", "someone/gguf", False),
),
patch.object(route.ModelConfig, "from_identifier", return_value = config),
patch.object(route, "_resolve_inherited_extra_args", return_value = extra_args),
patch.object(route, "_classify_diffusion_gguf", return_value = diffusion_kind),
patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
patch.object(route, "_effective_load_in_4bit", return_value = True),
patch.object(route, "_effective_parallel_slots", side_effect = lambda n, **_: n),
patch.object(route, "_guard_chat_load_against_training", new = lambda *a, **k: None),
):
return asyncio.run(route.validate_model(request, current_subject = "test-user"))
def test_a_denied_flag_is_refused_before_the_switch(self):
route = _load_route_module("inf_route_validate_denies_1")
with self.assertRaises(Exception) as caught:
self._validate(route, extra_args = ["--agent"])
self.assertEqual(getattr(caught.exception, "status_code", None), 400)
self.assertIn("managed by Unsloth Studio", str(caught.exception.detail))
def test_a_batch_below_the_slot_floor_is_refused_before_the_switch(self):
route = _load_route_module("inf_route_validate_denies_2")
with self.assertRaises(Exception) as caught:
self._validate(route, extra_args = ["-b", "2"], n_parallel = 4)
self.assertEqual(getattr(caught.exception, "status_code", None), 400)
self.assertIn("aborts on --batch-size", str(caught.exception.detail))
def test_a_list_the_load_would_accept_still_passes(self):
route = _load_route_module("inf_route_validate_denies_3")
resp = self._validate(route, extra_args = ["--numa", "distribute"], n_parallel = 4)
self.assertTrue(resp.is_gguf)
class TestEmbeddingSlotClampInTheBatchFloor(unittest.TestCase):
"""--embedding caps the batch at the micro-batch, which llama-server aborts below the slot
count; load_model clamps slots first, so the floor must use the clamped count."""
def _clamped(
self,
route,
*,
is_embedding,
extra_args,
slots = 4,
**kwargs,
):
config = SimpleNamespace(identifier = "someone/embed-gguf", gguf_file = None)
with patch.object(route, "_is_embedding_gguf", return_value = is_embedding):
return route._embedding_clamped_slots(
config,
slots,
extra_args = extra_args,
n_batch = kwargs.get("n_batch"),
n_ubatch = kwargs.get("n_ubatch"),
n_ctx = kwargs.get("n_ctx"),
)
def test_the_slots_follow_the_micro_batch_down(self):
route = _load_route_module("inf_route_embed_clamp_1")
self.assertEqual(
self._clamped(route, is_embedding = True, extra_args = ["-b", "2", "-ub", "2"]),
2,
)
def test_a_chat_gguf_keeps_the_slots_it_asked_for(self):
route = _load_route_module("inf_route_embed_clamp_2")
self.assertEqual(
self._clamped(route, is_embedding = False, extra_args = ["-b", "2", "-ub", "2"]),
4,
)
def test_defaults_clamp_nothing(self):
route = _load_route_module("inf_route_embed_clamp_3")
self.assertEqual(
self._clamped(route, is_embedding = True, extra_args = ["--numa", "distribute"]),
4,
)
def test_the_clamp_floors_at_one_slot(self):
# -b 0 gives a zero micro-batch; --parallel 0 fails arg parse; the floor is one slot.
route = _load_route_module("inf_route_embed_clamp_4")
self.assertEqual(
self._clamped(route, is_embedding = True, extra_args = ["-b", "0", "-ub", "0"]),
1,
)
def test_an_unreadable_header_leaves_the_refusal_alone(self):
# _is_embedding_gguf answers False for a GGUF that is not on this disk yet, so
# nothing is relaxed on a guess.
route = _load_route_module("inf_route_embed_clamp_5")
config = SimpleNamespace(identifier = "someone/gguf", gguf_file = None, gguf_hf_repo = None)
self.assertFalse(route._is_embedding_gguf(config))
def test_an_uncached_embedding_identifier_stays_fail_closed(self):
route = _load_route_module("inf_route_embed_clamp_6")
config = SimpleNamespace(
identifier = "Qwen/Qwen3-Embedding-4B-GGUF",
gguf_file = None,
gguf_hf_repo = "Qwen/Qwen3-Embedding-4B-GGUF",
gguf_variant = "Q4_K_M",
)
with patch.object(route, "_local_gguf_main_path", return_value = None):
self.assertFalse(route._is_embedding_gguf(config))
class TestValidateAllowsTheEmbeddingClampedBatch(TestValidateRefusesWhatLoadWouldRefuse):
"""Preflight must match load exactly, or the picker refuses switches load would complete."""
def test_an_embedding_gguf_may_batch_at_its_micro_batch(self):
route = _load_route_module("inf_route_validate_embed_1")
with patch.object(route, "_is_embedding_gguf", return_value = True):
resp = self._validate(
route,
extra_args = ["-b", "2", "-ub", "2"],
n_parallel = 4,
)
self.assertTrue(resp.is_gguf)
def test_a_chat_gguf_is_still_refused(self):
route = _load_route_module("inf_route_validate_embed_2")
with (
patch.object(route, "_is_embedding_gguf", return_value = False),
self.assertRaises(Exception) as caught,
):
self._validate(route, extra_args = ["-b", "2", "-ub", "2"], n_parallel = 4)
self.assertEqual(getattr(caught.exception, "status_code", None), 400)
self.assertIn("aborts on --batch-size", str(caught.exception.detail))
def test_an_embedding_gguf_below_its_own_floor_is_still_refused(self):
# llama-server aborts on a batch of 1 at any slot count.
route = _load_route_module("inf_route_validate_embed_3")
with (
patch.object(route, "_is_embedding_gguf", return_value = True),
self.assertRaises(Exception) as caught,
):
self._validate(route, extra_args = ["-b", "1", "-ub", "1"], n_parallel = 4)
self.assertEqual(getattr(caught.exception, "status_code", None), 400)