Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

22.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""/validate drops pass-through args for diffusion GGUFs only after classifying, like /load does."""

import asyncio
import importlib.util
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch

import pytest

from models.inference import ValidateModelRequest


_BACKEND_ROOT = Path(__file__).resolve().parent.parent


def _load_route_module(name: str):
    spec = importlib.util.spec_from_file_location(name, _BACKEND_ROOT / "routes/inference.py")
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


async def _noop_gpu_ids(_config, gpu_ids, **_kwargs):
    return gpu_ids, False


class TestValidateDropsDiffusionExtraArgs(unittest.TestCase):
    def _validate(self, route, *, diffusion_kind):
        seen: list = []

        def _capture(*_args, **kwargs):
            seen.append(kwargs.get("llama_extra_args"))

        request = ValidateModelRequest(
            model_path = "someone/diffusion-gguf",
            llama_extra_args = ["--ctx-size", "8192"],
        )
        config = SimpleNamespace(
            identifier = "someone/diffusion-gguf",
            display_name = "diffusion-gguf",
            is_gguf = True,
            is_lora = False,
            is_vision = False,
            gguf_file = None,
        )
        with (
            patch.object(
                route,
                "_resolve_model_identifier_for_request",
                return_value = ("someone/diffusion-gguf", "someone/diffusion-gguf", False),
            ),
            patch.object(route.ModelConfig, "from_identifier", return_value = config),
            patch.object(
                route,
                "_resolve_inherited_extra_args",
                return_value = ["--ctx-size", "8192"],
            ),
            patch.object(route, "_classify_diffusion_gguf", return_value = diffusion_kind),
            patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
            patch.object(route, "_effective_load_in_4bit", return_value = True),
            patch.object(route, "_guard_chat_load_against_training", new = _capture),
        ):
            asyncio.run(route.validate_model(request, current_subject = "test-user"))
        return seen

    def test_a_diffusion_gguf_is_estimated_without_them(self):
        route = _load_route_module("inf_route_diffusion_extra_args_1")
        self.assertEqual(self._validate(route, diffusion_kind = True), [[]])

    def test_an_ordinary_gguf_still_estimates_with_them(self):
        # The drop is narrow: a --ctx-size on an ordinary GGUF must reach the load estimate.
        route = _load_route_module("inf_route_diffusion_extra_args_2")
        self.assertEqual(
            self._validate(route, diffusion_kind = False),
            [["--ctx-size", "8192"]],
        )

    def test_an_inconclusive_gguf_keeps_them(self):
        # None is "nothing to read yet", not "diffusion". Dropping on it would strip a
        # working override from an ordinary model whose header has not arrived.
        route = _load_route_module("inf_route_diffusion_extra_args_3")
        self.assertEqual(
            self._validate(route, diffusion_kind = None),
            [["--ctx-size", "8192"]],
        )


class TestValidateJudgesTheListBeforeRewritingIt(unittest.TestCase):
    """Validate the extras before stripping -ngl from them, or a spelling /load refuses gets approved."""

    def _validate(
        self,
        route,
        *,
        extra_args,
        manual = True,
    ):
        request = ValidateModelRequest(
            model_path = "someone/gguf",
            llama_extra_args = extra_args,
            **({"gpu_memory_mode": "manual", "gpu_layers": 0} if manual else {}),
        )
        config = SimpleNamespace(
            identifier = "someone/gguf",
            display_name = "gguf",
            is_gguf = True,
            is_lora = False,
            is_vision = False,
            gguf_file = None,
        )
        with (
            patch.object(
                route,
                "_resolve_model_identifier_for_request",
                return_value = ("someone/gguf", "someone/gguf", False),
            ),
            patch.object(route.ModelConfig, "from_identifier", return_value = config),
            patch.object(route, "_resolve_inherited_extra_args", return_value = list(extra_args)),
            patch.object(route, "_classify_diffusion_gguf", return_value = False),
            patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
            patch.object(route, "_effective_load_in_4bit", return_value = True),
            patch.object(route, "_guard_chat_load_against_training", new = lambda *a, **k: None),
        ):
            return asyncio.run(route.validate_model(request, current_subject = "test-user"))

    def test_an_attached_offload_spelling_is_refused_not_translated(self):
        # llama.cpp looks up the whole token, so "--gpu-layers=20" is unknown to it.
        from fastapi import HTTPException

        route = _load_route_module("inf_route_validate_order_1")
        with self.assertRaises(HTTPException) as caught:
            self._validate(route, extra_args = ["--gpu-layers=20"])
        self.assertEqual(caught.exception.status_code, 400)
        self.assertIn("two separate arguments", str(caught.exception.detail))

    def test_a_malformed_layer_count_is_a_refusal_not_a_crash(self):
        # parse_gpu_layers_override raises on non-integers, so it must sit inside the 400 try.
        from fastapi import HTTPException

        route = _load_route_module("inf_route_validate_order_2")
        with self.assertRaises(HTTPException) as caught:
            self._validate(route, extra_args = ["-ngl", "bad"])
        self.assertEqual(caught.exception.status_code, 400)

    def test_a_well_formed_list_still_passes(self):
        route = _load_route_module("inf_route_validate_order_3")
        response = self._validate(route, extra_args = ["-ngl", "20"])
        self.assertTrue(getattr(response, "valid", True))


class TestValidateTranslatesManualNgl(unittest.TestCase):
    """/validate must promote -ngl to gpu_layers as /load does, or it approves a different load."""

    def _validate(
        self,
        route,
        *,
        gpu_layers,
        extra_args,
        diffusion_kind = True,
        gpu_memory_mode = "manual",
    ):
        seen: list = []

        def _capture(_config, request, **kwargs):
            seen.append((request.gpu_layers, kwargs.get("llama_extra_args")))

        request = ValidateModelRequest(
            model_path = "someone/diffusion-gguf",
            llama_extra_args = extra_args,
            gpu_memory_mode = gpu_memory_mode,
            gpu_layers = gpu_layers,
        )
        config = SimpleNamespace(
            identifier = "someone/diffusion-gguf",
            display_name = "diffusion-gguf",
            is_gguf = True,
            is_lora = False,
            is_vision = False,
            gguf_file = None,
        )
        with (
            patch.object(
                route,
                "_resolve_model_identifier_for_request",
                return_value = ("someone/diffusion-gguf", "someone/diffusion-gguf", False),
            ),
            patch.object(route.ModelConfig, "from_identifier", return_value = config),
            patch.object(route, "_resolve_inherited_extra_args", return_value = list(extra_args)),
            patch.object(route, "_classify_diffusion_gguf", return_value = diffusion_kind),
            patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
            patch.object(route, "_effective_load_in_4bit", return_value = True),
            patch.object(route, "_guard_chat_load_against_training", new = _capture),
        ):
            asyncio.run(route.validate_model(request, current_subject = "test-user"))
        return seen

    def test_an_explicit_layer_count_reaches_the_guard(self):
        route = _load_route_module("inf_route_manual_ngl_1")
        seen = self._validate(route, gpu_layers = 0, extra_args = ["-ngl", "20"])
        self.assertEqual(seen, [(20, [])])

    def test_a_zero_layer_override_is_read_the_same_way(self):
        # A 0-layer override is judged as a CPU-only load, not refused for VRAM it never uses.
        route = _load_route_module("inf_route_manual_ngl_2")
        seen = self._validate(route, gpu_layers = 20, extra_args = ["-ngl", "0"])
        self.assertEqual(seen, [(0, [])])

    def test_auto_mode_leaves_the_flag_alone(self):
        # Only manual mode owns these; in Auto the flag passes through to the loader.
        route = _load_route_module("inf_route_manual_ngl_3")
        seen: list = []

        def _capture(_config, request, **kwargs):
            seen.append((request.gpu_layers, kwargs.get("llama_extra_args")))

        request = ValidateModelRequest(
            model_path = "someone/gguf",
            llama_extra_args = ["-ngl", "20"],
        )
        config = SimpleNamespace(
            identifier = "someone/gguf",
            display_name = "gguf",
            is_gguf = True,
            is_lora = False,
            is_vision = False,
            gguf_file = None,
        )
        with (
            patch.object(
                route,
                "_resolve_model_identifier_for_request",
                return_value = ("someone/gguf", "someone/gguf", False),
            ),
            patch.object(route.ModelConfig, "from_identifier", return_value = config),
            patch.object(route, "_resolve_inherited_extra_args", return_value = ["-ngl", "20"]),
            patch.object(route, "_classify_diffusion_gguf", return_value = False),
            patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
            patch.object(route, "_effective_load_in_4bit", return_value = True),
            patch.object(route, "_guard_chat_load_against_training", new = _capture),
        ):
            asyncio.run(route.validate_model(request, current_subject = "test-user"))
        self.assertEqual(seen, [(request.gpu_layers, ["-ngl", "20"])])


@pytest.mark.parametrize("flag", ["-ngl", "--gpu-layers", "--n-gpu-layers"])
@pytest.mark.parametrize("value, layers", [("auto", -1), ("all", 2147483647)])
@pytest.mark.parametrize("mode", ["auto", "manual"])
def test_validate_preserves_gpu_layer_keyword_intent(flag, value, layers, mode):
    route = _load_route_module("inf_route_ngl_keywords")
    args = [flag, value]
    seen = TestValidateTranslatesManualNgl()._validate(
        route,
        gpu_layers = 0,
        extra_args = args,
        diffusion_kind = False,
        gpu_memory_mode = mode,
    )
    assert seen == ([(layers, [])] if mode == "manual" else [(0, args)])


@pytest.mark.parametrize("value, layers", [("auto", -1), ("all", 2147483647)])
@pytest.mark.parametrize("mode", ["auto", "manual"])
def test_load_normalizes_gpu_layer_keywords_before_model_work(monkeypatch, value, layers, mode):
    from fastapi import HTTPException
    from starlette.requests import Request
    from models.inference import LoadRequest
    import routes.inference as route

    seen = []

    def capture(request, **kwargs):
        seen.append((request.gpu_layers, request.llama_extra_args))
        raise HTTPException(status_code = 418, detail = "stopped before model resolution")

    async def no_ollama_lease(*args, **kwargs):
        return None

    monkeypatch.setattr(route, "_resolve_model_identifier_for_request", capture)
    monkeypatch.setattr(route, "_lease_ollama_model_ref", no_ollama_lease)
    monkeypatch.setattr(route, "_as_ollama_manifest_request", lambda request: request)
    monkeypatch.setattr(route, "_as_local_scan_folder_request", lambda request, owner: request)
    monkeypatch.setattr(route, "_repo_is_in_the_hub_cache", lambda model: None)
    monkeypatch.setattr(route, "_hub_cache_footprint", lambda model: None)
    monkeypatch.setattr(route, "_lora_base_already_in_the_hub_cache", lambda model: None)
    monkeypatch.setattr(route, "_note_lora_base_fetched_with_a_request_token", lambda *a, **k: None)
    args = ["-ngl", value]
    request = LoadRequest(
        model_path = "placeholder.gguf",
        gpu_memory_mode = mode,
        gpu_layers = 0,
        llama_extra_args = args,
    )

    with pytest.raises(HTTPException) as caught:
        asyncio.run(route._load_model_impl(request, Request({"type": "http", "headers": []}), "t"))

    assert caught.value.status_code == 418
    assert seen == ([(layers, [])] if mode == "manual" else [(0, args)])


class TestValidateTranslatesManualTensorSplit(unittest.TestCase):
    """/validate must promote -ts to tensor_split as /load does, or it judges a different split."""

    def _validate(
        self,
        route,
        *,
        gpu_layers,
        extra_args,
        tensor_split = None,
    ):
        seen: list = []

        def _capture(_config, request, **kwargs):
            seen.append(
                (
                    request.gpu_layers,
                    request.tensor_split,
                    kwargs.get("llama_extra_args"),
                )
            )

        request = ValidateModelRequest(
            model_path = "someone/moe-gguf",
            llama_extra_args = extra_args,
            gpu_memory_mode = "manual",
            gpu_layers = gpu_layers,
            tensor_split = tensor_split,
        )
        config = SimpleNamespace(
            identifier = "someone/moe-gguf",
            display_name = "moe-gguf",
            is_gguf = True,
            is_lora = False,
            is_vision = False,
            gguf_file = None,
        )
        with (
            patch.object(
                route,
                "_resolve_model_identifier_for_request",
                return_value = ("someone/moe-gguf", "someone/moe-gguf", False),
            ),
            patch.object(route.ModelConfig, "from_identifier", return_value = config),
            patch.object(route, "_resolve_inherited_extra_args", return_value = list(extra_args)),
            patch.object(route, "_classify_diffusion_gguf", return_value = False),
            patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
            patch.object(route, "_effective_load_in_4bit", return_value = True),
            patch.object(route, "_guard_chat_load_against_training", new = _capture),
        ):
            asyncio.run(route.validate_model(request, current_subject = "test-user"))
        return seen

    def test_an_explicit_tensor_split_reaches_the_guard(self):
        route = _load_route_module("inf_route_manual_ts_1")
        seen = self._validate(
            route,
            gpu_layers = 49,
            extra_args = ["-ts", "2.2,1", "-sm", "layer"],
        )
        self.assertEqual(seen, [(49, [2.2, 1.0], ["-sm", "layer"])])

    def test_extras_override_a_first_class_tensor_split(self):
        route = _load_route_module("inf_route_manual_ts_2")
        seen = self._validate(
            route,
            gpu_layers = 49,
            tensor_split = [1, 1],
            extra_args = ["--tensor-split", "3,1"],
        )
        self.assertEqual(seen, [(49, [3.0, 1.0], [])])


if __name__ == "__main__":
    unittest.main()


class TestValidateRefusesWhatLoadWouldRefuse(unittest.TestCase):
    """The picker unloads the current model once /validate approves, so /validate must refuse what
    /load would: a refusal keeps the model, a failed switch does not."""

    def _validate(
        self,
        route,
        *,
        extra_args,
        n_parallel = None,
        diffusion_kind = False,
    ):
        request = ValidateModelRequest(
            model_path = "someone/gguf",
            llama_extra_args = extra_args,
            n_parallel = n_parallel,
        )
        config = SimpleNamespace(
            identifier = "someone/gguf",
            display_name = "gguf",
            is_gguf = True,
            is_lora = False,
            is_vision = False,
            gguf_file = None,
        )
        with (
            patch.object(
                route,
                "_resolve_model_identifier_for_request",
                return_value = ("someone/gguf", "someone/gguf", False),
            ),
            patch.object(route.ModelConfig, "from_identifier", return_value = config),
            patch.object(route, "_resolve_inherited_extra_args", return_value = extra_args),
            patch.object(route, "_classify_diffusion_gguf", return_value = diffusion_kind),
            patch.object(route, "_resolve_gguf_gpu_ids_for_request", new = _noop_gpu_ids),
            patch.object(route, "_effective_load_in_4bit", return_value = True),
            patch.object(route, "_effective_parallel_slots", side_effect = lambda n, **_: n),
            patch.object(route, "_guard_chat_load_against_training", new = lambda *a, **k: None),
        ):
            return asyncio.run(route.validate_model(request, current_subject = "test-user"))

    def test_a_denied_flag_is_refused_before_the_switch(self):
        route = _load_route_module("inf_route_validate_denies_1")
        with self.assertRaises(Exception) as caught:
            self._validate(route, extra_args = ["--agent"])
        self.assertEqual(getattr(caught.exception, "status_code", None), 400)
        self.assertIn("managed by Unsloth Studio", str(caught.exception.detail))

    def test_a_batch_below_the_slot_floor_is_refused_before_the_switch(self):
        route = _load_route_module("inf_route_validate_denies_2")
        with self.assertRaises(Exception) as caught:
            self._validate(route, extra_args = ["-b", "2"], n_parallel = 4)
        self.assertEqual(getattr(caught.exception, "status_code", None), 400)
        self.assertIn("aborts on --batch-size", str(caught.exception.detail))

    def test_a_list_the_load_would_accept_still_passes(self):
        route = _load_route_module("inf_route_validate_denies_3")
        resp = self._validate(route, extra_args = ["--numa", "distribute"], n_parallel = 4)
        self.assertTrue(resp.is_gguf)


class TestEmbeddingSlotClampInTheBatchFloor(unittest.TestCase):
    """--embedding caps the batch at the micro-batch, which llama-server aborts below the slot
    count; load_model clamps slots first, so the floor must use the clamped count."""

    def _clamped(
        self,
        route,
        *,
        is_embedding,
        extra_args,
        slots = 4,
        **kwargs,
    ):
        config = SimpleNamespace(identifier = "someone/embed-gguf", gguf_file = None)
        with patch.object(route, "_is_embedding_gguf", return_value = is_embedding):
            return route._embedding_clamped_slots(
                config,
                slots,
                extra_args = extra_args,
                n_batch = kwargs.get("n_batch"),
                n_ubatch = kwargs.get("n_ubatch"),
                n_ctx = kwargs.get("n_ctx"),
            )

    def test_the_slots_follow_the_micro_batch_down(self):
        route = _load_route_module("inf_route_embed_clamp_1")
        self.assertEqual(
            self._clamped(route, is_embedding = True, extra_args = ["-b", "2", "-ub", "2"]),
            2,
        )

    def test_a_chat_gguf_keeps_the_slots_it_asked_for(self):
        route = _load_route_module("inf_route_embed_clamp_2")
        self.assertEqual(
            self._clamped(route, is_embedding = False, extra_args = ["-b", "2", "-ub", "2"]),
            4,
        )

    def test_defaults_clamp_nothing(self):
        route = _load_route_module("inf_route_embed_clamp_3")
        self.assertEqual(
            self._clamped(route, is_embedding = True, extra_args = ["--numa", "distribute"]),
            4,
        )

    def test_the_clamp_floors_at_one_slot(self):
        # -b 0 gives a zero micro-batch; --parallel 0 fails arg parse; the floor is one slot.
        route = _load_route_module("inf_route_embed_clamp_4")
        self.assertEqual(
            self._clamped(route, is_embedding = True, extra_args = ["-b", "0", "-ub", "0"]),
            1,
        )

    def test_an_unreadable_header_leaves_the_refusal_alone(self):
        # _is_embedding_gguf answers False for a GGUF that is not on this disk yet, so
        # nothing is relaxed on a guess.
        route = _load_route_module("inf_route_embed_clamp_5")
        config = SimpleNamespace(identifier = "someone/gguf", gguf_file = None, gguf_hf_repo = None)
        self.assertFalse(route._is_embedding_gguf(config))

    def test_an_uncached_embedding_identifier_stays_fail_closed(self):
        route = _load_route_module("inf_route_embed_clamp_6")
        config = SimpleNamespace(
            identifier = "Qwen/Qwen3-Embedding-4B-GGUF",
            gguf_file = None,
            gguf_hf_repo = "Qwen/Qwen3-Embedding-4B-GGUF",
            gguf_variant = "Q4_K_M",
        )
        with patch.object(route, "_local_gguf_main_path", return_value = None):
            self.assertFalse(route._is_embedding_gguf(config))


class TestValidateAllowsTheEmbeddingClampedBatch(TestValidateRefusesWhatLoadWouldRefuse):
    """Preflight must match load exactly, or the picker refuses switches load would complete."""

    def test_an_embedding_gguf_may_batch_at_its_micro_batch(self):
        route = _load_route_module("inf_route_validate_embed_1")
        with patch.object(route, "_is_embedding_gguf", return_value = True):
            resp = self._validate(
                route,
                extra_args = ["-b", "2", "-ub", "2"],
                n_parallel = 4,
            )
        self.assertTrue(resp.is_gguf)

    def test_a_chat_gguf_is_still_refused(self):
        route = _load_route_module("inf_route_validate_embed_2")
        with (
            patch.object(route, "_is_embedding_gguf", return_value = False),
            self.assertRaises(Exception) as caught,
        ):
            self._validate(route, extra_args = ["-b", "2", "-ub", "2"], n_parallel = 4)
        self.assertEqual(getattr(caught.exception, "status_code", None), 400)
        self.assertIn("aborts on --batch-size", str(caught.exception.detail))

    def test_an_embedding_gguf_below_its_own_floor_is_still_refused(self):
        # llama-server aborts on a batch of 1 at any slot count.
        route = _load_route_module("inf_route_validate_embed_3")
        with (
            patch.object(route, "_is_embedding_gguf", return_value = True),
            self.assertRaises(Exception) as caught,
        ):
            self._validate(route, extra_args = ["-b", "1", "-ub", "1"], n_parallel = 4)
        self.assertEqual(getattr(caught.exception, "status_code", None), 400)