Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Chat-load guard during training: sizing must match the effective load the backend performs."""
import tempfile
import asyncio
import importlib.util
import sys
import threading
import types
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from fastapi import HTTPException
from utils.hardware import DeviceType
import utils.hardware.hardware as _hw_module
_BACKEND_ROOT = Path(__file__).resolve().parent.parent
# Load training_vram.py standalone (avoids the heavy routes/__init__.py); its
# lazy hardware imports still resolve against the patched utils.hardware names.
_spec = importlib.util.spec_from_file_location(
"training_vram_load_test", _BACKEND_ROOT / "routes" / "training_vram.py"
)
tv = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(tv)
def _gguf_cfg(**overrides):
"""The GGUF model row the chat route reads, with per-test overrides."""
return SimpleNamespace(
**{
"gguf_file": None,
"gguf_mmproj_file": None,
"gguf_mtp_file": None,
"gguf_dspark_file": None,
"gguf_dflash_file": None,
"gguf_hf_repo": "org/repo",
"gguf_variant": "Q4_K_M",
**overrides,
}
)
def _model_cfg(**overrides):
"""A non-GGUF model row, with per-test overrides."""
return SimpleNamespace(
**{
"is_gguf": False,
"is_lora": False,
"is_vision": False,
"path": None,
"base_model": None,
"gguf_file": None,
**overrides,
}
)
class _GpuCacheResetMixin:
def tearDown(self):
_hw_module._physical_gpu_count = None
_hw_module._visible_gpu_count = None
def _devices(*free_specs):
"""Build a device list from (index, total, used) tuples."""
return [
{"index": i, "vram_total_gb": total, "vram_used_gb": used}
for (i, total, used) in free_specs
]
class TestCanLoadAutoHF(_GpuCacheResetMixin, unittest.TestCase):
def _run(self, *, selection_mode, required, usable):
meta = {"selection_mode": selection_mode, "required_gb": required, "usable_gb": usable}
with (
patch("utils.hardware.get_device", return_value = DeviceType.CUDA),
patch("utils.hardware.auto_select_gpu_ids", return_value = ([0], meta)) as auto_mock,
):
ok, info = tv.can_load_chat_during_training(
model_name = "unsloth/Qwen3-1.7B",
hf_token = None,
load_in_4bit = True,
max_seq_length = 0,
requested_gpu_ids = None,
is_gguf = False,
)
return ok, info, auto_mock
def test_fits_with_margin(self):
# free 60 >= 8*1.15+4 = 13.2
ok, info, auto_mock = self._run(selection_mode = "auto", required = 8.0, usable = 60.0)
self.assertTrue(ok)
self.assertEqual(info["mode"], "auto")
self.assertAlmostEqual(info["needed_gb"], 13.2, places = 3)
auto_mock.assert_called_once()
def test_too_tight_refuses(self):
# free 10 < 8*1.15+4 = 13.2: refuse even though raw 10 > 8
ok, _, _ = self._run(selection_mode = "auto", required = 8.0, usable = 10.0)
self.assertFalse(ok)
def test_fallback_all_refuses(self):
# Selector couldn't confirm placement -> default-deny to protect training.
ok, info = self._run(selection_mode = "fallback_all", required = 8.0, usable = 999.0)[:2]
self.assertFalse(ok)
class TestCanLoadExplicitHF(_GpuCacheResetMixin, unittest.TestCase):
def _run(
self,
*,
required,
devices,
gpu_ids,
resolved = None,
resolve_side_effect = None,
):
resolve_kwargs = (
{"side_effect": resolve_side_effect}
if resolve_side_effect
else {"return_value": resolved if resolved is not None else gpu_ids}
)
with (
patch("utils.hardware.get_device", return_value = DeviceType.CUDA),
patch("utils.hardware.estimate_required_model_memory_gb", return_value = (required, {})),
patch("utils.hardware.get_visible_gpu_utilization", return_value = {"devices": devices}),
patch("utils.hardware.resolve_requested_gpu_ids", **resolve_kwargs),
patch("utils.hardware.auto_select_gpu_ids") as auto_mock,
):
ok, info = tv.can_load_chat_during_training(
model_name = "m",
hf_token = None,
load_in_4bit = True,
max_seq_length = 0,
requested_gpu_ids = gpu_ids,
is_gguf = False,
)
return ok, info, auto_mock
def test_single_gpu_fits(self):
ok, info, auto_mock = self._run(required = 8.0, devices = _devices((0, 80, 20)), gpu_ids = [0])
self.assertTrue(ok)
self.assertEqual(info["mode"], "explicit")
auto_mock.assert_not_called()
def test_per_gpu_floor_blocks_uneven_split(self):
ok, info, _ = self._run(
required = 20.0, devices = _devices((0, 80, 35), (1, 80, 70)), gpu_ids = [0, 1]
)
self.assertFalse(ok)
self.assertAlmostEqual(info["min_free_gb"], 10.0, places = 3)
def test_per_gpu_floor_passes_when_even(self):
ok, _, _ = self._run(
required = 20.0, devices = _devices((0, 80, 50), (1, 80, 50)), gpu_ids = [0, 1]
)
self.assertTrue(ok)
def test_missing_gpu_counts_as_zero(self):
ok, _, _ = self._run(required = 5.0, devices = _devices((0, 80, 5)), gpu_ids = [3], resolved = [3])
self.assertFalse(ok)
def test_invalid_ids_does_not_block(self):
ok, info, _ = self._run(
required = 5.0,
devices = [],
gpu_ids = [99],
resolve_side_effect = ValueError("Invalid gpu_ids [99]"),
)
self.assertTrue(ok)
self.assertEqual(info["reason"], "invalid_gpu_ids")
class TestCanLoadGGUF(_GpuCacheResetMixin, unittest.TestCase):
def _run(
self,
*,
devices,
required_override = None,
estimate = None,
single_device_gpu = None,
gpu_ids = None,
gpu_ids_are_vulkan_ordinals = False,
vulkan_free_vram_gb = None,
):
with (
patch("utils.hardware.get_device", return_value = DeviceType.CUDA),
patch("utils.hardware.estimate_required_model_memory_gb", return_value = (estimate, {})),
patch("utils.hardware.get_visible_gpu_utilization", return_value = {"devices": devices}),
patch("utils.hardware.resolve_requested_gpu_ids", return_value = gpu_ids),
patch("utils.hardware.auto_select_gpu_ids") as auto_mock,
):
ok, info = tv.can_load_chat_during_training(
model_name = "unsloth/gemma-GGUF",
hf_token = None,
load_in_4bit = True,
max_seq_length = 0,
requested_gpu_ids = gpu_ids,
is_gguf = True,
gpu_ids_are_vulkan_ordinals = gpu_ids_are_vulkan_ordinals,
vulkan_free_vram_gb = vulkan_free_vram_gb,
required_override_gb = required_override,
single_device_gpu = single_device_gpu,
)
return ok, info, auto_mock
def test_override_fits(self):
ok, info, auto_mock = self._run(devices = _devices((0, 80, 20)), required_override = 10.0)
self.assertTrue(ok)
self.assertEqual(info["mode"], "gguf")
auto_mock.assert_not_called()
def test_no_per_gpu_floor_for_gguf(self):
# GGUF self-places, so the per-GPU floor that blocks HF does not apply.
ok, _, _ = self._run(devices = _devices((0, 80, 35), (1, 80, 70)), required_override = 20.0)
self.assertTrue(ok)
def test_no_per_gpu_floor_for_gguf_with_explicit_gpu_ids(self):
ok, info, _ = self._run(
devices = _devices((0, 80, 35), (1, 80, 70), (2, 80, 0)),
required_override = 20.0,
gpu_ids = [0, 1],
)
self.assertTrue(ok)
self.assertEqual(info["mode"], "gguf")
def test_single_device_uses_selected_gpu(self):
ok, info, _ = self._run(
devices = _devices((0, 80, 35), (1, 80, 70)),
required_override = 20.0,
single_device_gpu = "0",
)
self.assertTrue(ok)
self.assertEqual(info["usable_gb"], 45.0)
blocked, blocked_info, _ = self._run(
devices = _devices((0, 80, 35), (1, 80, 70)),
required_override = 20.0,
single_device_gpu = "1",
)
self.assertFalse(blocked)
self.assertEqual(blocked_info["usable_gb"], 10.0)
def test_vulkan_pin_takes_precedence_over_unknown_diffusion_fallback(self):
ok, info, _ = self._run(
devices = _devices((0, 80, 0)),
required_override = 20.0,
single_device_gpu = "0",
gpu_ids = [0],
gpu_ids_are_vulkan_ordinals = True,
vulkan_free_vram_gb = {0: 2.0, 1: 80.0},
)
self.assertFalse(ok)
self.assertEqual(info["mode"], "gguf_vulkan")
self.assertEqual(info["usable_gb"], 2.0)
def test_vulkan_multi_gpu_guard_counts_requested_devices(self):
ok, info, _ = self._run(
devices = _devices((0, 80, 0)),
required_override = 10.0,
gpu_ids = [0, 1],
gpu_ids_are_vulkan_ordinals = True,
vulkan_free_vram_gb = {0: 10.0, 1: 10.0, 2: 80.0},
)
self.assertTrue(ok)
self.assertEqual(info["mode"], "gguf_vulkan")
self.assertEqual(info["usable_gb"], 18.5)
def test_vulkan_uneven_pool_uses_fitting_subset(self):
ok, info, _ = self._run(
devices = _devices((0, 80, 0), (1, 80, 0)),
required_override = 10.0,
gpu_ids = [0, 1],
gpu_ids_are_vulkan_ordinals = True,
vulkan_free_vram_gb = {0: 20.0, 1: 2.0},
)
self.assertTrue(ok)
self.assertNotIn("per_gpu_needed_gb", info)
def test_vulkan_uneven_pool_is_order_independent(self):
ok, _, _ = self._run(
devices = _devices((0, 80, 0), (1, 80, 0)),
required_override = 10.0,
gpu_ids = [1, 0],
gpu_ids_are_vulkan_ordinals = True,
vulkan_free_vram_gb = {0: 20.0, 1: 2.0},
)
self.assertTrue(ok)
def test_vulkan_pool_with_insufficient_aggregate_refuses(self):
ok, _, _ = self._run(
devices = _devices((0, 80, 0), (1, 80, 0)),
required_override = 10.0,
gpu_ids = [0, 1],
gpu_ids_are_vulkan_ordinals = True,
vulkan_free_vram_gb = {0: 8.0, 1: 2.0},
)
self.assertFalse(ok)
def test_vulkan_auto_uses_full_vulkan_pool(self):
ok, info, _ = self._run(
devices = _devices((0, 80, 79)),
required_override = 20.0,
vulkan_free_vram_gb = {0: 30.0, 1: 30.0},
)
self.assertTrue(ok)
self.assertEqual(info["mode"], "gguf_vulkan")
self.assertEqual(info["usable_gb"], 55.5)
def test_single_device_unresolved_token_sizes_against_worst_device(self):
ok, info, _ = self._run(
devices = _devices((0, 80, 0)),
required_override = 20.0,
single_device_gpu = "GPU-uuid",
)
self.assertTrue(ok)
self.assertEqual(info["mode"], "single_device")
self.assertNotIn("reason", info)
def test_single_device_unresolved_token_refuses_when_worst_device_full(self):
ok, info, _ = self._run(
devices = _devices((0, 80, 78)),
required_override = 20.0,
single_device_gpu = "GPU-uuid",
)
self.assertFalse(ok)
self.assertNotEqual(info.get("reason"), "unresolved_gpu_id")
def test_single_device_unresolved_token_uses_min_free_not_aggregate(self):
ok, info, _ = self._run(
devices = _devices((0, 80, 78), (1, 80, 0), (2, 80, 0)),
required_override = 20.0,
single_device_gpu = "GPU-uuid",
)
self.assertFalse(ok)
self.assertEqual(info["mode"], "single_device")
def test_single_device_cpu_token_allows(self):
ok, info, _ = self._run(
devices = _devices((0, 80, 78)),
required_override = 20.0,
single_device_gpu = "",
)
self.assertTrue(ok)
self.assertEqual(info["reason"], "cpu_only")
def test_estimate_unavailable_refuses(self):
ok, info, _ = self._run(devices = _devices((0, 80, 0)), required_override = None, estimate = None)
self.assertFalse(ok)
self.assertEqual(info["reason"], "estimate_unavailable")
class TestCanLoadMisc(_GpuCacheResetMixin, unittest.TestCase):
def test_non_accelerator_allows(self):
with patch("utils.hardware.get_device", return_value = DeviceType.MLX):
ok, info = tv.can_load_chat_during_training(
model_name = "m",
hf_token = None,
load_in_4bit = True,
max_seq_length = 0,
requested_gpu_ids = None,
)
self.assertTrue(ok)
self.assertEqual(info["mode"], "non_accelerator")
def test_xpu_overcommit_is_refused(self):
with (
patch("utils.hardware.get_device", return_value = DeviceType.XPU),
patch(
"utils.hardware.auto_select_gpu_ids",
return_value = (
None,
{"selection_mode": "auto", "required_gb": 50.0, "usable_gb": 4.0},
),
),
):
ok, info = tv.can_load_chat_during_training(
model_name = "m",
hf_token = None,
load_in_4bit = True,
max_seq_length = 0,
requested_gpu_ids = None,
)
self.assertFalse(ok)
self.assertNotEqual(info.get("mode"), "non_accelerator")
def test_no_visible_gpus_refuses(self):
with (
patch("utils.hardware.get_device", return_value = DeviceType.CUDA),
patch("utils.hardware.get_visible_gpu_utilization", return_value = {"devices": []}),
patch("utils.hardware.auto_select_gpu_ids"),
):
ok, info = tv.can_load_chat_during_training(
model_name = "m",
hf_token = None,
load_in_4bit = True,
max_seq_length = 0,
requested_gpu_ids = None,
is_gguf = True,
required_override_gb = 8.0,
)
self.assertFalse(ok)
self.assertEqual(info["reason"], "no_visible_gpus")
def test_probe_exception_refuses(self):
with patch("utils.hardware.get_device", side_effect = RuntimeError("boom")):
ok, info = tv.can_load_chat_during_training(
model_name = "m",
hf_token = None,
load_in_4bit = True,
max_seq_length = 0,
requested_gpu_ids = None,
)
self.assertFalse(ok)
self.assertEqual(info["reason"], "probe_error")
def _load_inference_route():
spec = importlib.util.spec_from_file_location(
"inference_route_chatload_test", _BACKEND_ROOT / "routes" / "inference.py"
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _stub_guard_deps(
*,
training_active,
decision,
captured = None,
):
"""Stubs the guard's two lazy imports; captured records the can_load kwargs."""
# Import the gate within patch.dict's snapshot so no stale attribute makes a second lock.
import core.inference.llama_keepwarm # noqa: F401
core_training = types.ModuleType("core.training")
if isinstance(training_active, Exception):
def _raise():
raise training_active
core_training.get_training_backend = _raise
else:
core_training.get_training_backend = lambda: SimpleNamespace(
is_training_active = lambda: training_active
)
def _can_load(**kwargs):
if captured is not None:
captured.append(kwargs)
return decision
tv_stub = types.ModuleType("routes.training_vram")
tv_stub.can_load_chat_during_training = _can_load
return patch.dict(
sys.modules, {"core.training": core_training, "routes.training_vram": tv_stub}
)
class TestChatLoadGuardRoute(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.route = _load_inference_route()
def _guard(
self,
*,
config = None,
captured = None,
training_active,
decision,
gpu_memory_mode = "auto",
requested_gpu_ids = None,
gpu_ids_are_vulkan_ordinals = False,
llama_extra_args = None,
cache_type_kv = None,
tensor_parallel = False,
gpu_layers = -1,
):
config = config or SimpleNamespace(is_gguf = False, is_lora = False, path = None)
placement = self.route._LoadPlacement(
requested_gpu_ids,
requested_gpu_ids,
gpu_ids_are_vulkan_ordinals,
self.route._classify_diffusion_gguf(config) if config.is_gguf else False,
)
with _stub_guard_deps(
training_active = training_active, decision = decision, captured = captured
):
request = SimpleNamespace(
model_path = "unsloth/Qwen3-1.7B",
hf_token = None,
max_seq_length = 0,
cache_type_kv = cache_type_kv,
tensor_parallel = tensor_parallel,
gpu_memory_mode = gpu_memory_mode,
gpu_layers = gpu_layers,
)
self.route._guard_chat_load_against_training(
config,
request,
load_in_4bit = True,
placement = placement,
llama_extra_args = llama_extra_args,
)
def test_noop_when_training_inactive(self):
self._guard(training_active = False, decision = (False, {}))
def test_noop_when_training_state_unknown(self):
self._guard(training_active = RuntimeError("no backend"), decision = (False, {}))
def test_allows_when_fits(self):
self._guard(training_active = True, decision = (True, {"mode": "auto"}))
def test_diffusion_detection_uses_name_before_download(self):
config = SimpleNamespace(
identifier = "unsloth/DiffusionGemma-GGUF",
gguf_hf_repo = "unsloth/DiffusionGemma-GGUF",
gguf_file = None,
)
self.assertTrue(self.route._classify_diffusion_gguf(config))
def test_uncached_gguf_classification_remains_unknown(self):
config = SimpleNamespace(
identifier = "owner/renamed-model",
gguf_hf_repo = "owner/renamed-model",
gguf_variant = "Q4_K_M",
gguf_file = None,
)
self.assertIsNone(self.route._classify_diffusion_gguf(config))
def test_diffusion_detection_reuses_loader_metadata_probe(self):
import tempfile
seen = []
class _Probe:
is_diffusion = False
_architecture = None
def _read_gguf_metadata(self, path):
seen.append(path)
self.is_diffusion = True
with tempfile.TemporaryDirectory() as d:
model = Path(d) / "renamed.gguf"
model.write_bytes(b"GGUF")
config = SimpleNamespace(identifier = "local", gguf_file = str(model))
with patch.object(self.route, "LlamaCppBackend", _Probe):
self.assertTrue(self.route._classify_diffusion_gguf(config))
self.assertEqual(seen, [str(model)])
def test_local_chat_gguf_classification_is_definitive(self):
import tempfile
class _Probe:
is_diffusion = False
_architecture = "llama"
def _read_gguf_metadata(self, _path):
pass
with tempfile.TemporaryDirectory() as d:
model = Path(d) / "renamed.gguf"
model.write_bytes(b"GGUF")
config = SimpleNamespace(identifier = "local", gguf_file = str(model))
with patch.object(self.route, "LlamaCppBackend", _Probe):
self.assertFalse(self.route._classify_diffusion_gguf(config))
def test_manual_known_normal_gguf_bypasses_training_estimate(self):
captured = []
config = SimpleNamespace(is_gguf = True)
with patch.object(self.route, "_classify_diffusion_gguf", return_value = False) as classify:
self._guard(
config = config,
captured = captured,
training_active = True,
decision = (False, {"reason": "must not run"}),
gpu_memory_mode = "manual",
requested_gpu_ids = [1, 3],
)
classify.assert_called_once_with(config)
self.assertEqual(captured, [])
def _guard_diffusion_pick(
self,
captured,
fit = None,
gguf_file = None,
**kwargs,
):
config = SimpleNamespace(is_gguf = True, gguf_file = gguf_file)
backend = SimpleNamespace(
diffusion_split_supported = lambda: True,
_find_diffusion_assets = lambda: (["python"], "/bin/vs", None),
)
fits = []
with (
patch.object(
self.route.LlamaCppBackend,
"_diffusion_fit_gpus",
side_effect = lambda *a: fits.append(a) or fit,
),
patch.object(
self.route.LlamaCppBackend,
"_get_gguf_size_bytes",
side_effect = lambda _p: 9 * 1024**3,
),
patch.object(self.route, "_classify_diffusion_gguf", return_value = True),
patch.object(self.route, "get_llama_cpp_backend", return_value = backend),
patch.object(self.route, "_estimate_gguf_required_gb", return_value = 12.5),
patch.object(
self.route.LlamaCppBackend,
"_effective_gpu_count",
return_value = 2,
),
):
self._guard(
config = config,
captured = captured,
training_active = True,
decision = (True, {"mode": "single_device"}),
gpu_memory_mode = "manual",
requested_gpu_ids = [3, 1],
**kwargs,
)
return fits
def test_manual_diffusion_split_keeps_single_device_training_guard(self):
# An explicit layer count runs on the lowest picked GPU only.
captured = []
self._guard_diffusion_pick(captured, gpu_layers = 20)
self.assertEqual(len(captured), 1)
self.assertEqual(captured[0]["single_device_gpu"], "1")
self.assertEqual(captured[0]["requested_gpu_ids"], [3, 1])
def test_auto_layers_diffusion_split_is_budgeted_on_its_gpus(self):
# Without a layer count the runner splits what GPU 1 cannot hold (issue 6347); the guard
# budgets the decision's GPUs by weight bytes, not the context-inclusive estimate.
captured = []
with tempfile.NamedTemporaryFile(suffix = ".gguf") as f:
fits = self._guard_diffusion_pick(captured, fit = [1, 3], gguf_file = f.name)
self.assertEqual(fits, [(9 * 1024**3, [3, 1], "1", "/bin/vs")])
self.assertIsNone(captured[0]["single_device_gpu"])
self.assertEqual(captured[0]["requested_gpu_ids"], [1, 3])
def test_auto_layers_diffusion_pick_that_fits_one_gpu_keeps_its_budget(self):
captured = []
with tempfile.NamedTemporaryFile(suffix = ".gguf") as f:
self._guard_diffusion_pick(captured, fit = None, gguf_file = f.name)
self.assertEqual(captured[0]["single_device_gpu"], "1")
self.assertEqual(captured[0]["requested_gpu_ids"], [3, 1])
def test_diffusion_gguf_not_on_disk_keeps_the_single_device_budget(self):
# Nothing to read the launcher's weight size from: stay on the stricter budget.
captured = []
fits = self._guard_diffusion_pick(captured, fit = [1, 3])
self.assertEqual(fits, [])
self.assertEqual(captured[0]["single_device_gpu"], "1")
def _guard_zero_layer(self, *, diffusion_kind, captured):
"""Drive the guard with a manual zero-layer split and a shim that has --ngl."""
config = SimpleNamespace(is_gguf = True)
backend = SimpleNamespace(diffusion_split_supported = lambda: True)
with (
patch.object(self.route, "_classify_diffusion_gguf", return_value = diffusion_kind),
patch.object(self.route, "get_llama_cpp_backend", return_value = backend),
patch.object(self.route, "_estimate_gguf_required_gb", return_value = 12.5),
patch.object(self.route.LlamaCppBackend, "_effective_gpu_count", return_value = 2),
):
self._guard(
config = config,
captured = captured,
training_active = True,
decision = (False, {"reason": "must not run"}),
gpu_memory_mode = "manual",
gpu_layers = 0,
)
def test_zero_layer_diffusion_split_bypasses_the_training_guard(self):
"""A confirmed DiffusionGemma at ngl 0 places no layers, so it is not refused."""
captured = []
self._guard_zero_layer(diffusion_kind = True, captured = captured)
self.assertEqual(captured, [])
def test_zero_layer_unclassified_gguf_still_hits_the_training_guard(self):
"""An unreadable header is not a diffusion promise: --gpu-layers 0 on an
ordinary GGUF can still hold VRAM, so the estimate must run."""
captured = []
with self.assertRaises(HTTPException) as ctx:
self._guard_zero_layer(diffusion_kind = None, captured = captured)
self.assertEqual(ctx.exception.status_code, 409)
self.assertEqual(len(captured), 1)
def test_zero_layer_unclassified_gguf_is_not_sized_as_cpu_only(self):
"""Unclassified GGUF must not reach the guard as cpu_only; an empty GPU list admits."""
captured = []
with self.assertRaises(HTTPException):
self._guard_zero_layer(diffusion_kind = None, captured = captured)
self.assertEqual(len(captured), 1)
token = captured[0].get("single_device_gpu")
self.assertTrue(
token is None or str(token).strip() != "",
"an unclassified zero-layer GGUF must not be budgeted as CPU-only; "
f"single_device_gpu was {token!r}, which training_vram reads as cpu_only",
)
def test_unclassified_gguf_on_vulkan_build_budgets_as_ordinals(self):
captured = []
config = SimpleNamespace(is_gguf = True)
with (
patch.object(self.route, "_classify_diffusion_gguf", return_value = None),
patch.object(self.route, "_estimate_gguf_required_gb", return_value = 12.5),
patch.object(
self.route.LlamaCppBackend,
"_get_gpu_memory",
return_value = [(0, 2048, 8192), (1, 4096, 8192)],
),
):
self._guard(
config = config,
captured = captured,
training_active = True,
decision = (True, {"mode": "gguf_vulkan"}),
requested_gpu_ids = [0, 1],
gpu_ids_are_vulkan_ordinals = True,
)
self.assertEqual(len(captured), 1)
self.assertTrue(captured[0]["gpu_ids_are_vulkan_ordinals"])
self.assertEqual(captured[0]["vulkan_free_vram_gb"], {0: 2.0, 1: 4.0})
self.assertIsNone(captured[0]["single_device_gpu"])
def test_auto_vulkan_uses_vulkan_probe_for_training_budget(self):
captured = []
config = SimpleNamespace(is_gguf = True)
with (
patch.object(self.route, "_classify_diffusion_gguf", return_value = False),
patch.object(self.route, "_estimate_gguf_required_gb", return_value = 12.5),
patch.object(
self.route.LlamaCppBackend,
"_find_llama_server_binary",
return_value = "/tmp/llama-server",
),
patch.object(self.route.LlamaCppBackend, "_is_vulkan_backend", return_value = True),
patch.object(
self.route.LlamaCppBackend,
"_get_gpu_memory",
return_value = [(0, 3072, 0), (1, 2048, 8192)],
),
):
self._guard(
config = config,
captured = captured,
training_active = True,
decision = (True, {"mode": "gguf_vulkan"}),
)
self.assertEqual(captured[0]["vulkan_free_vram_gb"], {1: 2.0})
def test_unclassified_gguf_budget_depends_on_the_pin(self):
# An unclassifiable remote GGUF may be the CUDA-only diffusion runner.
cases = (
# No gpu_ids: fall back to the torch view (None); {} would read as no free VRAM.
(None, "single_device", None),
# An explicit pin must fail closed with an empty map.
([1], "gguf_vulkan", {}),
)
for requested_gpu_ids, mode, expected in cases:
with self.subTest(requested_gpu_ids = requested_gpu_ids):
captured = []
config = SimpleNamespace(is_gguf = True)
with (
patch.object(self.route, "_classify_diffusion_gguf", return_value = None),
patch.object(self.route, "_estimate_gguf_required_gb", return_value = 12.5),
patch.object(
self.route.LlamaCppBackend,
"_find_llama_server_binary",
return_value = "/tmp/llama-server",
),
patch.object(
self.route.LlamaCppBackend, "_is_vulkan_backend", return_value = True
),
):
self._guard(
config = config,
captured = captured,
training_active = True,
decision = (True, {"mode": mode}),
requested_gpu_ids = requested_gpu_ids,
)
self.assertEqual(len(captured), 1)
self.assertEqual(captured[0]["vulkan_free_vram_gb"], expected)
def test_refuses_with_headroom_number(self):
info = {"required_gb": 30.0, "usable_gb": 6.0, "needed_gb": 39.0, "mode": "auto"}
with self.assertRaises(HTTPException) as exc:
self._guard(training_active = True, decision = (False, info))
self.assertEqual(exc.exception.status_code, 409)
self.assertIn("39 GB", exc.exception.detail)
self.assertNotIn("30 GB", exc.exception.detail)
self.assertIn("including safety headroom", exc.exception.detail)
self.assertNotIn("chat is disabled", exc.exception.detail.lower())
def test_refuses_generic_when_unsizable(self):
with self.assertRaises(HTTPException) as exc:
self._guard(training_active = True, decision = (False, {"reason": "estimate_unavailable"}))
self.assertEqual(exc.exception.status_code, 409)
self.assertIn("could not be verified", exc.exception.detail)
def test_gguf_config_passes_is_gguf_and_override(self):
captured = []
config = SimpleNamespace(identifier = "unsloth/Canonical-Repo", is_gguf = True)
with patch.object(self.route, "_estimate_gguf_required_gb", return_value = 12.5):
self._guard(
config = config,
captured = captured,
training_active = True,
decision = (True, {}),
)
self.assertEqual(captured[0]["is_gguf"], True)
self.assertEqual(captured[0]["required_override_gb"], 12.5)
self.assertEqual(captured[0]["model_name"], config.identifier)
def test_vulkan_gguf_estimate_keeps_tensor_cache_coercion(self):
config = SimpleNamespace(is_gguf = True)
estimate_kwargs = {}
with (
patch.object(
self.route,
"_estimate_gguf_required_gb",
side_effect = lambda *args, **kwargs: estimate_kwargs.update(kwargs) or 12.5,
),
patch.object(
self.route.LlamaCppBackend,
"_effective_gpu_count",
return_value = 0,
),
patch.object(
self.route.LlamaCppBackend,
"_find_llama_server_binary",
return_value = "/fake/llama-server",
),
patch.object(self.route.LlamaCppBackend, "_is_vulkan_backend", return_value = True),
):
self._guard(
config = config,
training_active = True,
decision = (True, {}),
llama_extra_args = ["--split-mode", "tensor"],
cache_type_kv = "q4_0",
)
self.assertEqual(estimate_kwargs["cache_type_kv"], "q4_0")
self.assertTrue(estimate_kwargs["tensor_parallel"])
class TestEffectiveLoadIn4bit(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.route = _load_inference_route()
def _write_adapter(self, tmpdir, payload):
import json
(Path(tmpdir) / "adapter_config.json").write_text(json.dumps(payload))
def test_non_lora_returns_request(self):
cfg = SimpleNamespace(is_lora = False, path = None, base_model = None)
self.assertTrue(self.route._effective_load_in_4bit(cfg, True))
def test_lora_method_flips_to_16bit(self):
import tempfile
with tempfile.TemporaryDirectory() as d:
self._write_adapter(d, {"unsloth_training_method": "lora"})
cfg = SimpleNamespace(is_lora = True, path = d, base_model = "x")
self.assertFalse(self.route._effective_load_in_4bit(cfg, True))
def test_qlora_method_keeps_4bit(self):
import tempfile
with tempfile.TemporaryDirectory() as d:
self._write_adapter(d, {"unsloth_training_method": "qlora"})
cfg = SimpleNamespace(is_lora = True, path = d, base_model = "x")
self.assertTrue(self.route._effective_load_in_4bit(cfg, True))
def test_no_method_non_bnb_base_flips_to_16bit(self):
import tempfile
with tempfile.TemporaryDirectory() as d:
self._write_adapter(d, {})
cfg = SimpleNamespace(is_lora = True, path = d, base_model = "meta/Llama-3-8B")
self.assertFalse(self.route._effective_load_in_4bit(cfg, True))
def test_legacy_cpt_method_non_bnb_base_keeps_request(self):
import tempfile
with tempfile.TemporaryDirectory() as d:
self._write_adapter(d, {"unsloth_training_method": "CPT"})
cfg = SimpleNamespace(is_lora = True, path = d, base_model = "unsloth/Qwen3-4B")
self.assertTrue(self.route._effective_load_in_4bit(cfg, True))
def test_cpt_method_bnb_base_keeps_4bit(self):
import tempfile
with tempfile.TemporaryDirectory() as d:
self._write_adapter(d, {"unsloth_training_method": "CPT"})
cfg = SimpleNamespace(
is_lora = True, path = d, base_model = "unsloth/Qwen3-4B-unsloth-bnb-4bit"
)
self.assertTrue(self.route._effective_load_in_4bit(cfg, True))
def test_recorded_precision_wins(self):
import tempfile
for recorded in (True, False):
for requested in (True, False):
with tempfile.TemporaryDirectory() as d:
self._write_adapter(
d, {"unsloth_training_method": "CPT", "unsloth_load_in_4bit": recorded}
)
cfg = SimpleNamespace(is_lora = True, path = d, base_model = "unsloth/Qwen3-4B")
self.assertIs(self.route._effective_load_in_4bit(cfg, requested), recorded)
def test_malformed_adapter_config_returns_request(self):
import tempfile
with tempfile.TemporaryDirectory() as d:
(Path(d) / "adapter_config.json").write_text("[1, 2, 3]")
cfg = SimpleNamespace(is_lora = True, path = d, base_model = "x")
self.assertTrue(self.route._effective_load_in_4bit(cfg, True))
def _model_dir(
self,
root,
name,
config = None,
adapter = None,
):
import json
d = Path(root) / name
d.mkdir(parents = True)
(d / "config.json").write_text(json.dumps(config or {"model_type": "llama"}))
(d / "model.safetensors").write_bytes(b"")
if adapter is not None:
(d / "adapter_config.json").write_text(json.dumps(adapter))
return str(d)
def _studio_home(self):
import os
import tempfile
home = tempfile.TemporaryDirectory()
self.addCleanup(home.cleanup)
patcher = patch.dict(os.environ, {"UNSLOTH_STUDIO_HOME": home.name})
patcher.start()
self.addCleanup(patcher.stop)
return Path(home.name)
def test_full_finetune_output_loads_16bit(self):
home = self._studio_home()
path = self._model_dir(home / "outputs", "unsloth_Qwen3-0.6B_1771227800")
cfg = SimpleNamespace(is_lora = False, path = path, base_model = None)
self.assertFalse(self.route._effective_load_in_4bit(cfg, True))
ckpt = self._model_dir(Path(path), "checkpoint-10")
cfg = SimpleNamespace(is_lora = False, path = ckpt, base_model = None)
self.assertFalse(self.route._effective_load_in_4bit(cfg, True))
def test_quantized_or_foreign_models_keep_4bit(self):
home = self._studio_home()
outputs = home / "outputs"
quantized = self._model_dir(
outputs, "q", {"quantization_config": {"quant_method": "bitsandbytes"}}
)
adapter = self._model_dir(outputs, "a", adapter = {})
outside = self._model_dir(home / "exports", "merged")
for path in (quantized, adapter, outside, "unsloth/Qwen3-0.6B"):
cfg = SimpleNamespace(is_lora = False, path = path, base_model = None)
self.assertTrue(self.route._effective_load_in_4bit(cfg, True), path)
qlora = self._model_dir(outputs, "qlora", adapter = {"unsloth_training_method": "qlora"})
cfg = SimpleNamespace(is_lora = True, path = qlora, base_model = "x")
self.assertTrue(self.route._effective_load_in_4bit(cfg, True))
def test_native_audio_uses_full_precision_for_admission(self):
cfg = SimpleNamespace(
is_lora = False,
path = None,
base_model = None,
audio_type = "moss_tts_local",
)
self.assertFalse(self.route._effective_load_in_4bit(cfg, True))
class TestNativeAudioPlacementContracts(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.route = _load_inference_route()
def _preflight(
self,
*,
metadata,
availability = None,
requested = None,
selected = None,
):
config = SimpleNamespace(identifier = "test/native-audio", audio_type = "higgs_tts2")
request = SimpleNamespace(gpu_ids = requested, hf_token = None, max_seq_length = 2048)
placement = self.route._LoadPlacement(requested, None, False, False)
with (
patch("utils.hardware.get_device", return_value = DeviceType.CUDA),
patch(
"core.inference.native_audio.native_audio_security_targets",
return_value = [config.identifier],
),
patch(
"utils.hardware.prepare_gpu_selection",
return_value = (selected if selected is not None else [0], metadata),
),
patch.object(
self.route,
"_native_audio_post_handoff_free_gb",
return_value = availability,
),
patch.object(_hw_module, "IS_ROCM", False),
):
return asyncio.run(
self.route._preflight_native_audio_placement(config, request, placement)
)
def test_automatic_fallback_requires_attributable_resident_memory(self):
metadata = {"selection_mode": "fallback_all", "required_gb": 20.0, "usable_gb": 6.0}
with self.assertRaisesRegex(HTTPException, "No single GPU has enough free memory"):
self._preflight(metadata = metadata)
availability = self.route._NativeAudioAvailability({0: 24.0}, {0: 18.0})
placement = self._preflight(metadata = metadata, availability = availability)
self.assertEqual(placement.resolved_gpu_ids, [0])
self.assertEqual(placement.native_required_gb, 20.0)
self.assertEqual(placement.native_post_handoff_free_gb, {0: 24.0})
def test_minimax_uses_a_threshold_a_24_gb_card_can_report(self):
config = SimpleNamespace(
identifier = "MiniMaxAI/MiniMax-Music3",
audio_type = "minimax_music3",
)
request = SimpleNamespace(gpu_ids = None, hf_token = None, max_seq_length = 2048)
placement = self.route._LoadPlacement(None, None, False, False)
with (
patch("utils.hardware.get_device", return_value = DeviceType.CUDA),
patch.object(_hw_module, "IS_ROCM", False),
patch(
"core.inference.native_audio.native_audio_security_targets",
return_value = [config.identifier],
),
patch("utils.hardware.prepare_gpu_selection", return_value = ([1], {})) as prepare,
):
result = asyncio.run(
self.route._preflight_native_audio_placement(config, request, placement)
)
self.assertEqual(result.resolved_gpu_ids, [1])
self.assertEqual(result.native_required_gb, 23.0)
self.assertEqual(prepare.call_args.kwargs["required_override_gb"], 23.0)
def test_explicit_multi_gpu_is_rejected_before_load(self):
with self.assertRaisesRegex(HTTPException, "require one GPU"):
self._preflight(
metadata = {"selection_mode": "explicit"},
requested = [0, 1],
selected = [0, 1],
)
def _automatic_preflight(
self,
devices,
*,
required_gb,
availability = None,
):
"""The real automatic selector over ``devices`` ((index, total, used) tuples)."""
config = SimpleNamespace(identifier = "test/native-audio", audio_type = "higgs_tts2")
request = SimpleNamespace(gpu_ids = None, hf_token = None, max_seq_length = 2048)
placement = self.route._LoadPlacement(None, None, False, False)
with (
patch("utils.hardware.get_device", return_value = DeviceType.CUDA),
patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
patch.object(_hw_module, "IS_ROCM", False),
patch(
"utils.hardware.hardware.get_visible_gpu_utilization",
return_value = {"devices": _devices(*devices)},
),
patch(
"utils.hardware.hardware.get_parent_visible_gpu_ids",
return_value = [index for index, _total, _used in devices],
),
patch("utils.hardware.hardware._with_mig_instance_memory", side_effect = lambda d: d),
# Skip the host torch probe: it would be the first torch import, made from a worker thread.
patch("utils.hardware.hardware.rocm_gpu_ids_without_torch_kernels", return_value = set()),
patch(
"utils.hardware.hardware.estimate_required_model_memory_gb",
return_value = (required_gb, {"required_gb": required_gb}),
),
patch(
"core.inference.native_audio.native_audio_security_targets",
return_value = [config.identifier],
),
patch.object(
self.route,
"_native_audio_post_handoff_free_gb",
return_value = availability,
),
):
return asyncio.run(
self.route._preflight_native_audio_placement(config, request, placement)
)
def test_automatic_selection_never_pools_cards(self):
# GPU 0 holds the chat model: 4 GB free now, 24 GB once the handoff evicts it.
availability = self.route._NativeAudioAvailability({0: 24.0, 1: 6.0}, {0: 20.0})
placement = self._automatic_preflight(
((0, 24, 20), (1, 8, 2)), required_gb = 9.0, availability = availability
)
self.assertEqual(placement.resolved_gpu_ids, [0])
self.assertEqual(placement.native_post_handoff_free_gb, {0: 24.0})
def test_automatic_selection_weighs_a_card_outside_the_pooled_prefix(self):
availability = self.route._NativeAudioAvailability({0: 22.0, 1: 6.0, 2: 5.0}, {0: 20.0})
placement = self._automatic_preflight(
((0, 24, 22), (1, 8, 2), (2, 8, 3)), required_gb = 9.0, availability = availability
)
self.assertEqual(placement.resolved_gpu_ids, [0])
def test_automatic_selection_without_a_fitting_card_is_a_placement_refusal(self):
with self.assertRaises(HTTPException) as raised:
self._automatic_preflight(((0, 8, 2), (1, 8, 2)), required_gb = 9.0)
detail = str(raised.exception.detail)
self.assertIn("No single GPU has enough free memory", detail)
self.assertIn("CPU RAM", detail)
self.assertNotIn("Select a", detail)
def test_automatic_selection_keeps_the_freest_card_that_fits_now(self):
placement = self._automatic_preflight(((0, 24, 20), (1, 24, 2)), required_gb = 9.0)
self.assertEqual(placement.resolved_gpu_ids, [1])
self.assertEqual(placement.native_required_gb, 9.0)
def test_live_training_recheck_cannot_switch_off_selected_gpu(self):
with (
patch("utils.hardware.get_device", return_value = DeviceType.CUDA),
patch(
"utils.hardware.get_visible_gpu_utilization",
return_value = {"devices": _devices((0, 24, 12), (1, 48, 8))},
),
patch("utils.hardware.resolve_requested_gpu_ids", return_value = [0]),
patch("utils.hardware.auto_select_gpu_ids") as auto_select,
):
ok, info = tv.can_load_chat_during_training(
model_name = "m",
hf_token = None,
load_in_4bit = False,
max_seq_length = 2048,
requested_gpu_ids = [0],
required_override_gb = 16.0,
)
self.assertFalse(ok)
self.assertEqual(info["usable_gb"], 12.0)
auto_select.assert_not_called()
def test_worker_memory_reply_is_correlated_to_its_probe(self):
from core.inference.orchestrator import InferenceOrchestrator
backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
replies = iter(
[
{"type": "gpu_memory", "request_id": "old", "reclaimable_gpu_gb": {"0": 10.0}},
{"type": "gpu_memory", "request_id": "new", "reclaimable_gpu_gb": {"0": 4.0}},
]
)
backend._read_resp = lambda **_kwargs: next(replies)
backend._ensure_subprocess_alive = lambda: True
response = backend._wait_response("gpu_memory", timeout = 1.0, expected_request_id = "new")
self.assertEqual(response["reclaimable_gpu_gb"], {"0": 4.0})
def test_llama_floor_uses_the_child_to_physical_gpu_map(self):
from core.inference.llama_cpp import LlamaCppBackend
backend = LlamaCppBackend.__new__(LlamaCppBackend)
backend._process = SimpleNamespace(poll = lambda: None)
backend._gpu_ids = None
backend._child_gpu_physical_ids = (1,)
backend._stdout_lines = ["load_tensors: CUDA0 model buffer size = 10240.0 MiB"]
self.assertEqual(backend.reclaimable_gpu_memory_gb(), {1: 10.0})
def test_inflight_media_replacement_is_never_credited(self):
utilization = {"devices": [{"index": 0, "vram_total_gb": 24.0, "vram_used_gb": 18.0}]}
media = SimpleNamespace(
_generate_lock = threading.Lock(),
_lock = threading.Lock(),
_state = object(),
_loading = object(),
)
with (
patch("utils.hardware.get_visible_gpu_utilization", return_value = utilization),
patch("core.inference.gpu_arbiter.owner_snapshot", return_value = ("diffusion", 4)),
patch(
"core.inference.diffusion_engine_router.get_active_diffusion_engine",
return_value = media,
),
):
self.assertIsNone(self.route._native_audio_post_handoff_free_gb())
class TestValidateRefusesDuringTraining(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.route = _load_inference_route()
def _validate(
self,
*,
training_active,
decision,
captured = None,
load_in_4bit = True,
):
from models.inference import ValidateModelRequest
request = ValidateModelRequest(
model_path = "unsloth/Qwen3-1.7B", load_in_4bit = load_in_4bit, max_seq_length = 4096
)
cfg = _model_cfg(identifier = "unsloth/Qwen3-1.7B", display_name = "Qwen3-1.7B")
with (
patch.object(
self.route,
"_resolve_model_identifier_for_request",
return_value = ("unsloth/Qwen3-1.7B", "unsloth/Qwen3-1.7B", False),
),
patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg),
patch.object(self.route, "load_inference_config", return_value = {}),
_stub_guard_deps(training_active = training_active, decision = decision, captured = captured),
):
return asyncio.run(self.route.validate_model(request, current_subject = "test-user"))
def test_ok_when_training_inactive(self):
resp = self._validate(training_active = False, decision = (False, {}))
self.assertTrue(resp.valid)
def test_refuses_when_wont_fit(self):
info = {"required_gb": 40.0, "usable_gb": 5.0, "needed_gb": 50.0}
with self.assertRaises(HTTPException) as exc:
self._validate(training_active = True, decision = (False, info))
self.assertEqual(exc.exception.status_code, 409)
self.assertIn("training is running", exc.exception.detail)
def test_passes_real_load_settings_to_guard(self):
captured = []
self._validate(
training_active = True, decision = (True, {}), captured = captured, load_in_4bit = False
)
self.assertEqual(captured[0]["load_in_4bit"], False)
self.assertEqual(captured[0]["max_seq_length"], 4096)
def test_validate_forwards_manual_gpu_memory_mode_to_guard(self):
from models.inference import ValidateModelRequest
request = ValidateModelRequest(
model_path = "unsloth/model-GGUF",
gguf_variant = "Q4_K_M",
gpu_memory_mode = "manual",
)
cfg = _model_cfg(identifier = "unsloth/model-GGUF", display_name = "model-GGUF", is_gguf = True)
captured = {}
with (
patch.object(
self.route,
"_resolve_model_identifier_for_request",
return_value = ("unsloth/model-GGUF", "unsloth/model-GGUF", False),
),
patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg),
patch.object(self.route, "load_inference_config", return_value = {}),
patch.object(
self.route,
"_guard_chat_load_against_training",
lambda config, request, **kw: captured.update(request = request, **kw),
),
):
asyncio.run(self.route.validate_model(request, current_subject = "u"))
self.assertEqual(captured["request"].gpu_memory_mode, "manual")
def test_validate_forwards_inherited_extras_and_parallel_to_guard(self):
from models.inference import ValidateModelRequest
request = ValidateModelRequest(
model_path = "unsloth/Qwen3-1.7B",
max_seq_length = 4096,
cache_type_kv = "f32",
tensor_parallel = True,
)
cfg = _model_cfg(identifier = "unsloth/Qwen3-1.7B", display_name = "Qwen3-1.7B")
captured = {}
with (
patch.object(
self.route,
"_resolve_model_identifier_for_request",
return_value = ("unsloth/Qwen3-1.7B", "unsloth/Qwen3-1.7B", False),
),
patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg),
patch.object(self.route, "load_inference_config", return_value = {}),
patch.object(self.route, "_resolve_inherited_extra_args", return_value = ["-c", "32768"]),
patch.object(
self.route,
"_guard_chat_load_against_training",
lambda config, request, **kw: captured.update(request = request, **kw),
),
):
asyncio.run(self.route.validate_model(request, current_subject = "u"))
self.assertEqual(captured.get("llama_extra_args"), ["-c", "32768"])
self.assertIn("n_parallel", captured)
self.assertEqual(captured["request"].cache_type_kv, "f32")
self.assertTrue(captured["request"].tensor_parallel)
def test_metadata_probe_skips_training_guard(self):
from models.inference import ValidateModelRequest
request = ValidateModelRequest(
model_path = "unsloth/Qwen3-1.7B",
max_seq_length = 4096,
include_context_length = True,
)
cfg = _model_cfg(identifier = "unsloth/Qwen3-1.7B", display_name = "Qwen3-1.7B")
guard_called = []
with (
patch.object(
self.route,
"_resolve_model_identifier_for_request",
return_value = ("unsloth/Qwen3-1.7B", "unsloth/Qwen3-1.7B", False),
),
patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg),
patch.object(self.route, "load_inference_config", return_value = {}),
patch.object(
self.route,
"_guard_chat_load_against_training",
lambda *a, **kw: guard_called.append(True),
),
):
asyncio.run(self.route.validate_model(request, current_subject = "u"))
self.assertEqual(guard_called, [])
def test_metadata_probe_reports_diffusion(self):
from models.inference import ValidateModelRequest
request = ValidateModelRequest(
model_path = "unsloth/DiffusionGemma-GGUF",
include_context_length = True,
)
cfg = _model_cfg(
identifier = "unsloth/DiffusionGemma-GGUF",
display_name = "DiffusionGemma-GGUF",
is_gguf = True,
gguf_file = None,
gguf_hf_repo = "unsloth/DiffusionGemma-GGUF",
gguf_variant = None,
)
with (
patch.object(
self.route,
"_resolve_model_identifier_for_request",
return_value = (
"unsloth/DiffusionGemma-GGUF",
"unsloth/DiffusionGemma-GGUF",
False,
),
),
patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg),
patch.object(self.route, "load_inference_config", return_value = {}),
):
response = asyncio.run(self.route.validate_model(request, current_subject = "u"))
self.assertTrue(response.is_diffusion)
def _validate_gguf_template(
self,
*,
template,
canonical_path = "/picked/model.gguf",
):
from models.inference import ValidateModelRequest
request = ValidateModelRequest(
model_path = "model.gguf",
gguf_variant = "Q4_K_M",
native_path_lease = "signed-lease",
include_chat_template = True,
)
cfg = _model_cfg(
identifier = canonical_path,
display_name = "model.gguf",
is_gguf = True,
gguf_file = canonical_path,
)
import utils.models.gguf_metadata as gguf_meta
seen = {}
def _fake_read(path):
seen["path"] = path
return template
guard_called = []
with (
patch.object(
self.route,
"_resolve_model_identifier_for_request",
return_value = (canonical_path, "model.gguf", True),
),
patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg),
patch.object(self.route, "load_inference_config", return_value = {}),
patch.object(gguf_meta, "read_gguf_chat_template", _fake_read),
patch.object(
self.route,
"_guard_chat_load_against_training",
lambda *a, **kw: guard_called.append(True),
),
):
resp = asyncio.run(self.route.validate_model(request, current_subject = "u"))
return resp, seen, guard_called
def test_include_chat_template_reads_leased_gguf_embedded_template(self):
resp, seen, _ = self._validate_gguf_template(template = "{{ messages }}")
self.assertEqual(resp.chat_template, "{{ messages }}")
# Read only the leased file's embedded template; the grant covers just this path.
self.assertEqual(seen["path"], "/picked/model.gguf")
def test_include_chat_template_skips_training_guard(self):
_, _, guard_called = self._validate_gguf_template(template = "{{ messages }}")
self.assertEqual(guard_called, [])
def test_include_chat_template_over_cap_is_dropped(self):
from picker.schemas import MAX_CHAT_TEMPLATE_BYTES
resp, _, _ = self._validate_gguf_template(template = "a" * (MAX_CHAT_TEMPLATE_BYTES + 1))
self.assertIsNone(resp.chat_template)
class TestRemoteGgufComputeReserve(unittest.TestCase):
"""Check the remote GGUF compute reserve without the other tests' zero-cost stub."""
@classmethod
def setUpClass(cls):
cls.route = _load_inference_route()
def _reserve(self, **kwargs):
# Drop LLAMA_ARG_BATCH / LLAMA_ARG_UBATCH: a host exporting either moves the effective
# micro-batch off _DEFAULT_N_UBATCH and fails these for reasons unrelated to slot count.
import os
env = {
key: value
for key, value in os.environ.items()
if key not in ("LLAMA_ARG_BATCH", "LLAMA_ARG_UBATCH")
}
with patch.dict(os.environ, env, clear = True):
return self.route._remote_gguf_compute_reserve_gb(max_seq_length = 4096, **kwargs)
def test_reserve_is_the_mask_plus_an_activation_ceiling(self):
"""Charge the activation ceiling once per micro-batch and output rows per slot."""
from core.inference.llama_cpp import (
_ASSUMED_MAX_ACTIVATION_WIDTH,
_ASSUMED_MAX_VOCAB,
LlamaCppBackend,
)
ubatch = LlamaCppBackend._DEFAULT_N_UBATCH
mask = 4096 * ubatch * 2
def expected(slots):
rows = min(ubatch, slots * (1 + LlamaCppBackend._UNKNOWN_SPEC_DRAFT_N_MAX))
flat = _ASSUMED_MAX_ACTIVATION_WIDTH * ubatch * 4 + _ASSUMED_MAX_VOCAB * rows * 4
return (mask + flat * LlamaCppBackend._COMPUTE_BUFFER_SAFETY) / (1024**3)
self.assertAlmostEqual(self._reserve(n_parallel = 1), expected(1), places = 6)
self.assertAlmostEqual(self._reserve(n_parallel = 2), expected(2), places = 6)
self.assertLess(
self._reserve(n_parallel = 2) - self._reserve(n_parallel = 1),
_ASSUMED_MAX_ACTIVATION_WIDTH * ubatch * 4 / (1024**3) / 4,
)
def test_an_older_build_reserves_a_row_per_micro_batch_token(self):
"""Before ggml-org/llama.cpp#23861 the output rows cover the whole micro-batch,
which the remote guard charges when the binary reports such a build."""
from core.inference.llama_cpp import _ASSUMED_MAX_VOCAB, LlamaCppBackend
ubatch = LlamaCppBackend._DEFAULT_N_UBATCH
with patch.object(LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, **k: False):
current = self._reserve(n_parallel = 1)
with patch.object(LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, **k: True):
older = self._reserve(n_parallel = 1)
rows = (ubatch - (1 + LlamaCppBackend._UNKNOWN_SPEC_DRAFT_N_MAX)) * _ASSUMED_MAX_VOCAB * 4
self.assertAlmostEqual(
older - current, rows * LlamaCppBackend._COMPUTE_BUFFER_SAFETY / (1024**3), places = 6
)
def test_tensor_mode_replicates_the_whole_buffer_on_every_device(self):
"""Tensor mode reserves the mask, activations and output rows on each device."""
from core.inference.llama_cpp import LlamaCppBackend
for older_build in (False, True):
with patch.object(
LlamaCppBackend, "reserves_micro_batch_outputs", lambda *a, _o = older_build, **k: _o
):
single = self._reserve(n_parallel = 2)
tensor = self._reserve(n_parallel = 2, n_devices = 2, tensor_parallel = True)
self.assertAlmostEqual(tensor, 2 * single, places = 6)
def test_diffusion_reserves_nothing(self):
"""The default micro-batch is a llama-server notion: a diffusion estimate has no ubatch to
assume, and charging one would refuse loads that fit."""
self.assertEqual(self._reserve(n_parallel = 1, is_diffusion = True), 0.0)
class TestEstimateGgufRequiredGb(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.route = _load_inference_route()
def test_local_sums_split_shards(self):
import tempfile
with tempfile.TemporaryDirectory() as d:
p = Path(d)
(p / "model-00001-of-00002.gguf").write_bytes(b"x" * 1000)
(p / "model-00002-of-00002.gguf").write_bytes(b"y" * 2000)
cfg = SimpleNamespace(
gguf_file = str(p / "model-00001-of-00002.gguf"),
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_hf_repo = None,
gguf_variant = None,
)
gb = self.route._estimate_gguf_required_gb(cfg)
self.assertAlmostEqual(gb, 3000 / (1024**3), places = 9)
@staticmethod
def _dspark_capable(supported = True):
"""The sizing gate asks the binary whether it can run draft-dspark, so the
probe must be stubbed or these assertions track the host's llama.cpp."""
from core.inference.llama_cpp import LlamaCppBackend
return patch.object(
LlamaCppBackend,
"probe_server_capabilities",
classmethod(lambda cls, binary = None: {"supports_dspark": supported}),
)
@staticmethod
def _eagle3_capable(supported):
from core.inference.llama_cpp import LlamaCppBackend
return patch.object(
LlamaCppBackend,
"probe_server_capabilities",
classmethod(lambda cls, binary = None: {"supports_eagle3": supported}),
)
def test_local_dspark_sidecar_is_only_counted_when_requested(self):
import tempfile
with tempfile.TemporaryDirectory() as d:
p = Path(d)
target = p / "model.gguf"
sidecar = p / "dspark-model-Q8_0.gguf"
target.write_bytes(b"x" * 2000)
sidecar.write_bytes(b"y" * 3000)
cfg = SimpleNamespace(
gguf_file = str(target),
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = str(sidecar),
gguf_hf_repo = None,
gguf_variant = None,
)
with (
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
self._dspark_capable(),
):
off_gb = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "off",
)
dspark_gb = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "dspark",
)
extras_gb = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "off",
llama_extra_args = ["--spec-type", "draft-dspark"],
)
self.assertAlmostEqual(off_gb, 2000 / (1024**3), places = 9)
self.assertAlmostEqual(dspark_gb, 5000 / (1024**3), places = 9)
self.assertAlmostEqual(extras_gb, 5000 / (1024**3), places = 9)
def test_local_eagle3_drafter_paired_by_the_export_is_charged(self):
"""A Studio export's drafter.json pairs an EAGLE-3 GGUF that Auto attaches at launch."""
import json
import tempfile
with tempfile.TemporaryDirectory() as d:
p = Path(d)
target = p / "model.gguf"
target.write_bytes(b"x" * 2000)
(p / "eagle3-model-F16.gguf").write_bytes(b"y" * 3000)
cfg = SimpleNamespace(
gguf_file = str(target),
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_hf_repo = None,
gguf_variant = None,
)
with (
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
):
with self._eagle3_capable(True):
bare = self.route._estimate_gguf_required_gb(cfg, speculative_type = "auto")
(p / "drafter.json").write_text(
json.dumps({"strategy": "eagle3", "files": ["eagle3-model-F16.gguf"]})
)
auto = self.route._estimate_gguf_required_gb(cfg, speculative_type = "auto")
off = self.route._estimate_gguf_required_gb(cfg, speculative_type = "off")
# A binary without draft-eagle3 never opens the sidecar.
with self._eagle3_capable(False):
incapable = self.route._estimate_gguf_required_gb(
cfg, speculative_type = "eagle3"
)
self.assertAlmostEqual(bare, 2000 / (1024**3), places = 9)
self.assertAlmostEqual(auto, 5000 / (1024**3), places = 9)
self.assertAlmostEqual(off, 2000 / (1024**3), places = 9)
self.assertAlmostEqual(incapable, 2000 / (1024**3), places = 9)
def test_forced_dspark_on_an_incapable_binary_charges_no_drafter_at_all(self):
"""A forced DSpark falls back to --spec-default, which loads no drafter, so charging MTP
would refuse a fitting load; Auto still charges MTP."""
import tempfile
with tempfile.TemporaryDirectory() as d:
p = Path(d)
target = p / "model.gguf"
mtp = p / "mtp-model.gguf"
target.write_bytes(b"x" * 2000)
mtp.write_bytes(b"y" * 3000)
cfg = SimpleNamespace(
gguf_file = str(target),
gguf_mmproj_file = None,
gguf_mtp_file = str(mtp),
gguf_dspark_file = None,
gguf_hf_repo = None,
gguf_variant = None,
)
with (
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
self._dspark_capable(False),
):
forced = self.route._estimate_gguf_required_gb(cfg, speculative_type = "dspark")
auto = self.route._estimate_gguf_required_gb(cfg, speculative_type = "auto")
self.assertAlmostEqual(forced, 2000 / (1024**3), places = 9)
self.assertAlmostEqual(auto, 5000 / (1024**3), places = 9)
def test_validate_request_carries_the_mode_the_load_will_use(self):
"""/validate must get the load mode or its VRAM verdict disagrees with /load."""
from models.inference import ValidateModelRequest
req = ValidateModelRequest(
model_path = "unsloth/DeepSeek-V4-Flash-0731-GGUF",
speculative_type = "off",
spec_draft_n_max = 3,
)
self.assertEqual(req.speculative_type, "off")
self.assertEqual(req.spec_draft_n_max, 3)
self.assertIsNone(ValidateModelRequest(model_path = "org/repo").speculative_type)
def test_dspark_sidecar_is_not_charged_to_a_binary_that_cannot_run_it(self):
"""The loader skips the ~11 GB fetch when llama.cpp has no usable draft-dspark, so charging
it would refuse a load that never opens it and evict nothing."""
import tempfile
with tempfile.TemporaryDirectory() as d:
p = Path(d)
target = p / "model.gguf"
sidecar = p / "dspark-model-Q8_0.gguf"
target.write_bytes(b"x" * 2000)
sidecar.write_bytes(b"y" * 3000)
cfg = SimpleNamespace(
gguf_file = str(target),
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = str(sidecar),
gguf_hf_repo = None,
gguf_variant = None,
)
with patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0):
with self._dspark_capable(False):
incapable = self.route._estimate_gguf_required_gb(
cfg, speculative_type = "dspark"
)
with self._dspark_capable(True):
capable = self.route._estimate_gguf_required_gb(cfg, speculative_type = "dspark")
self.assertAlmostEqual(incapable, 2000 / (1024**3), places = 9)
self.assertAlmostEqual(capable, 5000 / (1024**3), places = 9)
def test_split_dspark_sidecar_counts_every_shard(self):
"""Discovery hands back shard 1, so sizing it with stat() alone would let the
guard admit a load that evicts the training run it exists to protect."""
import tempfile
with tempfile.TemporaryDirectory() as d:
p = Path(d)
target = p / "model.gguf"
target.write_bytes(b"x" * 2000)
shard1 = p / "dspark-model-Q8_0-00001-of-00002.gguf"
shard1.write_bytes(b"y" * 3000)
(p / "dspark-model-Q8_0-00002-of-00002.gguf").write_bytes(b"z" * 4000)
cfg = SimpleNamespace(
gguf_file = str(target),
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = str(shard1),
gguf_hf_repo = None,
gguf_variant = None,
)
with patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0):
with self._dspark_capable():
gb = self.route._estimate_gguf_required_gb(cfg, speculative_type = "dspark")
self.assertAlmostEqual(gb, 9000 / (1024**3), places = 9)
@staticmethod
def _dflash_capable(supported = True):
"""Patches the probe so the DFlash sizing gate sees a binary able to run draft-dflash."""
from core.inference.llama_cpp import LlamaCppBackend
return patch.object(
LlamaCppBackend,
"probe_server_capabilities",
classmethod(lambda cls, binary = None: {"supports_dflash": supported}),
)
def test_auto_prices_the_mtp_sidecar_under_tensor_split(self):
"""#11308: Auto on tensor split launches the loadable MTP sidecar, so it is the one charged."""
import tempfile
import core.inference.llama_cpp as llama_cpp_module
with tempfile.TemporaryDirectory() as d:
p = Path(d)
target = p / "model.gguf"
sidecar = p / "dflash-kquant.gguf"
mtp = p / "MTP" / "mtp-model.gguf"
mtp.parent.mkdir()
target.write_bytes(b"x" * 2000)
sidecar.write_bytes(b"y" * 3000)
mtp.write_bytes(b"z" * 5000)
cfg = _gguf_cfg(
gguf_file = str(target),
gguf_dflash_file = str(sidecar),
gguf_mtp_file = str(mtp),
gguf_hf_repo = None,
gguf_variant = None,
)
from core.inference.llama_cpp import LlamaCppBackend
def _caps(mtp_token = "draft-mtp"):
return patch.object(
LlamaCppBackend,
"probe_server_capabilities",
classmethod(
lambda cls, binary = None: {"supports_dflash": True, "mtp_token": mtp_token}
),
)
def _estimate(
loads,
mtp_token = "draft-mtp",
**kw,
):
with (
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
patch.object(
llama_cpp_module, "_mtp_drafter_loads_standalone", lambda _p: loads
),
_caps(mtp_token),
):
return self.route._estimate_gguf_required_gb(cfg, speculative_type = "auto", **kw)
def _resident(**kw):
with (
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(
llama_cpp_module, "_mtp_drafter_loads_standalone", lambda _p: True
),
_caps(),
):
return self.route._gguf_resident_file_gb(cfg, speculative_type = "auto", **kw)
resident_tensor = _resident(tensor_parallel = True)
resident_layer = _resident()
tensor = _estimate(True, tensor_parallel = True)
tensor_extras = _estimate(True, llama_extra_args = ["-sm", "tensor"])
layer = _estimate(True)
unloadable = _estimate(False, tensor_parallel = True)
no_mtp_binary = _estimate(True, mtp_token = None, tensor_parallel = True)
self.assertAlmostEqual(tensor, 7000 / (1024**3), places = 9)
self.assertAlmostEqual(tensor_extras, 7000 / (1024**3), places = 9)
self.assertAlmostEqual(layer, 5000 / (1024**3), places = 9)
self.assertAlmostEqual(unloadable, 5000 / (1024**3), places = 9)
self.assertAlmostEqual(no_mtp_binary, 5000 / (1024**3), places = 9)
self.assertAlmostEqual(resident_tensor, 7000 / (1024**3), places = 9)
self.assertAlmostEqual(resident_layer, 5000 / (1024**3), places = 9)
def test_auto_drops_a_neox_dflash_charge_on_an_mrope_vision_load(self):
"""#13511: Auto launches no DFlash drafter that an image would break, so none is charged."""
import struct
import tempfile
def _header(path, arch, kv, size):
entries = [("general.architecture", arch), *kv.items()]
out = [struct.pack("<IIQQ", 0x46554747, 3, 0, len(entries))]
for key, val in entries:
out.append(struct.pack("<Q", len(key)) + key.encode())
if isinstance(val, str):
out.append(struct.pack("<IQ", 8, len(val)) + val.encode())
elif isinstance(val, bool):
out.append(struct.pack("<I?", 7, val))
else:
out.append(
struct.pack("<IIQ", 9, 5, len(val)) + struct.pack(f"<{len(val)}i", *val)
)
blob = b"".join(out)
path.write_bytes(blob + b"\0" * (size - len(blob)))
return str(path)
sections = [11, 11, 10, 0]
with tempfile.TemporaryDirectory() as d:
p = Path(d)
target = _header(
p / "model.gguf", "qwen35", {"qwen35.rope.dimension_sections": sections}, 2000
)
neox = _header(p / "dflash-neox.gguf", "dflash", {}, 3000)
mrope = _header(
p / "dflash-mrope.gguf",
"dflash",
{"dflash.rope.dimension_sections": sections},
3000,
)
mmproj = _header(p / "mmproj-F16.gguf", "clip", {"clip.has_vision_encoder": True}, 7000)
from core.inference.llama_cpp import LlamaCppBackend
def _estimate(dflash, **kw):
cfg = _gguf_cfg(
gguf_file = target,
gguf_dflash_file = dflash,
gguf_mmproj_file = mmproj,
gguf_hf_repo = None,
gguf_variant = None,
)
with (
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
patch.object(
LlamaCppBackend,
"probe_server_capabilities",
classmethod(lambda cls, binary = None: {"supports_dflash": True}),
),
):
return self.route._estimate_gguf_required_gb(cfg, **kw)
auto_neox = _estimate(neox, speculative_type = "auto")
auto_mrope = _estimate(mrope, speculative_type = "auto")
auto_neox_vision_off = _estimate(neox, speculative_type = "auto", disable_vision = True)
forced_neox = _estimate(neox, speculative_type = "dflash")
self.assertAlmostEqual(auto_neox, 9000 / (1024**3), places = 9)
self.assertAlmostEqual(auto_mrope, 12000 / (1024**3), places = 9)
self.assertAlmostEqual(auto_neox_vision_off, 5000 / (1024**3), places = 9)
self.assertAlmostEqual(forced_neox, 12000 / (1024**3), places = 9)
def test_extra_args_drafter_is_charged_once_when_it_is_the_local_sidecar(self):
"""A --model-draft naming the discovered sidecar is charged once, by resolved path."""
import os
import tempfile
with tempfile.TemporaryDirectory() as d:
p = Path(d)
target = p / "model.gguf"
sidecar = p / "dflash-kquant.gguf"
target.write_bytes(b"x" * 2000)
sidecar.write_bytes(b"y" * 3000)
link = p / "linked-dflash.gguf"
os.symlink(sidecar, link)
elsewhere = p / "other" / "dflash-elsewhere.gguf"
elsewhere.parent.mkdir()
elsewhere.write_bytes(b"z" * 4000)
cfg = _gguf_cfg(
gguf_file = str(target),
gguf_dflash_file = str(sidecar),
gguf_hf_repo = None,
gguf_variant = None,
)
with (
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
self._dflash_capable(),
):
plain = self.route._estimate_gguf_required_gb(cfg, speculative_type = "dflash")
same = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "dflash",
llama_extra_args = ["--model-draft", str(sidecar)],
)
through_link = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "dflash",
llama_extra_args = ["--model-draft", str(link)],
)
separate = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "dflash",
llama_extra_args = ["--model-draft", str(elsewhere)],
)
self.assertAlmostEqual(plain, 5000 / (1024**3), places = 9)
self.assertAlmostEqual(same, 5000 / (1024**3), places = 9)
self.assertAlmostEqual(through_link, 5000 / (1024**3), places = 9)
self.assertAlmostEqual(separate, 6000 / (1024**3), places = 9)
def test_extras_owning_spec_type_charge_only_their_own_drafter(self):
"""--spec-type in the extras ends _build_speculative_flags before discovery's sidecar, so
llama-server opens only the extras' --model-draft; charging both bills two drafters."""
import tempfile
with tempfile.TemporaryDirectory() as d:
p = Path(d)
target = p / "model.gguf"
sidecar = p / "dflash-kquant.gguf"
custom = p / "custom-dflash.gguf"
target.write_bytes(b"x" * 2000)
sidecar.write_bytes(b"y" * 3000)
custom.write_bytes(b"z" * 4000)
cfg = _gguf_cfg(
gguf_file = str(target),
gguf_dflash_file = str(sidecar),
gguf_hf_repo = None,
gguf_variant = None,
)
with (
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
self._dflash_capable(),
):
owned = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "auto",
llama_extra_args = [
"--spec-type",
"draft-dflash",
"--model-draft",
str(custom),
],
)
self.assertAlmostEqual(owned, 6000 / (1024**3), places = 9)
def test_remote_weights_stay_in_the_estimate_beside_a_local_extra_args_drafter(self):
"""Drafters are companions, not local weights, so they join the sizing branch."""
import tempfile
import utils.models.model_config as mc
cfg = _gguf_cfg()
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 10 * 1024**3)
with tempfile.TemporaryDirectory() as d:
drafter = Path(d) / "dflash-kquant.gguf"
drafter.write_bytes(b"y" * 3000)
with (
patch.object(mc, "list_gguf_variants", return_value = ([variant], False)),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_companion_bytes", return_value = 0),
self._dflash_capable(),
):
gb = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "dflash",
llama_extra_args = ["--model-draft", str(drafter)],
)
self.assertAlmostEqual(gb, 10.0 + 3000 / (1024**3), places = 9)
def test_remote_threads_token_and_adds_companions(self):
import utils.models.model_config as mc
cfg = SimpleNamespace(
gguf_file = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_hf_repo = "org/repo",
gguf_variant = "Q4_K_M",
)
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 10 * 1024**3)
captured = {}
def fake_list(repo, hf_token = None):
captured["token"] = hf_token
return ([variant], True)
with (
patch.object(mc, "list_gguf_variants", fake_list),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
patch.object(
self.route, "_remote_gguf_companion_bytes", return_value = 2 * 1024**3
) as comp,
self._dspark_capable(),
):
gb = self.route._estimate_gguf_required_gb(
cfg,
hf_token = "tok",
speculative_type = "dspark",
)
self.assertEqual(captured["token"], "tok")
self.assertAlmostEqual(gb, 12.0, places = 6)
self.assertTrue(comp.call_args.kwargs["include_mmproj"])
self.assertFalse(comp.call_args.kwargs["include_mtp"])
self.assertTrue(comp.call_args.kwargs["include_dspark"])
def test_remote_companions_choose_preferred_dspark_sidecar(self):
siblings = [
SimpleNamespace(rfilename = "mtp-model.gguf", size = 100),
SimpleNamespace(rfilename = "dspark/dspark-model-BF16.gguf", size = 300),
SimpleNamespace(rfilename = "dspark/dspark-model-Q8_0.gguf", size = 200),
SimpleNamespace(rfilename = "dflash-model-Q8_0.gguf", size = 400),
]
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
total = self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = "tok",
include_mmproj = False,
include_dspark = True,
)
self.assertEqual(total, 300)
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
dspark_only = self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = "tok",
include_mmproj = False,
include_mtp = False,
include_dspark = True,
)
self.assertEqual(dspark_only, 200)
def test_auto_charges_only_dspark_when_a_repo_publishes_both_sidecars(self):
"""Under Auto, a listed DSpark sidecar stands down the DFlash fetch, so only DSpark is charged."""
both = [
SimpleNamespace(rfilename = "dspark/dspark-model-Q8_0.gguf", size = 200),
SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 400),
]
def _companion_bytes(siblings, **kwargs):
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
return self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = False,
**kwargs,
)
self.assertEqual(
_companion_bytes(both, include_dspark = True, include_dflash = True, dspark_first = True),
200,
)
self.assertEqual(
_companion_bytes(
[both[1]], include_dspark = True, include_dflash = True, dspark_first = True
),
400,
)
self.assertEqual(_companion_bytes(both, include_dflash = True), 400)
def test_extras_owning_the_spec_type_are_not_charged_the_repo_sidecar(self):
"""With --spec-type set, no repo sidecar is launched; only the user's --model-draft is charged."""
import utils.models.model_config as mc
cfg = _gguf_cfg()
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 1024**3)
import tempfile
_tmp = tempfile.TemporaryDirectory()
self.addCleanup(_tmp.cleanup)
_draft = Path(_tmp.name) / "d.gguf"
_draft.write_bytes(b"x" * 512)
# A real sidecar so a stub returning 0 cannot make this pass.
_sidecar = 7 * 1024**3
def _companions(repo, **kw):
return (
_sidecar
if (kw["include_mtp"] or kw["include_dspark"] or kw["include_dflash"])
else 0
)
def _estimate(extras):
with (
patch.object(
mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)
),
patch.object(self.route, "_remote_gguf_companion_bytes", _companions),
self._dflash_capable(),
):
return self.route._estimate_gguf_required_gb(
cfg, speculative_type = "auto", llama_extra_args = extras
)
baseline = _estimate([])
self.assertGreater(baseline, _sidecar / (1024**3))
for extras in (
["--spec-type", "draft-dflash", "--model-draft", str(_draft)],
["--spec-type", "draft-dspark", "--model-draft", str(_draft)],
):
self.assertAlmostEqual(
_estimate(extras),
baseline - _sidecar / (1024**3) + 512 / (1024**3),
places = 9,
msg = extras,
)
def test_a_remote_extras_drafter_is_sized_from_its_own_repository(self):
"""--spec-draft-hf names a separate repo that loads beside the target, so its drafter must
be sized."""
import utils.models.model_config as mc
from core.inference.llama_cpp import LlamaCppBackend
cfg = _gguf_cfg()
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 1024**3)
siblings = [
SimpleNamespace(rfilename = "drafter-Q4_K_M-00001-of-00002.gguf", size = 3 * 1024**3 // 2),
SimpleNamespace(rfilename = "drafter-Q4_K_M-00002-of-00002.gguf", size = 3 * 1024**3 // 2),
SimpleNamespace(rfilename = "drafter-Q8_0.gguf", size = 2 * 1024**3),
SimpleNamespace(rfilename = "drafter-F16-00001-of-00002.gguf", size = 5 * 1024**3),
SimpleNamespace(rfilename = "notes.md", size = 9 * 1024**3),
]
def _estimate(extras):
with (
patch.object(
mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)
),
patch.object(self.route, "_remote_gguf_companion_bytes", return_value = 0),
patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
),
patch.object(
LlamaCppBackend,
"probe_server_capabilities",
classmethod(
lambda cls, binary = None: {
"supports_dspark": True,
"supports_dflash": True,
}
),
),
):
return self.route._estimate_gguf_required_gb(
cfg, speculative_type = "auto", llama_extra_args = extras
)
for own, remote in (
(["--spec-type", "draft-dspark"], ["--spec-draft-hf", "org/drafter"]),
(["--spec-type", "draft-dflash"], ["-hfd", "org/drafter"]),
):
self.assertAlmostEqual(
_estimate(own + remote),
_estimate(own) + 3 * 1024**3 / (1024**3),
places = 9,
msg = remote,
)
self.assertAlmostEqual(
_estimate(["--spec-type", "draft-dflash", "-hfd", "org/drafter:Q8_0"]),
_estimate(["--spec-type", "draft-dflash"]) + 2 * 1024**3 / (1024**3),
places = 9,
)
def test_an_unreadable_remote_drafter_repo_still_pays_a_flat_reserve(self):
"""An unreadable remote drafter pays a flat reserve; zero would admit an overcommit."""
import utils.models.model_config as mc
from core.inference.llama_cpp import LlamaCppBackend
cfg = _gguf_cfg()
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 1024**3)
reserve = self.route._REMOTE_DRAFTER_RESERVE_BYTES
self.assertGreater(reserve, 0)
def _estimate(extras, listing):
with (
patch.object(
mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)
),
patch.object(self.route, "_remote_gguf_companion_bytes", return_value = 0),
patch("huggingface_hub.model_info", **listing),
patch.object(
LlamaCppBackend,
"probe_server_capabilities",
classmethod(
lambda cls, binary = None: {
"supports_dspark": True,
"supports_dflash": True,
}
),
),
):
return self.route._estimate_gguf_required_gb(
cfg, speculative_type = "auto", llama_extra_args = extras
)
_raises = {"side_effect": OSError("gated repo")}
_empty = {"return_value": SimpleNamespace(siblings = [])}
base = _estimate(["--spec-type", "draft-dflash"], _raises)
for extras, listing in (
(["--spec-type", "draft-dflash", "-hfd", "org/drafter"], _raises),
(["--spec-type", "draft-dflash", "-hfd", "org/drafter"], _empty),
(["--spec-type", "draft-dflash", "-hfd", "not-a-repo-id"], _empty),
):
self.assertAlmostEqual(
_estimate(extras, listing),
base + reserve / (1024**3),
places = 9,
msg = extras,
)
def test_a_priced_remote_extras_drafter_is_not_charged_twice(self):
"""--spec-draft-hf is priced from its own listing; the target repo sidecar is never charged"""
import utils.models.model_config as mc
from core.inference.llama_cpp import LlamaCppBackend
cfg = _gguf_cfg()
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 1024**3)
for extras, kind in (
(["--spec-type", "draft-dspark", "--spec-draft-hf", "org/drafter"], "include_dspark"),
(["--spec-type", "draft-dflash", "-hfd", "org/drafter"], "include_dflash"),
):
with (
patch.object(
mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)
),
patch.object(self.route, "_remote_gguf_companion_bytes", return_value = 0) as comp,
patch.object(
LlamaCppBackend,
"probe_server_capabilities",
classmethod(
lambda cls, binary = None: {
"supports_dspark": True,
"supports_dflash": True,
}
),
),
):
self.route._estimate_gguf_required_gb(
cfg, speculative_type = "auto", llama_extra_args = extras
)
self.assertFalse(comp.call_args.kwargs[kind], extras)
self.assertFalse(comp.call_args.kwargs["include_mtp"], extras)
def test_the_unreadable_drafter_reserve_covers_the_largest_drafter_class(self):
"""The unlistable-drafter reserve must cover the largest class: --spec-draft-hf takes any repo."""
self.assertGreaterEqual(self.route._REMOTE_DRAFTER_RESERVE_BYTES, 11 * 1024**3)
def test_an_unlistable_remote_drafter_is_measured_from_the_local_cache(self):
"""An unlistable drafter is measured from the local HF cache, with whole-shard bounds."""
cached = SimpleNamespace(
repo_id = "org/drafter",
revisions = [
SimpleNamespace(
refs = {"main"},
last_modified = 2.0,
files = [
SimpleNamespace(
file_name = "big-00001-of-00002.gguf", size_on_disk = 16 * 1024**3
),
SimpleNamespace(
file_name = "big-00002-of-00002.gguf", size_on_disk = 14 * 1024**3
),
SimpleNamespace(file_name = "notes.txt", size_on_disk = 10),
],
),
SimpleNamespace(
refs = set(),
last_modified = 1.0,
files = [SimpleNamespace(file_name = "old-F16.gguf", size_on_disk = 60 * 1024**3)],
),
],
)
with (
patch("huggingface_hub.model_info", side_effect = OSError("no network")),
patch(
"huggingface_hub.scan_cache_dir",
return_value = SimpleNamespace(repos = [cached]),
),
):
charged = self.route._remote_drafter_repo_bytes("org/drafter", hf_token = None)
self.assertEqual(charged, 30 * 1024**3)
def test_a_local_model_draft_that_is_not_on_disk_is_not_priced_as_a_repo(self):
"""A --model-draft path that is not on disk costs nothing; it is not priced as a repo id."""
import utils.models.model_config as mc
cfg = _gguf_cfg()
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 4 * 1024**3)
with (
patch.object(mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)),
patch.object(self.route, "_remote_gguf_companion_bytes", return_value = 0),
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
patch("huggingface_hub.model_info", side_effect = AssertionError("priced as a repo")),
):
charged = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "auto",
llama_extra_args = ["--model-draft", "/nope/does-not-exist.gguf"],
)
self.assertAlmostEqual(charged, 4.0, places = 6)
def test_only_the_winning_draft_flag_decides_repo_or_path(self):
"""Draft flags are last-wins, so the winning flag alone decides repo or path."""
import utils.models.model_config as mc
cfg = _gguf_cfg()
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 4 * 1024**3)
with (
patch.object(mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)),
patch.object(self.route, "_remote_gguf_companion_bytes", return_value = 0),
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
patch("huggingface_hub.model_info", side_effect = AssertionError("priced as a repo")),
):
charged = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "auto",
llama_extra_args = [
"--spec-draft-hf",
"org/drafter",
"--model-draft",
"/nope/does-not-exist.gguf",
],
)
self.assertAlmostEqual(charged, 4.0, places = 6)
def test_the_cached_drafter_scan_reads_the_cache_studio_is_pointed_at(self):
"""The drafter cache scan uses the configured HF cache, not the import-time default."""
seen = {}
cached = SimpleNamespace(
repo_id = "org/drafter",
revisions = [
SimpleNamespace(
refs = {"main"},
last_modified = 2.0,
files = [
SimpleNamespace(file_name = "drafter-Q4_K_M.gguf", size_on_disk = 7 * 1024**3)
],
)
],
)
def fake_scan(cache_dir = None, **kwargs):
seen["cache_dir"] = cache_dir
return SimpleNamespace(repos = [cached])
with (
patch("huggingface_hub.model_info", side_effect = OSError("no network")),
patch("huggingface_hub.scan_cache_dir", side_effect = fake_scan),
patch("utils.hf_cache_settings.active_hf_hub_cache", return_value = "/elsewhere/hub"),
):
charged = self.route._remote_drafter_repo_bytes("org/drafter", hf_token = None)
self.assertEqual(seen["cache_dir"], "/elsewhere/hub")
self.assertEqual(charged, 7 * 1024**3)
def test_underscore_spelled_draft_flags_classify_as_remote(self):
"""Underscore spellings like --spec_draft_hf must classify as remote, not be compared as raw
tokens."""
import utils.models.model_config as mc
cfg = _gguf_cfg()
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 4 * 1024**3)
with (
patch.object(mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)),
patch.object(self.route, "_remote_gguf_companion_bytes", return_value = 0),
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
patch.object(self.route, "_remote_drafter_repo_bytes", return_value = 6 * 1024**3),
):
charged = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "auto",
llama_extra_args = ["--spec_draft_hf", "org/drafter"],
)
self.assertAlmostEqual(charged, 10.0, places = 6)
def test_a_listing_that_carries_no_sizes_still_pays_the_reserve(self):
"""A complete family with unlisted sizes still pays the reserve; it is unmeasured, not free."""
sizeless = SimpleNamespace(
siblings = [SimpleNamespace(rfilename = "drafter-Q4_K_M.gguf", size = None)]
)
with (
patch("huggingface_hub.model_info", return_value = sizeless),
patch("huggingface_hub.scan_cache_dir", return_value = SimpleNamespace(repos = [])),
):
charged = self.route._remote_drafter_repo_bytes("org/drafter", hf_token = None)
self.assertEqual(charged, self.route._REMOTE_DRAFTER_RESERVE_BYTES)
def test_extras_naming_their_own_drafter_do_not_also_pay_for_a_dflash_sidecar(self):
"""The extras' drafter is the one the loader launches, so the Auto promotion that would find
the sidecar never runs; billing both charges for weights only one loads."""
import utils.models.model_config as mc
cfg = _gguf_cfg()
variant = SimpleNamespace(
filename = "model-Q4_K_M.gguf", quant = "Q4_K_M", size_bytes = 10 * 1024**3
)
siblings = [SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 2 * 1024**3)]
with (
patch.object(mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)),
patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
),
patch.object(self.route, "_remote_drafter_repo_bytes", return_value = 3 * 1024**3),
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
self._dflash_capable(),
):
charged = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "auto",
llama_extra_args = ["--spec-draft-hf", "org/drafter"],
)
self.assertAlmostEqual(charged, 13.0, places = 6)
def test_a_split_drafter_sized_in_part_is_not_charged_its_known_half(self):
"""A partly sized split drafter is charged by cache or reserve, never its known half."""
partial_sizes = SimpleNamespace(
siblings = [
SimpleNamespace(rfilename = "drafter-00001-of-00002.gguf", size = 3 * 1024**3),
SimpleNamespace(rfilename = "drafter-00002-of-00002.gguf", size = None),
]
)
with (
patch("huggingface_hub.model_info", return_value = partial_sizes),
patch("huggingface_hub.scan_cache_dir", return_value = SimpleNamespace(repos = [])),
patch("utils.hf_cache_settings.active_hf_hub_cache", return_value = "/hub"),
):
charged = self.route._remote_drafter_repo_bytes("org/drafter", hf_token = None)
self.assertEqual(charged, self.route._REMOTE_DRAFTER_RESERVE_BYTES)
def test_a_remote_drafter_that_is_neither_listable_nor_cached_pays_the_reserve(self):
"""With the listing refused and nothing cached, the reserve is a cushion, not a bound."""
with (
patch("huggingface_hub.model_info", side_effect = OSError("no network")),
patch("huggingface_hub.scan_cache_dir", return_value = SimpleNamespace(repos = [])),
):
charged = self.route._remote_drafter_repo_bytes("org/drafter", hf_token = None)
self.assertEqual(charged, self.route._REMOTE_DRAFTER_RESERVE_BYTES)
def test_a_cached_drafter_is_narrowed_by_the_quant_tag(self):
"""llama.cpp's :quant is its own narrowing, so a repo with several quants must not be
charged its F16 for a Q4_K_M request; the listing path applies the same rule."""
cached = SimpleNamespace(
repo_id = "org/drafter",
revisions = [
SimpleNamespace(
files = [
SimpleNamespace(file_name = "drafter-F16.gguf", size_on_disk = 20 * 1024**3),
SimpleNamespace(file_name = "drafter-Q4_K_M.gguf", size_on_disk = 3 * 1024**3),
]
)
],
)
with (
patch("huggingface_hub.model_info", side_effect = OSError("no network")),
patch(
"huggingface_hub.scan_cache_dir",
return_value = SimpleNamespace(repos = [cached]),
),
):
charged = self.route._remote_drafter_repo_bytes("org/drafter:Q4_K_M", hf_token = None)
self.assertEqual(charged, 3 * 1024**3)
def test_a_cpu_offloaded_extras_drafter_is_not_charged_vram(self):
"""-ngld 0 keeps the drafter in host memory, so charging it to the training job's VRAM is
the wrong resource and 409s a load that takes no VRAM for it."""
import utils.models.model_config as mc
cfg = _gguf_cfg()
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 1024**3)
extras = ["--spec-type", "draft-dspark", "--spec-draft-hf", "org/drafter"]
with (
patch.object(mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)),
patch.object(self.route, "_remote_gguf_companion_bytes", return_value = 0),
patch.object(self.route, "_remote_drafter_repo_bytes", return_value = 8 * 1024**3),
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
):
on_gpu = self.route._estimate_gguf_required_gb(
cfg, speculative_type = "auto", llama_extra_args = extras
)
on_cpu = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "auto",
llama_extra_args = [*extras, "--spec-draft-ngl", "0"],
)
self.assertAlmostEqual(on_gpu - on_cpu, 8.0, places = 6)
def test_a_bare_model_draft_overrides_the_repository_sidecar(self):
"""A bare --model-draft overrides the repo sidecar, so only one drafter is charged."""
import tempfile
import utils.models.model_config as mc
with tempfile.TemporaryDirectory() as d:
target = Path(d) / "model.gguf"
sidecar = Path(d) / "dspark-model-Q8_0.gguf"
custom = Path(d) / "elsewhere.gguf"
target.write_bytes(b"t" * 2000)
sidecar.write_bytes(b"s" * 3000)
custom.write_bytes(b"c" * 4000)
cfg = _gguf_cfg(
gguf_file = str(target),
gguf_dspark_file = str(sidecar),
gguf_hf_repo = None,
gguf_variant = None,
)
with (
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
self._dspark_capable(),
):
charged = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "auto",
llama_extra_args = ["--model-draft", str(custom)],
)
self.assertAlmostEqual(charged, 6000 / (1024**3), places = 9)
def test_a_cpu_pinned_discovered_sidecar_is_not_charged_vram(self):
"""A -ngld 0 drafter runs host-resident, so it is not charged against training VRAM."""
import tempfile
with tempfile.TemporaryDirectory() as d:
target = Path(d) / "model.gguf"
sidecar = Path(d) / "dspark-model-Q8_0.gguf"
target.write_bytes(b"t" * 2000)
sidecar.write_bytes(b"s" * 3000)
cfg = _gguf_cfg(
gguf_file = str(target),
gguf_dspark_file = str(sidecar),
gguf_hf_repo = None,
gguf_variant = None,
)
with (
patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 0.0),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
self._dspark_capable(),
):
on_gpu = self.route._estimate_gguf_required_gb(cfg, speculative_type = "auto")
on_cpu = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "auto",
llama_extra_args = ["--spec-draft-ngl", "0"],
)
self.assertAlmostEqual(on_gpu, 5000 / (1024**3), places = 9)
self.assertAlmostEqual(on_cpu, 2000 / (1024**3), places = 9)
def test_a_drafter_repo_with_only_a_partial_set_is_charged_nothing(self):
"""The bound is zero when every family is an incomplete split, since the fetch refuses them
all; the unreadable-listing reserve would 409 a load for VRAM nothing takes."""
partial = SimpleNamespace(
siblings = [SimpleNamespace(rfilename = "drafter-00001-of-00002.gguf", size = 4 * 1024**3)]
)
with patch("huggingface_hub.model_info", return_value = partial):
charged = self.route._remote_drafter_repo_bytes("org/drafter", hf_token = None)
self.assertEqual(charged, 0)
def test_a_partial_dflash_shard_set_is_not_charged(self):
"""The fetch refuses a family whose encoded shard count is short, so a listing caught
mid-publication must not bill its listed half; the DSpark path filters the same way."""
partial = [
SimpleNamespace(rfilename = "dflash-kquant-00001-of-00002.gguf", size = 400),
]
whole = partial + [
SimpleNamespace(rfilename = "dflash-kquant-00002-of-00002.gguf", size = 300),
]
def _companion_bytes(siblings):
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
return self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = False,
include_dflash = True,
)
self.assertEqual(_companion_bytes(partial), 0)
self.assertEqual(_companion_bytes(whole), 700)
def test_auto_tells_the_companion_sizing_that_dspark_comes_first(self):
"""Both kinds can be asked for remotely, so the caller passes the Auto rule down."""
import utils.models.model_config as mc
cfg = _gguf_cfg()
variant = SimpleNamespace(quant = "Q4_K_M", size_bytes = 1024**3)
with (
patch.object(mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)),
patch.object(self.route, "_remote_gguf_companion_bytes", return_value = 0) as comp,
self._dspark_capable(),
):
self.route._estimate_gguf_required_gb(cfg, speculative_type = "auto")
self.assertTrue(comp.call_args.kwargs["dspark_first"])
self.route._estimate_gguf_required_gb(cfg, speculative_type = "dflash")
self.assertFalse(comp.call_args.kwargs["dspark_first"])
_MULTI_FAMILY_SIBLINGS = [
SimpleNamespace(rfilename = "model-A-Q4_K_M.gguf", size = 10 * 1024**3),
SimpleNamespace(rfilename = "model-B-Q4_K_M.gguf", size = 10 * 1024**3),
SimpleNamespace(rfilename = "dflash-model-A-Q8_0.gguf", size = 1024**3),
SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 4 * 1024**3),
]
def test_remote_dflash_sizing_bounds_every_candidate_the_fallback_can_reach(self):
"""Remote DFlash sizing bounds each candidate; listings give no readable headers."""
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = self._MULTI_FAMILY_SIBLINGS),
):
total = self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = False,
include_dflash = True,
)
self.assertEqual(total, 4 * 1024**3)
def test_remote_dflash_sizing_totals_every_shard_of_a_split_sidecar(self):
"""A split DFlash sidecar is sized as the sum of every shard, since the whole set stays resident."""
siblings = [
SimpleNamespace(rfilename = "model-Q4_K_M.gguf", size = 10 * 1024**3),
SimpleNamespace(rfilename = "dflash-split-00001-of-00002.gguf", size = 1024**3),
SimpleNamespace(rfilename = "dflash-split-00002-of-00002.gguf", size = 1024**3),
SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 3 * 1024**3 // 2),
]
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
total = self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = False,
include_dflash = True,
)
self.assertEqual(total, 2 * 1024**3)
def test_remote_dflash_sizing_charges_a_split_set_once(self):
"""Every shard is a listed dflash- name, so summing candidates instead of taking the safe
maximum across shard sets would double-charge this repo and 409 a load that fits."""
siblings = [
SimpleNamespace(rfilename = "model-Q4_K_M.gguf", size = 10 * 1024**3),
SimpleNamespace(rfilename = "dflash-split-00001-of-00002.gguf", size = 1024**3),
SimpleNamespace(rfilename = "dflash-split-00002-of-00002.gguf", size = 1024**3),
]
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
total = self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = False,
include_dflash = True,
)
self.assertEqual(total, 2 * 1024**3)
def test_remote_dflash_sizing_ignores_a_nested_dflash_named_weight(self):
"""The picker is root level only, so a dflash GGUF under quants/ is an ordinary weight,
never the drafter; charging it tracks a file the load cannot reach."""
siblings = [
SimpleNamespace(rfilename = "model-Q4_K_M.gguf", size = 10 * 1024**3),
SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 1024**3),
SimpleNamespace(rfilename = "quants/dflash-model-Q8_0.gguf", size = 9 * 1024**3),
]
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
total = self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = False,
include_dflash = True,
)
self.assertEqual(total, 1024**3)
def test_remote_estimate_bounds_the_dflash_fallback_end_to_end(self):
"""The guard's estimate must share the bound, or a multi-family repo is under-charged."""
import utils.models.model_config as mc
cfg = _gguf_cfg()
variant = SimpleNamespace(
filename = "model-A-Q4_K_M.gguf", quant = "Q4_K_M", size_bytes = 10 * 1024**3
)
with (
patch.object(mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = self._MULTI_FAMILY_SIBLINGS),
),
self._dflash_capable(),
):
gb = self.route._estimate_gguf_required_gb(cfg, speculative_type = "dflash")
self.assertAlmostEqual(gb, 14.0, places = 6)
def test_auto_does_not_charge_dflash_when_extra_args_own_speculation(self):
"""Extra args setting --spec-type stop Auto promotion, so the sidecar never opens; charging
it would 409 a chat load for ~1.5 GiB, while extra draft-dflash still pays."""
import utils.models.model_config as mc
cfg = _gguf_cfg()
variant = SimpleNamespace(
filename = "model-Q4_K_M.gguf", quant = "Q4_K_M", size_bytes = 10 * 1024**3
)
siblings = [SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 2 * 1024**3)]
with (
patch.object(mc, "list_gguf_variants", lambda repo, hf_token = None: ([variant], False)),
patch.object(self.route, "_remote_gguf_compute_reserve_gb", return_value = 0.0),
patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
),
self._dflash_capable(),
):
owned = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "auto",
llama_extra_args = ["--spec-type", "ngram-mod"],
)
asked = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "auto",
llama_extra_args = ["--spec-type", "draft-dflash"],
)
forced = self.route._estimate_gguf_required_gb(
cfg,
speculative_type = "dflash",
llama_extra_args = ["--spec-type", "ngram-mod"],
)
self.assertAlmostEqual(owned, 10.0, places = 6)
self.assertAlmostEqual(asked, 12.0, places = 6)
self.assertAlmostEqual(forced, 10.0, places = 6)
def test_remote_dflash_sizing_drops_a_candidate_too_big_to_be_a_drafter(self):
"""Root dflash GGUF is refused when oversized, so charging it would 409 unloaded bytes."""
siblings = [
SimpleNamespace(rfilename = "model-Q4_K_M.gguf", size = 10 * 1024**3),
SimpleNamespace(rfilename = "dflash-model-BF16.gguf", size = 40 * 1024**3),
SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 1024**3),
]
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
charged = self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = False,
include_dflash = True,
weight_bytes = 10 * 1024**3,
)
self.assertEqual(charged, 1024**3)
def test_remote_dspark_sizing_totals_every_shard_of_a_split_sidecar(self):
"""llama-server maps every shard, so pricing only the ranked pick halved a two-shard sidecar
and let the guard admit a load that evicts the training run it protects."""
siblings = [
SimpleNamespace(rfilename = "dspark/dspark-00001-of-00002.gguf", size = 5 * 1024**3),
SimpleNamespace(rfilename = "dspark/dspark-00002-of-00002.gguf", size = 5 * 1024**3),
]
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
charged = self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = False,
include_dspark = True,
)
self.assertEqual(charged, 10 * 1024**3)
def test_auto_budgets_dflash_when_the_dspark_set_is_incomplete(self):
"""A listing missing a DSpark shard is never loaded: the fetch refuses it and falls through
to DFlash, which can be larger, so granting DSpark first refusal under-charged."""
siblings = [
SimpleNamespace(rfilename = "dspark/dspark-00001-of-00002.gguf", size = 1024**3),
SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 4 * 1024**3),
]
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
charged = self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = False,
include_dspark = True,
include_dflash = True,
dspark_first = True,
)
self.assertEqual(charged, 4 * 1024**3)
def _auto_companion_bytes(self, siblings):
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
return self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = True,
include_dspark = True,
include_dflash = True,
dspark_first = True,
)
def test_auto_does_not_charge_the_mtp_drafter_dflash_replaces(self):
"""Under Auto DFlash replaces the MTP drafter, so charge the larger one, not the sum."""
siblings = [
SimpleNamespace(rfilename = "mtp-model.gguf", size = 1024**3),
SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 3 * 1024**3),
]
self.assertEqual(self._auto_companion_bytes(siblings), 3 * 1024**3)
def test_auto_keeps_the_mtp_charge_when_the_dflash_candidates_may_all_fail(self):
"""If every DFlash candidate may be rejected, MTP is kept, so charge the larger of the two."""
siblings = [
SimpleNamespace(rfilename = "mtp-model.gguf", size = 5 * 1024**3),
SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 1024**3),
]
self.assertEqual(self._auto_companion_bytes(siblings), 5 * 1024**3)
def test_auto_charges_the_largest_reachable_dflash_against_the_mtp_drafter(self):
"""Under Auto, the largest reachable DFlash is compared against MTP, not summed or first-ranked."""
siblings = [
SimpleNamespace(rfilename = "mtp-model.gguf", size = 2 * 1024**3),
*self._MULTI_FAMILY_SIBLINGS,
]
self.assertEqual(self._auto_companion_bytes(siblings), 4 * 1024**3)
def test_auto_charges_dspark_alone_over_both_of_the_others(self):
"""A listed DSpark sidecar settles the load under Auto, so MTP and DFlash are not resident."""
siblings = [
SimpleNamespace(rfilename = "mtp-model.gguf", size = 1024**3),
SimpleNamespace(rfilename = "dspark/dspark-model-Q8_0.gguf", size = 2 * 1024**3),
SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 3 * 1024**3),
]
self.assertEqual(self._auto_companion_bytes(siblings), 2 * 1024**3)
def test_auto_still_charges_the_mtp_drafter_when_the_repo_ships_no_sidecar(self):
"""Control: with nothing to promote, Auto launches MTP and keeps its charge."""
siblings = [
SimpleNamespace(rfilename = "mtp-model.gguf", size = 1024**3),
SimpleNamespace(rfilename = "model-Q4_K_M.gguf", size = 10 * 1024**3),
]
self.assertEqual(self._auto_companion_bytes(siblings), 1024**3)
def test_an_explicit_request_is_not_the_auto_race(self):
"""With dspark_first off, a caller's narrowed kinds keep every charge it passed in."""
siblings = [
SimpleNamespace(rfilename = "mtp-model.gguf", size = 1024**3),
SimpleNamespace(rfilename = "dflash-kquant.gguf", size = 3 * 1024**3),
]
with patch(
"huggingface_hub.model_info",
return_value = SimpleNamespace(siblings = siblings),
):
total = self.route._remote_gguf_companion_bytes(
"org/repo",
hf_token = None,
include_mmproj = False,
include_mtp = True,
include_dflash = True,
)
self.assertEqual(total, 4 * 1024**3)
def test_native_drafter_accept_applies_the_lease_before_the_scan_reads(self):
"""The lease must apply before from_identifier's discovery scan reads a symlinked DFlash header."""
import os
import tempfile
with tempfile.TemporaryDirectory() as d:
leased = Path(d) / "leased"
leased.mkdir()
outside = Path(d) / "outside"
outside.mkdir()
weight = leased / "model-Q4_K_M.gguf"
weight.write_bytes(b"x")
inside = leased / "dflash-kquant.gguf"
inside.write_bytes(b"y")
target = outside / "dflash-escape.gguf"
target.write_bytes(b"z")
escape = leased / "dflash-escape.gguf"
os.symlink(target, escape)
accept = self.route._native_drafter_accept
self.assertTrue(accept(str(inside), str(weight), "dflash", str(leased)))
self.assertFalse(accept(str(target.resolve()), str(weight), "dflash", str(leased)))
def test_remote_unknown_variant_returns_none(self):
import utils.models.model_config as mc
cfg = SimpleNamespace(
gguf_file = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_hf_repo = "org/repo",
gguf_variant = "Q8_0",
)
with patch.object(
mc,
"list_gguf_variants",
return_value = ([SimpleNamespace(quant = "Q4_K_M", size_bytes = 1)], False),
):
self.assertIsNone(self.route._estimate_gguf_required_gb(cfg))
def test_local_adds_kv_cache(self):
import tempfile
with tempfile.TemporaryDirectory() as d:
p = Path(d) / "model.gguf"
p.write_bytes(b"x" * 1000)
cfg = SimpleNamespace(
gguf_file = str(p),
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_hf_repo = None,
gguf_variant = None,
)
with patch.object(self.route, "_estimate_gguf_kv_gb", return_value = 2.0):
gb = self.route._estimate_gguf_required_gb(cfg, max_seq_length = 8192)
self.assertAlmostEqual(gb, 1000 / (1024**3) + 2.0, places = 6)
def test_kv_helper_graceful_on_non_gguf(self):
import tempfile
with tempfile.TemporaryDirectory() as d:
p = Path(d) / "not-a.gguf"
p.write_bytes(b"not a gguf")
self.assertEqual(self.route._estimate_gguf_kv_gb(str(p), 4096), 0.0)
def test_kv_sizes_at_larger_of_max_seq_len_and_ctx_override(self):
seen = {}
class _FakeBackend:
_context_length = 2048
supports_kv_unified = True
def _read_gguf_metadata(self, path):
pass
def _can_estimate_kv(self):
return True
@classmethod
def probe_server_capabilities(cls):
return {"supports_kv_unified": cls.supports_kv_unified}
def _estimate_kv_cache_bytes(
self,
ctx,
cache_type = None,
n_parallel = 1,
swa_full = False,
kv_unified = False,
n_ubatch = None,
ctx_checkpoints = 0,
flash_attn = True,
):
seen["ctx"] = ctx
seen["ctx_checkpoints"] = ctx_checkpoints
seen["cache_type"] = cache_type
seen["n_parallel"] = n_parallel
seen["swa_full"] = swa_full
seen["kv_unified"] = kv_unified
seen["n_ubatch"] = n_ubatch
seen["flash_attn"] = flash_attn
return ctx * n_parallel * (1024**2)
_PIPELINE_PER_DEVICE_OVERHEAD_MIB = 0
# zeroed: this test pins the kv sizing, not the compute buffers
def _estimate_compute_buffer_bytes(
self,
*,
n_ubatch = None,
n_parallel = 1,
per_device_tensor = False,
vocab_ceiling = None,
):
seen["compute_n_ubatch"] = n_ubatch
return 0
def _compute_buffer_ctx_bytes(
self,
n_ctx,
n_ubatch = None,
cache_type_kv = None,
*,
layer_split = False,
flash_attn = True,
n_parallel = 1,
):
return 0
with patch.object(self.route, "LlamaCppBackend", _FakeBackend):
r = self.route
self.assertAlmostEqual(
r._estimate_gguf_kv_gb("m", 4096, ["--ctx-size", "131072"]), 128.0
)
self.assertEqual(seen["ctx"], 131072)
self.assertEqual(seen["n_parallel"], 1)
self.assertFalse(seen["swa_full"])
# load_model always adds --flash-attn on when the build supports it.
self.assertTrue(seen["flash_attn"])
self.assertAlmostEqual(r._estimate_gguf_kv_gb("m", 4096, ["--ctx-size", "1024"]), 4.0)
self.assertEqual(seen["ctx"], 4096)
self.assertAlmostEqual(r._estimate_gguf_kv_gb("m", 0, None), 2.0)
self.assertEqual(seen["ctx"], 2048)
self.assertAlmostEqual(r._estimate_gguf_kv_gb("m", 4096, ["--ctx-size", "oops"]), 4.0)
self.assertAlmostEqual(r._estimate_gguf_kv_gb("m", 4096, None, 4), 16.0)
self.assertEqual(seen["n_parallel"], 4)
self.assertTrue(seen["kv_unified"])
r._estimate_gguf_kv_gb("m", 4096, ["--no-kv-unified"], 4)
self.assertFalse(seen["kv_unified"])
_FakeBackend.supports_kv_unified = False
r._estimate_gguf_kv_gb("m", 4096, None, 4)
self.assertFalse(seen["kv_unified"])
r._estimate_gguf_kv_gb("m", 4096, None, 1, "f32")
self.assertEqual(seen["cache_type"], "f32")
r._estimate_gguf_kv_gb("m", 4096, ["--cache-type-v", "f32"])
self.assertEqual(seen["cache_type"], "f32")
with patch.dict(self.route.os.environ, {"LLAMA_ARG_CACHE_TYPE_K": "f32"}):
r._estimate_gguf_kv_gb("m", 4096)
self.assertEqual(seen["cache_type"], "f32")
with patch.dict(
self.route.os.environ,
{
"LLAMA_ARG_CACHE_TYPE_K": "q4_0",
"LLAMA_ARG_CACHE_TYPE_V": "q4_0",
},
):
r._estimate_gguf_kv_gb("m", 4096)
self.assertEqual(seen["cache_type"], "q4_0")
r._estimate_gguf_kv_gb(
"m",
4096,
["--cache-type-k", "q4_0", "--cache-type-v", "q4_0"],
tensor_parallel = True,
)
self.assertEqual(seen["cache_type"], "q4_0")
r._estimate_gguf_kv_gb(
"m",
4096,
["--cache-type-k", "f32", "--cache-type-v", "q4_0"],
tensor_parallel = True,
)
self.assertEqual(seen["cache_type"], "f32")
r._estimate_gguf_kv_gb("m", 4096, ["--swa_full"])
self.assertTrue(seen["swa_full"])
r._estimate_gguf_kv_gb("m", 4096, ["--kv_unified", "--ubatch_size", "256"])
self.assertTrue(seen["kv_unified"])
self.assertEqual(seen["n_ubatch"], 256)
class TestLoadModelGuardIntegration(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.route = _load_inference_route()
def test_cuda_gpu_ids_allow_matching_draft_device(self):
self.route._reject_draft_device_with_gpu_ids(
[1],
["--spec-draft-device", "CUDA0"],
gpu_ids_are_vulkan_ordinals = False,
)
def test_vulkan_gpu_ids_reject_underscore_draft_device_alias(self):
with self.assertRaises(HTTPException) as exc:
self.route._reject_draft_device_with_gpu_ids(
[0],
["--spec_draft_device", "Vulkan1"],
gpu_ids_are_vulkan_ordinals = True,
)
self.assertEqual(exc.exception.status_code, 400)
self.assertIn("Vulkan1", exc.exception.detail)
def test_refusal_409_and_no_unload(self):
import contextlib
from unittest.mock import MagicMock
from models.inference import LoadRequest
inf = SimpleNamespace(active_model_name = None)
inf.unload_model = MagicMock()
inf._shutdown_subprocess = MagicMock()
llama = SimpleNamespace(is_loaded = False, model_identifier = None, hf_variant = None)
llama.unload_model = MagicMock()
cfg = SimpleNamespace(
is_gguf = False,
is_lora = False,
path = None,
base_model = None,
identifier = "unsloth/Qwen3-1.7B",
)
request = LoadRequest(model_path = "unsloth/Qwen3-1.7B")
info = {"required_gb": 40.0, "usable_gb": 5.0, "needed_gb": 50.0, "mode": "auto"}
with (
patch("utils.transformers_version.latest_tier_active_for", return_value = False),
patch.object(self.route, "validate_extra_args", return_value = None),
patch.object(
self.route,
"_resolve_model_identifier_for_request",
return_value = ("unsloth/Qwen3-1.7B", "unsloth/Qwen3-1.7B", False),
),
patch.object(self.route, "resolve_effective_chat_template_override", return_value = None),
patch.object(self.route, "get_inference_backend", return_value = inf),
patch.object(self.route, "get_llama_cpp_backend", return_value = llama),
patch.object(
self.route, "_hf_offline_if_unreachable", lambda: contextlib.nullcontext()
),
patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg),
_stub_guard_deps(training_active = True, decision = (False, info)),
):
with self.assertRaises(HTTPException) as exc:
asyncio.run(
self.route.load_model(request, fastapi_request = MagicMock(), current_subject = "u")
)
self.assertEqual(exc.exception.status_code, 409)
inf.unload_model.assert_not_called()
inf._shutdown_subprocess.assert_not_called()
llama.unload_model.assert_not_called()
def test_gguf_inherited_draft_device_rejected_under_vulkan_gpu_ids(self):
import contextlib
from unittest.mock import MagicMock
from models.inference import LoadRequest
inf = SimpleNamespace(active_model_name = None)
inf.unload_model = MagicMock()
inf._shutdown_subprocess = MagicMock()
llama = SimpleNamespace(
is_loaded = True,
model_identifier = "x.gguf",
hf_variant = None,
hf_repo = None,
gguf_path = None,
extra_args = ["--spec-draft-device", "CUDA1"],
extra_args_source = ("x.gguf", None),
last_load_intent = None,
layer_preserves_tensor_intent = False,
is_vulkan_build = lambda: True,
adopt_load_intent_if_matched = lambda intent: False,
)
llama.unload_model = MagicMock()
cfg = _model_cfg(is_gguf = True, identifier = "x.gguf", display_name = "x", gguf_variant = None)
request = LoadRequest(model_path = "x.gguf", gpu_ids = [0], max_seq_length = 4096)
captured = []
with (
patch("utils.transformers_version.latest_tier_active_for", return_value = False),
patch.object(
self.route,
"validate_extra_args",
side_effect = lambda args: list(args) if args else None,
),
patch.object(
self.route,
"_resolve_model_identifier_for_request",
return_value = ("x.gguf", "x.gguf", False),
),
patch.object(self.route, "resolve_effective_chat_template_override", return_value = None),
patch.object(
self.route,
"_resolve_gguf_gpu_ids_for_request",
return_value = ([0], True),
),
patch.object(self.route, "get_inference_backend", return_value = inf),
patch.object(self.route, "get_llama_cpp_backend", return_value = llama),
patch.object(
self.route, "_hf_offline_if_unreachable", lambda: contextlib.nullcontext()
),
patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg),
_stub_guard_deps(training_active = True, decision = (True, {}), captured = captured),
):
with self.assertRaises(HTTPException) as exc:
asyncio.run(
self.route.load_model(request, fastapi_request = MagicMock(), current_subject = "u")
)
self.assertEqual(exc.exception.status_code, 400)
self.assertIn("draft-model device", exc.exception.detail)
self.assertIn("set it to none", exc.exception.detail)
self.assertEqual(captured, [])
inf.unload_model.assert_not_called()
llama.unload_model.assert_not_called()
if __name__ == "__main__":
unittest.main()