Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Load-Model memory estimate tests (POST /api/inference/estimate-memory) on synthetic GGUF headers."""
import inspect
import json
import os
import sys
import types as _types
from pathlib import Path
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
# Stub needs get_logger: process-wide setdefault, and freshness_flow calls it at import.
_structlog_stub = _types.ModuleType("structlog")
_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
sys.modules.setdefault("structlog", _structlog_stub)
# Only stub httpx when missing; otherwise huggingface_hub.errors imports break.
try:
import httpx as _httpx_real # noqa: F401
except ImportError:
_httpx_stub = _types.ModuleType("httpx")
for _exc_name in (
"ConnectError",
"TimeoutException",
"ReadTimeout",
"ReadError",
"RemoteProtocolError",
"CloseError",
"HTTPError",
"RequestError",
):
setattr(_httpx_stub, _exc_name, type(_exc_name, (Exception,), {}))
class _FakeTimeout:
def __init__(self, *a, **kw):
pass
_httpx_stub.Timeout = _FakeTimeout
_httpx_stub.Response = type("Response", (), {})
_httpx_stub.Client = type(
"Client",
(),
{
"__init__": lambda self, **kw: None,
"__enter__": lambda self: self,
"__exit__": lambda self, *a: None,
},
)
sys.modules["httpx"] = _httpx_stub
import asyncio # noqa: E402
from types import SimpleNamespace # noqa: E402
import routes.inference as ri # noqa: E402
from models.inference import EstimateMemoryRequest # noqa: E402
from utils.models.model_config import ModelConfig # noqa: E402
# Load the real GGUF builder by path; `tests` is not importable on every runner.
import importlib.util as _ilu # noqa: E402
_kv_spec = _ilu.spec_from_file_location(
"_kv_cache_estimation_for_memory_estimate",
Path(__file__).resolve().parent / "test_kv_cache_estimation.py",
)
_kv_mod = _ilu.module_from_spec(_kv_spec)
_kv_spec.loader.exec_module(_kv_mod)
_make_gguf_bytes = _kv_mod._make_gguf_bytes
_GIB = 1024**3
@pytest.fixture(autouse = True)
def _clear_estimate_caches():
"""Both module caches outlive a test, so clear them each time or stale entries satisfy assertions."""
ri._estimate_files_cache.clear()
ri._estimate_config_cache.clear()
yield
ri._estimate_files_cache.clear()
ri._estimate_config_cache.clear()
# No sliding window, so the SWA resolver (which can reach HF) never fires.
_GQA_FIELDS = {
"context_length": 8192,
"block_count": 12,
"attention.head_count": 8,
"attention.head_count_kv": 4,
"attention.key_length": 64,
"attention.value_length": 64,
"embedding_length": 512,
}
def _write_gguf(
tmp_path: Path,
arch: str,
fields: dict,
name: str = "model.gguf",
) -> str:
kv = {"general.architecture": arch}
kv.update({f"{arch}.{k}": v for k, v in fields.items()})
path = tmp_path / name
path.write_bytes(_make_gguf_bytes(arch, kv))
return str(path)
def _write_gguf_with_embeddings(
tmp_path: Path,
arch: str,
fields: dict,
name: str = "model.gguf",
) -> str:
"""Carries token_embd.weight like real weights; metadata-only drafts are refused, not charged."""
import numpy as np
from gguf import GGUFWriter
path = tmp_path / name
writer = GGUFWriter(str(path), arch)
for key, value in fields.items():
writer.add_uint32(f"{arch}.{key}", int(value))
writer.add_tensor("token_embd.weight", np.zeros((64, 64), dtype = np.float32))
writer.write_header_to_file()
writer.write_kv_data_to_file()
writer.write_tensors_to_file()
writer.close()
return str(path)
@pytest.fixture
def gqa_gguf(tmp_path) -> str:
return _write_gguf(tmp_path, "qwen3", _GQA_FIELDS)
@pytest.fixture
def dimless_gguf(tmp_path) -> str:
"""A layer count with no KV dims, as real truncated GGUFs have; why kv_estimable is its own field."""
return _write_gguf(tmp_path, "qwen3", {"block_count": 12}, name = "dimless.gguf")
class TestOffloadedLayerFraction:
"""Share of the weights the requested offload keeps on the GPU."""
def test_auto_is_full(self):
assert ri._gguf_offloaded_layer_fraction("auto", None, 12) == 1.0
assert ri._gguf_offloaded_layer_fraction(None, 4, 12) == 1.0
@pytest.mark.parametrize("gpu_layers", [None, -1])
def test_manual_without_a_layer_count_is_full(self, gpu_layers):
assert ri._gguf_offloaded_layer_fraction("manual", gpu_layers, 12) == 1.0
@pytest.mark.parametrize("layer_count", [None, 0])
def test_unknown_layer_count_is_full(self, layer_count):
assert ri._gguf_offloaded_layer_fraction("manual", 20, layer_count) == 1.0
def test_manual_split_uses_block_count_plus_one(self):
# +1 is the output layer, matching the UI slider ceiling.
assert ri._gguf_offloaded_layer_fraction("manual", 20, 65) == pytest.approx(20 / 66)
def test_over_large_gpu_layers_clamps(self):
assert ri._gguf_offloaded_layer_fraction("manual", 999, 65) == 1.0
def test_zero_layers_is_cpu_only(self):
assert ri._gguf_offloaded_layer_fraction("manual", 0, 65) == 0.0
def test_a_layer_count_in_the_extras_wins_in_either_mode(self):
# Auto emits -ngl -1 then extras, so a user count last-wins at the child.
assert ri._gguf_offloaded_layer_fraction("auto", None, 12, ["-ngl", "0"]) == 0.0
assert ri._gguf_offloaded_layer_fraction(
"auto", None, 12, ["--gpu-layers", "6"]
) == pytest.approx(6 / 13)
assert ri._gguf_offloaded_layer_fraction("auto", None, 12, ["-ngl", "-1"]) == 1.0
assert ri._gguf_offloaded_layer_fraction("auto", None, 12, ["-ngl", "999"]) == 1.0
assert ri._gguf_offloaded_layer_fraction("auto", None, 12, ["-ngl", "abc"]) == 1.0
assert ri._gguf_offloaded_layer_fraction("manual", 0, 12, ["-ngl", "999"]) == 1.0
assert ri._gguf_offloaded_layer_fraction("manual", 12, 12, ["-ngl", "0"]) == 0.0
assert ri._gguf_offloaded_layer_fraction("manual", 0, 12, ["--top-k", "40"]) == 0.0
class TestGgufRuntimeBytes:
"""The context-dependent half, priced off a real (synthetic) header."""
def test_kv_grows_with_context(self, gqa_gguf):
sizes = [ri._gguf_runtime_bytes(gqa_gguf, ctx).kv_bytes for ctx in (1024, 4096, 16384)]
assert all(a < b for a, b in zip(sizes, sizes[1:])), sizes
assert sizes[0] > 0
def test_quantized_cache_is_smaller_than_f16(self, gqa_gguf):
f16 = ri._gguf_runtime_bytes(gqa_gguf, 8192, cache_type_kv = "f16")
q4 = ri._gguf_runtime_bytes(gqa_gguf, 8192, cache_type_kv = "q4_0")
assert q4.kv_bytes < f16.kv_bytes
assert q4.n_ctx == f16.n_ctx == 8192
def test_reports_what_it_priced(self, gqa_gguf):
runtime = ri._gguf_runtime_bytes(gqa_gguf, 4096, cache_type_kv = "q8_0", n_parallel = 1)
assert runtime.kv_estimable is True
assert runtime.n_ctx == 4096
assert runtime.cache_type_kv == "q8_0"
assert runtime.n_parallel == 1
assert runtime.layer_count == _GQA_FIELDS["block_count"]
def test_zero_context_prices_the_native_one(self, gqa_gguf):
runtime = ri._gguf_runtime_bytes(gqa_gguf, 0)
assert runtime.n_ctx == _GQA_FIELDS["context_length"]
assert runtime.kv_bytes > 0
@pytest.mark.parametrize("pooling_type, priced_ubatch", [(1, 2048), (2, 2048), (3, None)])
def test_embedding_micro_batch_matches_the_launch(self, tmp_path, pooling_type, priced_ubatch):
gguf = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "pooling_type": pooling_type})
assert ri._gguf_runtime_bytes(gguf, 2048).n_ubatch == priced_ubatch
def test_undeclared_embedding_pooling_keeps_the_default_micro_batch(self, tmp_path):
gguf = _write_gguf(tmp_path, "nomic-bert", _GQA_FIELDS)
assert ri._gguf_runtime_bytes(gguf, 2048).n_ubatch is None
def test_missing_dims_are_unknown_not_zero(self, dimless_gguf):
# kv_bytes == 0 means could not size; only the flag separates it from a real zero.
runtime = ri._gguf_runtime_bytes(dimless_gguf, 32768)
assert runtime.kv_estimable is False
assert runtime.kv_bytes == 0
assert runtime.compute_bytes == 0
assert runtime.n_ctx == 0
assert runtime.layer_count == 12
assert ri._gguf_offloaded_layer_fraction("manual", 0, runtime.layer_count) == 0.0
def test_a_recurrent_model_keeps_its_layer_count(self, tmp_path):
"""SSM models like Mamba have no attention dims, so their layer count must be kept."""
mamba = _write_gguf(
tmp_path,
"mamba",
{
"block_count": 24,
"embedding_length": 2048,
"context_length": 8192,
"ssm.conv_kernel": 4,
"ssm.inner_size": 4096,
"ssm.state_size": 16,
"ssm.time_step_rank": 128,
},
name = "mamba.gguf",
)
runtime = ri._gguf_runtime_bytes(mamba, 32768)
assert runtime.kv_estimable is False
assert runtime.layer_count == 24
assert ri._gguf_offloaded_layer_fraction("manual", 0, runtime.layer_count) == 0.0
def test_unreadable_file_is_unknown(self, tmp_path):
junk = tmp_path / "not-a-gguf.gguf"
junk.write_bytes(b"not a gguf header")
assert ri._gguf_runtime_bytes(str(junk), 4096) == ri._GGUF_RUNTIME_UNKNOWN
class TestKvGbWrapperCompatibility:
"""Training admission guard reads this scalar: it must stay exactly (KV + compute) in GB."""
@pytest.mark.parametrize(
"kwargs",
[
dict(),
dict(cache_type_kv = "q8_0"),
dict(n_parallel = 4),
dict(n_batch = 2048, n_ubatch = 512),
dict(ctx_checkpoints = 4),
dict(n_devices = 2),
dict(is_diffusion = True),
dict(llama_extra_args = ["-c", "16384"]),
],
)
def test_wrapper_equals_summed_tuple(self, gqa_gguf, kwargs):
runtime = ri._gguf_runtime_bytes(gqa_gguf, 4096, **kwargs)
assert ri._estimate_gguf_kv_gb(gqa_gguf, 4096, **kwargs) == pytest.approx(
(runtime.kv_bytes + runtime.compute_bytes) / _GIB
)
def test_unsizable_header_is_zero_gb(self, dimless_gguf):
assert ri._estimate_gguf_kv_gb(dimless_gguf, 32768) == 0.0
def test_the_guard_keeps_the_larger_context_the_panel_does_not(self, gqa_gguf):
"""The panel shows the smaller -c that launches; the guard reserves the larger one."""
smaller = ["-c", "8192"]
guard = ri._gguf_runtime_bytes(gqa_gguf, 131072, smaller)
panel = ri._gguf_runtime_bytes(gqa_gguf, 131072, smaller, ctx_last_wins = True)
assert guard.n_ctx == 131072
assert panel.n_ctx == 8192
assert panel.kv_bytes < guard.kv_bytes
assert ri._estimate_gguf_kv_gb(gqa_gguf, 131072, llama_extra_args = smaller) == pytest.approx(
(guard.kv_bytes + guard.compute_bytes) / _GIB
)
larger = ["-c", "262144"]
assert (
ri._gguf_runtime_bytes(gqa_gguf, 131072, larger, ctx_last_wins = True).n_ctx
== ri._gguf_runtime_bytes(gqa_gguf, 131072, larger).n_ctx
== 262144
)
assert (
ri._gguf_runtime_bytes(gqa_gguf, 131072, ["-c", "0"], ctx_last_wins = True).n_ctx
== _GQA_FIELDS["context_length"]
)
class TestResidentFileGb:
"""The weights term, obtained by subtracting the context term back out."""
def test_subtracts_the_local_arm_context_term(self, gqa_gguf, monkeypatch):
ctx_term = ri._estimate_gguf_kv_gb(gqa_gguf, 0, None)
assert ctx_term > 0, "fixture must have a non-trivial context term to subtract"
config = SimpleNamespace(identifier = "local/model", gguf_file = gqa_gguf, is_gguf = True)
monkeypatch.setattr(ri, "_estimate_gguf_required_gb", lambda cfg, **kw: 3.0 + ctx_term)
assert ri._gguf_resident_file_gb(config) == pytest.approx(3.0)
def test_unresolvable_size_is_not_cached(self, gqa_gguf, monkeypatch):
# None is usually an in-flight download, so it must not be cached.
config = SimpleNamespace(identifier = "local/model", gguf_file = gqa_gguf, is_gguf = True)
monkeypatch.setattr(ri, "_estimate_gguf_required_gb", lambda cfg, **kw: None)
assert ri._gguf_resident_file_gb(config) is None
assert ri._estimate_files_cache == {}
class TestMemoryBreakdown:
"""Composition of files + runtime into the panel's itemization."""
WEIGHTS_GB = 4.0
# Pinned: the synthetic header is tiny and would make the whole files total companions.
MAIN_BYTES = 3 * 1024**3
COMPANION_BYTES = int(WEIGHTS_GB * 1024**3) - MAIN_BYTES
@pytest.fixture
def config(self, gqa_gguf):
return SimpleNamespace(
identifier = "local/model",
gguf_file = gqa_gguf,
is_gguf = True,
gguf_mmproj_file = None,
)
@pytest.fixture(autouse = True)
def _fixed_files(self, monkeypatch):
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: self.WEIGHTS_GB)
real_size = ri.LlamaCppBackend._get_gguf_size_bytes
def _size(path):
name = os.path.basename(str(path))
return self.MAIN_BYTES if name == "model.gguf" else real_size(path)
monkeypatch.setattr(ri.LlamaCppBackend, "_get_gguf_size_bytes", staticmethod(_size))
def test_weights_do_not_move_with_context(self, config, gqa_gguf):
small = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 2048)
large = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 32768)
assert small.weights_bytes == large.weights_bytes
assert large.kv_bytes > small.kv_bytes
assert large.total_bytes > small.total_bytes
def test_total_is_the_sum_of_its_parts(self, config, gqa_gguf):
b = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 8192)
assert b.weights_bytes == int(round(self.WEIGHTS_GB * _GIB))
assert b.total_bytes == b.weights_bytes + b.kv_bytes + b.compute_bytes
def test_auto_placement_puts_everything_on_the_gpu(self, config, gqa_gguf):
b = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 8192, gpu_memory_mode = "auto")
assert b.gpu_bytes == b.total_bytes
assert b.kv_on_gpu is True
assert b.gpu_layers is None
def test_an_auto_load_with_an_extras_ngl_is_not_fully_resident(self, config, gqa_gguf):
full = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 8192, gpu_memory_mode = "auto")
on_cpu = ri._gguf_memory_breakdown(
config,
gqa_gguf,
n_ctx = 8192,
gpu_memory_mode = "auto",
llama_extra_args = ["-ngl", "0"],
)
assert on_cpu.total_bytes == full.total_bytes
assert on_cpu.kv_on_gpu is False
assert on_cpu.gpu_bytes == full.gpu_bytes - (
self.MAIN_BYTES + full.kv_bytes + full.compute_bytes
)
def test_a_smaller_ctx_override_is_the_context_priced(self, config, gqa_gguf):
b = ri._gguf_memory_breakdown(
config, gqa_gguf, n_ctx = 131072, llama_extra_args = ["-c", "8192"]
)
bigger = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 131072)
assert b.n_ctx == 8192
assert b.kv_bytes < bigger.kv_bytes
def test_a_manual_layer_count_in_the_extras_is_priced(self, config, gqa_gguf):
manual = dict(gpu_memory_mode = "manual", gpu_layers = 0, n_ctx = 8192)
field_only = ri._gguf_memory_breakdown(config, gqa_gguf, **manual)
overridden = ri._gguf_memory_breakdown(
config, gqa_gguf, llama_extra_args = ["-ngl", "999"], **manual
)
assert field_only.gpu_bytes < overridden.gpu_bytes
assert overridden.gpu_bytes == overridden.total_bytes
def test_an_extras_split_mode_decides_the_priced_mode(self, config, gqa_gguf):
def priced(extras, toggle):
return ri._gguf_memory_breakdown(
config,
gqa_gguf,
n_ctx = 32768,
llama_extra_args = extras,
tensor_parallel = toggle,
n_devices = 2,
).compute_bytes
layer, tensor = priced(None, False), priced(None, True)
assert layer != tensor, "fixture must separate the two modes for this to prove anything"
assert priced(["--split-mode", "tensor"], False) == tensor
assert priced(["--split-mode", "layer"], True) == layer
def test_manual_split_divides_the_cache_with_the_weights(self, config, gqa_gguf):
# KV for CPU-resident layers lives in host RAM (offload_kqv per layer).
b = ri._gguf_memory_breakdown(
config, gqa_gguf, n_ctx = 8192, gpu_memory_mode = "manual", gpu_layers = 6
)
fraction = 6 / float(_GQA_FIELDS["block_count"] + 1)
assert b.layer_count == _GQA_FIELDS["block_count"]
assert b.gpu_layers == 6
assert b.gpu_bytes == (
int(self.MAIN_BYTES * fraction)
+ self.COMPANION_BYTES
+ int(b.kv_bytes * fraction)
+ b.compute_bytes
)
assert b.gpu_bytes < b.total_bytes
assert b.kv_bytes > int(b.kv_bytes * fraction)
def test_a_partial_offload_splits_only_the_main_weight(self, config, gqa_gguf):
# --gpu-layers moves the model only; projector and drafter follow their own flags.
zero = ri._gguf_memory_breakdown(
config, gqa_gguf, n_ctx = 8192, gpu_memory_mode = "manual", gpu_layers = 0
)
assert zero.gpu_bytes == self.COMPANION_BYTES
assert zero.kv_on_gpu is False
full = ri._gguf_memory_breakdown(
config,
gqa_gguf,
n_ctx = 8192,
gpu_memory_mode = "manual",
gpu_layers = _GQA_FIELDS["block_count"] + 1,
)
assert full.gpu_bytes == full.total_bytes
def test_no_mmproj_offload_moves_the_projector_and_only_it(self, config, gqa_gguf, tmp_path):
projector = tmp_path / "mmproj-model.gguf"
projector.write_bytes(b"\0" * 1024)
config.gguf_mmproj_file = str(projector)
expected = self.COMPANION_BYTES - projector.stat().st_size
pinned = ri._gguf_memory_breakdown(
config,
gqa_gguf,
n_ctx = 8192,
gpu_memory_mode = "manual",
gpu_layers = 0,
llama_extra_args = ["--no-mmproj-offload"],
)
assert pinned.gpu_bytes == expected
def test_no_kv_offload_moves_the_cache_off_the_gpu(self, config, gqa_gguf):
b = ri._gguf_memory_breakdown(
config, gqa_gguf, n_ctx = 8192, llama_extra_args = ["--no-kv-offload"]
)
assert b.kv_on_gpu is False
assert b.kv_bytes > 0
assert b.gpu_bytes == b.weights_bytes + b.compute_bytes
assert b.total_bytes == b.weights_bytes + b.kv_bytes + b.compute_bytes
def test_unsizable_files_produce_no_breakdown(self, config, gqa_gguf, monkeypatch):
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: None)
assert ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 8192) is None
def _repo_config(**overrides) -> ModelConfig:
"""Repo GGUF with gguf_file None even when cached; the estimate must not fetch the network for it."""
fields = dict(
identifier = "org/model-GGUF",
display_name = "model (Q4_K_M)",
path = "org/model-GGUF",
is_local = False,
is_cached = False,
is_vision = False,
is_lora = False,
is_gguf = True,
gguf_file = None,
gguf_hf_repo = "org/model-GGUF",
gguf_variant = "Q4_K_M",
)
fields.update(overrides)
return ModelConfig(**fields)
def _local_config(gguf_path: str, **overrides) -> ModelConfig:
"""A config that already names a real file: the local-folder / native-pick shape."""
fields = dict(
identifier = str(Path(gguf_path).parent),
display_name = Path(gguf_path).name,
path = str(Path(gguf_path).parent),
is_local = True,
is_cached = True,
is_vision = False,
is_lora = False,
is_gguf = True,
gguf_file = gguf_path,
)
fields.update(overrides)
return ModelConfig(**fields)
class TestLocalizedEstimateConfig:
"""Point a repository-shaped config at the weights already on this disk."""
def test_config_naming_a_real_file_is_returned_unchanged(self, gqa_gguf):
# Identity: re-running companion detection could scan outside a native grant's directory.
config = _local_config(gqa_gguf)
assert ri._localized_estimate_config(config, gqa_gguf) is config
def test_a_hand_added_projector_above_the_snapshot_is_priced(self, gqa_gguf, tmp_path):
projector = tmp_path / "mmproj-F16.gguf"
projector.write_bytes(b"x")
config = _repo_config(is_vision = True, gguf_local_mmproj_file = str(projector))
local = ri._localized_estimate_config(config, gqa_gguf)
assert local.gguf_mmproj_file == str(projector)
def test_repo_config_is_copied_never_mutated(self, gqa_gguf):
# The original config is shared via _estimate_config_cache and must not be mutated.
config = _repo_config()
localized = ri._localized_estimate_config(config, gqa_gguf)
assert localized is not config
assert localized.gguf_file == gqa_gguf
assert config.gguf_file is None
assert localized.identifier == config.identifier
assert localized.gguf_hf_repo == config.gguf_hf_repo
assert localized.gguf_variant == config.gguf_variant
def test_real_sibling_projector_is_detected(self, tmp_path):
weight = _write_gguf(tmp_path, "qwen3", _GQA_FIELDS, name = "model-Q4_K_M.gguf")
projector = _write_gguf(tmp_path, "clip", {"block_count": 2}, name = "mmproj-model.gguf")
localized = ri._localized_estimate_config(_repo_config(is_vision = True), weight)
assert localized.gguf_mmproj_file == projector
assert localized.gguf_mtp_file is None
assert localized.gguf_dspark_file is None
assert localized.gguf_dflash_file is None
def test_non_vision_config_skips_the_projector(self, tmp_path):
weight = _write_gguf(tmp_path, "qwen3", _GQA_FIELDS, name = "model-Q4_K_M.gguf")
_write_gguf(tmp_path, "clip", {"block_count": 2}, name = "mmproj-model.gguf")
localized = ri._localized_estimate_config(_repo_config(is_vision = False), weight)
assert localized.gguf_mmproj_file is None
def test_drafter_detectors_receive_the_weight_and_its_directory(self, gqa_gguf, monkeypatch):
from utils.models import model_config as mc
seen = {}
def _stub(name, result):
def _detect(
path,
search_root = None,
*a,
**kw,
):
seen[name] = (path, search_root)
return result
return _detect
monkeypatch.setattr(mc, "detect_mtp_file", _stub("mtp", "/tmp/mtp.gguf"))
monkeypatch.setattr(mc, "detect_dspark_file", _stub("dspark", "/tmp/dspark.gguf"))
monkeypatch.setattr(mc, "detect_dflash_file", _stub("dflash", "/tmp/dflash.gguf"))
localized = ri._localized_estimate_config(_repo_config(), gqa_gguf)
assert localized.gguf_mtp_file == "/tmp/mtp.gguf"
assert localized.gguf_dspark_file == "/tmp/dspark.gguf"
assert localized.gguf_dflash_file == "/tmp/dflash.gguf"
expected = (gqa_gguf, str(Path(gqa_gguf).parent))
assert seen == {"mtp": expected, "dspark": expected, "dflash": expected}
def test_a_raising_detector_still_yields_a_usable_config(self, gqa_gguf, monkeypatch):
from utils.models import model_config as mc
monkeypatch.setattr(mc, "detect_mtp_file", lambda *a, **kw: "/tmp/mtp.gguf")
def _boom(*a, **kw):
raise OSError("permission denied walking the snapshot")
monkeypatch.setattr(mc, "detect_dspark_file", _boom)
localized = ri._localized_estimate_config(_repo_config(), gqa_gguf)
assert localized.gguf_file == gqa_gguf
assert localized.gguf_mtp_file == "/tmp/mtp.gguf"
assert localized.gguf_dspark_file is None
def test_localized_config_takes_the_local_files_arm(self, gqa_gguf, monkeypatch):
# Making the remote reserve raise turns taking the repository arm into a failure.
def _remote_arm_taken(*a, **kw):
raise AssertionError("a localized config must not be priced as a repository")
monkeypatch.setattr(ri, "_remote_gguf_compute_reserve_gb", _remote_arm_taken)
ctx_term = ri._estimate_gguf_kv_gb(gqa_gguf, 0, None)
monkeypatch.setattr(ri, "_estimate_gguf_required_gb", lambda cfg, **kw: 5.0 + ctx_term)
raw = _repo_config()
localized = ri._localized_estimate_config(raw, gqa_gguf)
assert ri._gguf_resident_file_gb(localized) == pytest.approx(5.0)
ri._estimate_files_cache.clear()
with pytest.raises(AssertionError, match = "must not be priced as a repository"):
ri._gguf_resident_file_gb(raw)
class TestTokenFingerprint:
"""Both TTL caches are keyed per token, since a gated repo resolves per token."""
def test_absent_token_is_empty(self):
assert ri._estimate_token_fingerprint(None) == ""
assert ri._estimate_token_fingerprint("") == ""
def test_stable_and_distinct(self):
a = ri._estimate_token_fingerprint("hf_aaaaaaaaaaaaaaaaaaaa")
assert a == ri._estimate_token_fingerprint("hf_aaaaaaaaaaaaaaaaaaaa")
assert a != ri._estimate_token_fingerprint("hf_bbbbbbbbbbbbbbbbbbbb")
def test_does_not_carry_the_token(self):
token = "hf_supersecretvalue123"
fingerprint = ri._estimate_token_fingerprint(token)
assert token not in fingerprint
assert fingerprint and len(fingerprint) == 16
def test_config_cache_is_not_shared_across_tokens(self, monkeypatch):
calls = []
def _from_identifier(
model_id,
hf_token = None,
**kw,
):
calls.append(hf_token)
return SimpleNamespace(identifier = model_id, resolved_with = hf_token)
monkeypatch.setattr(ri.ModelConfig, "from_identifier", staticmethod(_from_identifier))
# Answer the on-disk gate so this tests token scoping, not the gate.
monkeypatch.setattr(ri, "_estimate_target_is_on_this_disk", lambda _id: True)
first = ri._cached_estimate_config("org/gated", "Q4_K_M", "token-a", False)
second = ri._cached_estimate_config("org/gated", "Q4_K_M", "token-b", False)
assert first.resolved_with == "token-a"
assert second.resolved_with == "token-b"
assert calls == ["token-a", "token-b"]
assert ri._cached_estimate_config("org/gated", "Q4_K_M", "token-a", False) is first
assert calls == ["token-a", "token-b"]
def test_files_cache_is_not_shared_across_tokens(self, gqa_gguf, monkeypatch):
config = _local_config(gqa_gguf)
ctx_term = ri._estimate_gguf_kv_gb(gqa_gguf, 0, None)
by_token = {"token-a": 1.0 + ctx_term, "token-b": 7.0 + ctx_term}
calls = []
def _required(cfg, **kw):
calls.append(kw.get("hf_token"))
return by_token[kw["hf_token"]]
monkeypatch.setattr(ri, "_estimate_gguf_required_gb", _required)
assert ri._gguf_resident_file_gb(config, hf_token = "token-a") == pytest.approx(1.0)
assert ri._gguf_resident_file_gb(config, hf_token = "token-b") == pytest.approx(7.0)
assert calls == ["token-a", "token-b"]
assert ri._gguf_resident_file_gb(config, hf_token = "token-a") == pytest.approx(1.0)
assert calls == ["token-a", "token-b"]
def _estimate(fastapi_request = None, **kwargs):
"""Call the route directly, bypassing the auth dependency."""
return asyncio.run(
ri.estimate_memory(
EstimateMemoryRequest(**kwargs),
fastapi_request = fastapi_request,
current_subject = "test",
)
)
def _request_with_slots(slots: int):
"""A stand-in for the FastAPI request carrying the server's slot default."""
return SimpleNamespace(
app = SimpleNamespace(
state = SimpleNamespace(
llama_parallel_slots = slots,
)
)
)
class TestEstimateMemoryRoute:
"""The three "cannot size this" answers, and what they must not do first."""
def test_ollama_manifest_ref_is_refused_before_materializing(self, monkeypatch):
# Fires every slider tick: must bail on the prefix with no disk write or resolution.
def boom(*a, **kw):
raise AssertionError("estimate-memory must not materialize an Ollama ref")
monkeypatch.setattr(ri, "materialize_ollama_model_ref", boom)
monkeypatch.setattr(ri, "acquire_ollama_model_ref", boom)
monkeypatch.setattr(ri, "_cached_estimate_config", boom)
resp = _estimate(model_path = "ollama-manifest:sha256-deadbeef")
assert resp.available is False
assert resp.reason == "unsupported_source"
def test_managed_caller_cannot_size_an_owner_only_path(self, monkeypatch):
from utils import openai_auto_switch_settings as settings
def boom(*a, **kw):
raise AssertionError("estimate-memory must not size a refused path")
monkeypatch.setattr(ri.account_access, "managed_account", lambda: True)
monkeypatch.setattr(ri.account_access, "require_model_access", lambda *a, **kw: None)
monkeypatch.setattr(settings, "get_model_override", lambda key: {})
monkeypatch.setattr(
ri, "get_llama_cpp_backend", lambda: SimpleNamespace(last_load_intent = None)
)
monkeypatch.setattr(ri, "_cached_estimate_config", boom)
resp = _estimate(
model_path = "/models/m.gguf", llama_extra_args = ["-md", "/home/owner/private.gguf"]
)
assert resp.available is False
assert resp.reason == "unsizable"
def test_the_estimate_asks_the_question_the_worker_asks_of_the_stack(self, monkeypatch):
import utils.mlx_repair as repair
monkeypatch.setattr(repair, "is_apple_silicon", lambda: True)
for blockers, available in (([], True), (["mlx-vlm 0.0.1 is too old"], False)):
monkeypatch.setattr(repair, "mlx_stack_blockers", lambda b = blockers: b)
assert ri._mlx_estimate_available() is available
def test_non_gguf_model_is_not_priced_without_mlx(self, monkeypatch):
monkeypatch.setattr(
ri,
"_cached_estimate_config",
lambda *a, **kw: SimpleNamespace(is_gguf = False, identifier = "org/model"),
)
monkeypatch.setattr(ri, "_mlx_estimate_available", lambda: False)
resp = _estimate(model_path = "org/model")
assert resp.available is False
assert resp.reason == "not_gguf"
def test_a_native_audio_load_is_not_priced_as_an_mlx_one(self, monkeypatch, tmp_path):
import core.inference.native_audio as native_audio
self._mlx_target(monkeypatch, str(tmp_path))
priced = []
monkeypatch.setattr(
ri, "_mlx_estimate_fitted_context", lambda *a, **kw: priced.append(a) or None
)
monkeypatch.setattr(native_audio, "is_native_audio_model", lambda name: name == "org/tts")
resp = _estimate(model_path = "org/tts")
assert (resp.available, resp.reason, priced) == (False, "not_gguf", [])
assert _estimate(model_path = "org/model").reason != "not_gguf"
@staticmethod
def _mlx_target(monkeypatch, model_dir, **config):
"""A resolved non-GGUF config on an MLX host, with its weights at *model_dir*."""
monkeypatch.setattr(
ri,
"_cached_estimate_config",
lambda *a, **kw: SimpleNamespace(is_gguf = False, identifier = "org/model", **config),
)
monkeypatch.setattr(ri, "_mlx_estimate_available", lambda: True)
monkeypatch.setattr(ri, "_local_mlx_model_dir", lambda config: model_dir)
@staticmethod
def _record_breakdown(monkeypatch, **fields):
"""Install a breakdown stub, returning the dict its call is recorded into."""
import core.inference.mlx_memory as mlx_memory
seen = {}
def _breakdown(
model_dir,
*,
n_ctx,
kv_bits = None,
load_in_4bit = False,
**kw,
):
seen.update(
model_dir = model_dir, n_ctx = n_ctx, kv_bits = kv_bits, load_in_4bit = load_in_4bit
)
return mlx_memory.MlxMemoryBreakdown(n_ctx = n_ctx, **fields)
monkeypatch.setattr(mlx_memory, "mlx_memory_breakdown", _breakdown)
return seen
def test_an_explicit_drafter_is_priced_as_the_load_would_attach_it(self, monkeypatch):
import core.inference.mlx_inference as mlx_inference
import core.inference.mlx_memory as mlx_memory
from core.inference import mlx_speculative as spec
self._mlx_target(monkeypatch, "/models/thing")
sources = (spec.DrafterSource("mtp", "/big", True), spec.DrafterSource("mtp", "/ok", False))
monkeypatch.setattr(
spec,
"resolve_speculation",
lambda mode, *a, **k: spec.SpecResolution(mode, sources if mode == "mtp" else ()),
)
monkeypatch.setattr(spec, "speculation_refusal", lambda **_: None)
fit, fit_widths = False, []
monkeypatch.setattr(
mlx_inference,
"mlx_drafter_fit",
lambda d, ceiling, pinned, source, reserve = True, **_: (
None
if fit in ("unpriced", "unsizable")
else source.path == "/ok" and (pinned or 0) <= 8192 and not fit
if reserve
else (pinned or 0) <= 32768, # MLX's own cap, spending the prompt cache's share
None if pinned or fit in ("unpriced", "unsizable") else 12288,
),
)
priced_fit = mlx_inference.mlx_drafter_fit
monkeypatch.setattr(
mlx_inference,
"mlx_drafter_fit",
lambda *a, kv_bits = None, **k: fit_widths.append(kv_bits) or priced_fit(*a, **k),
)
monkeypatch.setattr(ri, "_mlx_estimate_ceiling", lambda model_dir: 32768)
monkeypatch.setattr(
ri, "_mlx_estimate_fitted_context", lambda *a, **kw: 24576 if kw else 20480
)
priced = []
monkeypatch.setattr(
mlx_memory,
"mlx_memory_breakdown",
lambda d, *, n_ctx, **kw: priced.append((n_ctx, kw.get("drafter"), kw.get("vision")))
or (
None
if fit == "unsizable" and kw.get("drafter")
else mlx_memory.MlxMemoryBreakdown(1, 0, 0, 1, 1, n_ctx = n_ctx)
),
)
rooms, estimates, load_estimate = [], [], ri._mlx_load_estimate
monkeypatch.setattr(
mlx_inference,
"mlx_prompt_cache_room",
lambda d, n_ctx, source, **_: rooms.append((n_ctx, source.path)) or 7,
)
monkeypatch.setattr(
ri,
"_mlx_load_estimate",
lambda *a: (result := load_estimate(*a), estimates.append(result[1]))[0],
)
assert _estimate(model_path = "org/model", speculative_type = "mtp").context_fitted == 12288
for pinned in (4096, 16384, 65536):
_estimate(model_path = "org/model", speculative_type = "mtp", max_seq_length = pinned)
_estimate(model_path = "org/model", speculative_type = "mtp", mlx_kv_quant = "8")
assert fit_widths[-1] == 8
_estimate(model_path = "org/model")
_estimate(model_path = "o/m", speculative_type = "ngram", mlx_kv_quant = "auto", mlx_kv_bits = 8)
_estimate(model_path = "org/model", spec_draft_model = "org/d") # named under Auto
fit = "unfit"
assert _estimate(model_path = "org/model", speculative_type = "mtp").context_fitted == 12288
fit = "unpriced" # no fit to go on: the drafter the load attaches is still charged
_estimate(model_path = "org/model", speculative_type = "mtp")
fit = "unsizable" # one the breakdown cannot size leaves the rest of the load priced
assert _estimate(model_path = "org/model", speculative_type = "mtp").available
assert [route for *_, route in priced] == [True] * 5 + [None] + [True] * 6
assert [row[:2] for row in priced[:7] + priced[8:]] == [
(12288, ("/ok", False)),
(4096, ("/ok", False)),
(16384, ("/big", True)),
(65536, None), # past MLX's own cap the load falls back, so no drafter is charged
(12288, ("/ok", False)),
(20480, None),
(24576, None),
(12288, ("/big", True)),
(24576, ("/big", True)),
(24576, ("/big", True)),
(24576, None),
]
# Only a drafter that spent the prompt cache's share caps the cache admission charges.
assert rooms == [(16384, "/big"), (12288, "/big"), (24576, "/big")]
assert [e.cache_room for e in estimates if e and e.cache_room] == [7, 7, 7]
def test_a_drafter_is_fitted_at_the_load_kv_width(self, monkeypatch):
import core.inference.mlx_inference as mlx_inference
import core.inference.mlx_memory as mlx_memory
from core.inference.mlx_speculative import DrafterSource
asked = []
monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda **_: 1)
monkeypatch.setattr(mlx_inference, "mlx_kv_quant_is_refused", lambda _d, _b: False)
monkeypatch.setattr(
mlx_memory,
"mlx_fit_outcome",
lambda *_a, **k: asked.append(k["kv_bits"]) or ("fits", None),
)
monkeypatch.setattr(
mlx_memory, "mlx_headroom", lambda *_a, **k: asked.append(k["kv_bits"]) or 1
)
source = DrafterSource("mtp", "/d", False)
mlx_inference.mlx_drafter_fit("/m", 8, None, source, load_in_4bit = False, kv_bits = 8)
mlx_inference.mlx_prompt_cache_room("/m", 8, source, load_in_4bit = False, kv_bits = 8)
assert asked == [8, 8]
monkeypatch.setattr(mlx_inference, "mlx_kv_quant_is_refused", lambda _d, _b: True)
mlx_inference.mlx_drafter_fit("/m", 8, None, source, load_in_4bit = False, kv_bits = 8)
assert asked[-1] is None
def test_a_drafter_attaches_where_its_priced_fit_holds_and_a_pin_fits_whole(self, monkeypatch):
import core.inference.mlx_inference as mlx_inference
import core.inference.mlx_memory as mlx_memory
from core.inference.mlx_speculative import DrafterSource
budgets = []
monkeypatch.setattr(mlx_inference, "mlx_vlm_snapshot_store_available", lambda: True)
monkeypatch.setattr(
mlx_inference,
"mlx_memory_budget",
lambda *, retains_history: budgets.append(retains_history) or 8 << 30,
)
priced = []
def fit(pinned, outcome, *bare):
monkeypatch.setattr(
mlx_memory,
"mlx_fit_outcome",
lambda d, *, max_ctx, vision, drafter = None, **_: priced.append(
(max_ctx, drafter, vision)
)
or ((outcome, 4096) if drafter else ("fits", bare[0])),
)
source = DrafterSource("dflash", "/drafter", False)
return mlx_inference.mlx_drafter_fit(
"/m", 32768, pinned, source, load_in_4bit = False, costless = bool(bare)
)
attaches = [fit(None, o)[0] for o in ("fits", "fitted", "unsizable", "no_fit")]
assert attaches == [True, True, None, False]
assert (fit(8192, "fits"), fit(8192, "fitted")) == ((True, 4096), (False, 4096))
assert (
priced
== [(32768, ("/drafter", False), True)] * 4 + [(8192, ("/drafter", False), True)] * 2
)
costless = [fit(None, *c) for c in (("fits", 4096), ("fits", 8192), ("unsizable", 4096))]
assert costless == [(True, 4096), (False, 4096), (None, 4096)]
asked = {}
monkeypatch.setattr(
mlx_memory, "mlx_fit_outcome", lambda d, **kw: asked.update(kw) or ("fits", 4096)
)
source = DrafterSource("dflash", "/drafter", False)
mlx_inference.mlx_drafter_fit(
"/m",
32768,
None,
source,
load_in_4bit = False,
offload = (None, 0.6),
prefill_chunk = 8192,
)
assert (asked["offload"], asked["prefill_chunk"]) == ((None, 0.6), 8192)
assert set(budgets) == {True}
mlx_inference.mlx_drafter_fit("/m", 32768, None, source, load_in_4bit = False, reserve = False)
assert budgets[-1] is False # MLX's own cap: the prompt cache's share is spendable
seen = []
def costless(attached, alone):
monkeypatch.setattr(
mlx_memory,
"mlx_fit_outcome",
lambda d, *, drafter = None, offload = None, prefill_chunk = None, **_: seen.append(
(bool(drafter), offload, prefill_chunk)
)
or (attached if drafter else alone),
)
return mlx_inference.mlx_drafter_fit(
"/m",
32768,
None,
source,
load_in_4bit = False,
costless = True,
offload = (None, 0.6),
prefill_chunk = 8192,
baseline = {"offload": (None, 0.5), "prefill_chunk": 4096},
)[0]
assert costless(("fitted", 10240), ("fitted", 8192)) is True
assert seen == [(True, (None, 0.6), 8192), (False, (None, 0.5), 4096)]
assert costless(("fits", None), ("fitted", 8192)) is True
assert costless(("fitted", 9216), ("fitted", 11520)) is False
assert costless(("fitted", 8192), ("fits", None)) is False
assert costless(("fits", None), ("unsizable", None)) is False
def test_mlx_model_not_on_disk_is_not_downloaded(self, monkeypatch):
self._mlx_target(monkeypatch, None)
resp = _estimate(model_path = "org/model")
assert resp.available is False
assert resp.reason == "not_downloaded"
def test_mlx_model_whose_cache_cannot_be_read_is_unsizable(self, monkeypatch):
self._mlx_target(monkeypatch, "/models/thing")
import core.inference.mlx_memory as mlx_memory
monkeypatch.setattr(mlx_memory, "mlx_memory_breakdown", lambda *a, **kw: None)
resp = _estimate(model_path = "org/model")
assert resp.available is False
assert resp.reason == "unsizable"
assert resp.total_bytes == 0
def test_mlx_model_is_priced_and_itemized(self, monkeypatch, tmp_path):
self._mlx_target(monkeypatch, "/models/thing")
seen = self._record_breakdown(
monkeypatch,
weights_bytes = 8_000_000_000,
kv_bytes = 600_000_000,
compute_bytes = 400_000_000,
total_bytes = 9_000_000_000,
gpu_bytes = 9_000_000_000,
layer_count = 36,
)
resp = _estimate(
model_path = "org/model",
max_seq_length = 16384,
mlx_kv_bits = 4,
cache_type_kv = "q8_0",
)
assert (resp.available, resp.reason, resp.kv_estimable) == (True, None, True)
assert (resp.weights_bytes, resp.kv_bytes, resp.compute_bytes) == (
8_000_000_000,
600_000_000,
400_000_000,
)
assert (resp.total_bytes, resp.gpu_bytes, resp.n_ctx, resp.layer_count) == (
9_000_000_000,
9_000_000_000,
16384,
36,
)
assert seen == {
"model_dir": "/models/thing",
"n_ctx": 16384,
"kv_bits": 4,
"load_in_4bit": True,
}
_estimate(
model_path = "org/model",
max_seq_length = 16384,
mlx_kv_bits = 4,
load_in_4bit = False,
)
assert seen["load_in_4bit"] is False
import utils.transformers_version as tv
asked, guarded, real_guard = [], [], ri._offline_guarded
monkeypatch.setattr(
tv,
"latest_tier_active_for",
lambda name, token = None: asked.append((name, token)) or True,
)
monkeypatch.setattr(
ri,
"_offline_guarded",
lambda t, fn, *a, **kw: guarded.append(fn) or real_guard(t, fn, *a, **kw),
)
_estimate(model_path = "org/model", max_seq_length = 16384, load_in_4bit = True, hf_token = "tok")
assert seen["load_in_4bit"] is False
assert asked == [("org/model", "tok")] and guarded == [tv.latest_tier_active_for]
monkeypatch.setattr(tv, "latest_tier_active_for", lambda *a, **kw: False)
monkeypatch.setattr(
ri,
"_cached_estimate_config",
lambda *a, **kw: SimpleNamespace(
is_gguf = False,
identifier = "org/model",
is_lora = True,
path = str(tmp_path),
base_model = "org/base",
),
)
seen.clear()
resp = _estimate(model_path = "org/model", max_seq_length = 16384, load_in_4bit = True)
assert (resp.available, resp.reason, resp.total_bytes) == (False, "unsizable", 0)
assert seen == {}
def test_a_full_finetune_output_is_priced_at_the_16_bit_it_loads_at(
self, monkeypatch, tmp_path
):
self._mlx_target(monkeypatch, "/models/thing", is_lora = False, path = str(tmp_path))
seen = self._record_breakdown(
monkeypatch,
weights_bytes = 1,
kv_bytes = 1,
compute_bytes = 1,
total_bytes = 3,
gpu_bytes = 3,
layer_count = 2,
)
import utils.transformers_version as tv
monkeypatch.setattr(tv, "latest_tier_active_for", lambda *a, **kw: False)
monkeypatch.setattr(ri, "is_full_finetune_output", lambda path: path == str(tmp_path))
_estimate(model_path = "org/model", max_seq_length = 4096, load_in_4bit = True)
assert seen["load_in_4bit"] is False
monkeypatch.setattr(ri, "is_full_finetune_output", lambda path: False)
_estimate(model_path = "org/model", max_seq_length = 4096, load_in_4bit = True)
assert seen["load_in_4bit"] is True
def test_the_snapshot_priced_is_the_one_the_load_would_open(self, monkeypatch, tmp_path):
import utils.models.model_config as mc
asked, found = [], []
snaps, ref = tmp_path / "snapshots", tmp_path / "refs" / "main"
monkeypatch.setattr(ri, "_estimate_hf_cache_roots", lambda: [None])
monkeypatch.setattr(
mc, "_iter_hf_cache_snapshots", lambda name, cache_dir = None: asked.append(name) or found
)
if ri._mlx_estimate_available():
for alias in ("unsloth/Qwen3-4B-bnb-4bit", "unsloth/Qwen3-4B-unsloth-bnb-4bit"):
assert (
ri._local_mlx_model_dir(
SimpleNamespace(path = None, is_local = False, identifier = alias)
)
is None
)
assert asked == ["unsloth/Qwen3-4B", "unsloth/Qwen3-4B"]
for made in (
"stub/config.json",
"new/config.json",
"new/model.safetensors",
"old/config.json",
"old/model.safetensors",
):
(snaps / made).parent.mkdir(parents = True, exist_ok = True)
(snaps / made).write_text("{}")
ref.parent.mkdir()
found[:], new_, old = (
[snaps / "stub", snaps / "new", snaps / "old"],
str(snaps / "new"),
str(snaps / "old"),
)
config = SimpleNamespace(path = None, is_local = False, is_lora = False, identifier = "org/model")
for named, expected in (
("old", old),
("stub", None),
("collected", new_),
("../snapshots/old", new_),
(None, new_),
):
ref.write_text(named) if named else ref.unlink()
assert ri._local_mlx_model_dir(config) == expected
def test_a_part_finished_download_is_not_priced_as_a_smaller_model(self, tmp_path):
def index(weight_map):
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": weight_map})
)
(tmp_path / "config.json").write_text("{}")
config = SimpleNamespace(path = str(tmp_path), is_local = True, is_lora = False)
five = [f"model-0000{i}-of-00005.safetensors" for i in range(1, 6)]
for name in five[:2]:
(tmp_path / name).write_bytes(b"")
assert ri._local_mlx_model_dir(config) is None
for name in five[2:]:
(tmp_path / name).write_bytes(b"")
assert ri._local_mlx_model_dir(config) == str(tmp_path)
(tmp_path / "adapter-00001-of-00002.safetensors").write_bytes(b"")
assert ri._local_mlx_model_dir(config) == str(tmp_path)
(tmp_path / "config.json").write_text(json.dumps({"vision_config": {}}))
(tmp_path / "weights").mkdir()
nested = [f"model-0000{i}-of-00002.safetensors" for i in (1, 2)]
for name in nested:
(tmp_path / "weights" / name).write_bytes(b"")
index({n: f"weights/{n}" for n in nested})
assert ri._local_mlx_model_dir(config) == str(tmp_path)
(tmp_path / "weights" / nested[1]).unlink()
assert ri._local_mlx_model_dir(config) is None
def test_shard_names_that_count_nothing_are_counted_by_the_index(self, tmp_path):
(tmp_path / "config.json").write_text("{}")
config = SimpleNamespace(path = str(tmp_path), is_local = True, is_lora = False)
(tmp_path / "model.safetensors").write_bytes(b"")
assert ri._local_mlx_model_dir(config) == str(tmp_path)
(tmp_path / "model.safetensors").unlink()
(tmp_path / "weights").mkdir()
named = [f"model-{i:05d}.safetensors" for i in range(1, 45)] + [
"mtp.safetensors",
"weights/extra.safetensors",
]
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {n: n for n in named}})
)
(tmp_path / named[0]).write_bytes(b"")
assert ri._local_mlx_model_dir(config) is None
for name in named[1:]:
(tmp_path / name).write_bytes(b"")
assert ri._local_mlx_model_dir(config) == str(tmp_path)
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
str(i): f"model-{i:05d}-of-00099.safetensors" for i in range(1, 100)
}
}
)
)
assert ri._local_mlx_model_dir(config) == str(tmp_path)
def test_an_mlx_estimate_prices_the_offload_the_load_applies(self, monkeypatch, tmp_path):
import core.inference.mlx_inference as mlx_inference
import core.inference.mlx_memory as mlx_memory
import core.inference.mlx_speculative as mlx_speculative
self._mlx_target(monkeypatch, str(tmp_path))
seen, four_bit, planned = [], {"on": False}, []
monkeypatch.setattr(ri, "_mlx_estimate_load_in_4bit", lambda *a: four_bit["on"])
monkeypatch.setattr(
mlx_memory, "mlx_offloadable_parts", lambda model_dir, vision = None: ("experts",)
)
monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: object())
monkeypatch.setattr(mlx_inference, "mlx_memory_budget", lambda retains_history: 1 << 40)
monkeypatch.setattr(
mlx_memory,
"mlx_offload_plan",
lambda d, **kw: planned.append(kw)
or (
kw["ple"],
(0.62 if kw["drafter"] else 0.6) if kw["experts"] == "auto" else kw["experts"],
8192,
),
)
monkeypatch.setattr(
mlx_memory,
"mlx_memory_breakdown",
lambda model_dir, *, n_ctx, **kw: seen.append(
(kw.get("offload"), kw.get("prefill_chunk"))
)
or mlx_memory.MlxMemoryBreakdown(1, 1, 1, 3, 3, n_ctx),
)
estimate = lambda **knobs: _estimate(model_path = "org/model", max_seq_length = 4096, **knobs)
price = lambda **knobs: estimate(**knobs) and seen[-1][0]
(tmp_path / "config.json").write_text(json.dumps({"quantization": {"bits": 4}}))
assert price(mlx_ple_offload = "off", mlx_expert_offload = 0.5) == (None, 0.5)
assert (
seen[-1][1] is None and not planned
) # a hand-set fraction prefills in the load's own chunk
off = estimate(mlx_ple_offload = "off", mlx_expert_offload = "off")
assert off.mlx_offload.offloadable == ["experts"] and seen[-1] == (None, None)
assert price(mlx_ple_offload = "off") == (None, 0.6) and planned[-1]["context"] == 4096
assert seen[-1][1] == 8192
assert planned[-1]["vision"] is None
with monkeypatch.context() as patch:
source = mlx_speculative.DrafterSource("mtp", str(tmp_path), True)
patch.setattr(
mlx_speculative,
"resolve_speculation",
lambda mode, named, **where: mlx_speculative.SpecResolution(mode, (source,)),
)
patch.setattr(ri, "_mlx_estimate_speculates", lambda request, model_dir: False)
estimate(mlx_ple_offload = "off", speculative_type = "mtp")
assert planned[-1]["drafter"] is None # refused (KV quantization): no room to make
patch.setattr(ri, "_mlx_estimate_speculates", lambda request, model_dir: True)
fits = []
patch.setattr(
mlx_inference,
"mlx_drafter_fit",
lambda *a, **kw: fits.append((kw["offload"], kw.get("baseline"))) or (True, None),
)
assert price(mlx_ple_offload = "off") == (None, 0.62)
assert [p["drafter"] for p in planned[-2:]] == [(str(tmp_path), True), None]
assert fits[-1] == ((None, 0.62), {"offload": (None, 0.6), "prefill_chunk": 8192})
estimate(mlx_ple_offload = "off", speculative_type = "mtp")
assert planned[-1]["drafter"] == (str(tmp_path), True)
assert fits[-1] == ((None, 0.62), None)
assert planned[-1]["vision"] is True and len(planned) == 5
four_bit["on"] = True # already quantized: nothing for the load to quantize, so it offloads
assert price(mlx_ple_offload = 1.0) == (1.0, 0.6)
(tmp_path / "config.json").write_text("{}")
assert price(mlx_ple_offload = 1.0) is None and len(planned) == 6
off = estimate(mlx_ple_offload = "off", mlx_expert_offload = "off")
assert off.mlx_offload.offloadable == []
four_bit["on"] = False
assert price(mlx_ple_offload = 1.0) == (1.0, 0.6)
def test_a_speculative_mlx_estimate_plans_offload_without_the_history_reserve(
self, monkeypatch, tmp_path
):
import core.inference.mlx_inference as mlx_inference
import core.inference.mlx_memory as mlx_memory
self._mlx_target(monkeypatch, str(tmp_path))
(tmp_path / "config.json").write_text(json.dumps({"quantization": {"bits": 4}}))
reserves = []
monkeypatch.setattr(ri, "_mlx_estimate_load_in_4bit", lambda *a: False)
monkeypatch.setattr(
mlx_memory, "mlx_offloadable_parts", lambda d, vision = None: ("experts",)
)
monkeypatch.setattr(mlx_memory, "mlx_offload_planner", lambda: object())
monkeypatch.setattr(mlx_inference, "mlx_vlm_snapshot_store_available", lambda: False)
monkeypatch.setattr(
mlx_inference,
"mlx_memory_budget",
lambda retains_history: reserves.append(retains_history) or 1 << 40,
)
monkeypatch.setattr(mlx_memory, "mlx_offload_plan", lambda d, **kw: (None, 0.5))
monkeypatch.setattr(
mlx_memory,
"mlx_memory_breakdown",
lambda model_dir, *, n_ctx, **kw: mlx_memory.MlxMemoryBreakdown(1, 1, 1, 3, 3, n_ctx),
)
estimate = lambda: _estimate(model_path = "org/model", max_seq_length = 4096)
estimate()
assert reserves[-1] is True
# The load speculates through mlx-vlm, which keeps no history without the snapshot store.
monkeypatch.setattr(ri, "_mlx_estimate_speculates", lambda request, model_dir: True)
estimate()
assert reserves[-1] is False
def test_an_mlx_estimate_prices_max_seq_length_not_n_ctx(self, monkeypatch):
self._mlx_target(monkeypatch, "/models/thing")
seen = self._record_breakdown(
monkeypatch,
weights_bytes = 1,
kv_bytes = 1,
compute_bytes = 1,
total_bytes = 3,
gpu_bytes = 3,
)
_estimate(model_path = "org/model", max_seq_length = 8192, n_ctx = 32768)
assert seen["n_ctx"] == 8192
def test_an_unpinned_mlx_estimate_prices_the_fit_and_reports_it(self, monkeypatch, tmp_path):
from core.inference import mlx_inference
from core.inference.runtime_context import MAX_REQUESTABLE_CONTEXT
write = (tmp_path / "config.json").write_text
write(json.dumps({"max_position_embeddings": 262_144}))
self._mlx_target(monkeypatch, str(tmp_path))
asked, fit = {}, {"answer": 24_576}
monkeypatch.setattr(
mlx_inference,
"mlx_fit_to_memory",
lambda model_dir, ceiling, **kw: (
asked.update(kw, ceiling = ceiling, dir = model_dir) or fit["answer"]
),
)
seen = self._record_breakdown(
monkeypatch,
weights_bytes = 1,
kv_bytes = 1,
compute_bytes = 1,
total_bytes = 3,
gpu_bytes = 3,
)
widths, refuses = [], {"answer": False}
monkeypatch.setattr(
mlx_inference,
"mlx_kv_quant_is_refused",
lambda *a: widths.append(a) or refuses["answer"],
)
resp = _estimate(model_path = "org/model", n_ctx = 32_768, mlx_kv_bits = 4)
assert (seen["n_ctx"], seen["kv_bits"], resp.context_fitted) == (24_576, 4, 24_576)
assert widths == [(str(tmp_path), 4)]
assert asked == {
"ceiling": 262_144,
"retains_history": True,
"dir": str(tmp_path),
"kv_bits": 4,
"load_in_4bit": seen["load_in_4bit"],
}
widths.clear()
resp = _estimate(model_path = "org/model", n_ctx = 32_768, mlx_kv_quant = "4")
assert (seen["kv_bits"], resp.context_fitted, widths) == (4, 24_576, [(str(tmp_path), 4)])
widths.clear()
monkeypatch.setattr(mlx_inference, "mlx_vlm_snapshot_store_available", lambda: False)
resp = _estimate(model_path = "org/model", mlx_kv_quant = "tq-4", mlx_kv_bits = 4)
turboquant = mm.MlxTurboQuant(4)
assert (asked["kv_bits"], seen["kv_bits"], widths) == (turboquant, turboquant, [])
assert (asked.pop("vision"), asked["retains_history"]) == (True, False)
refuses["answer"] = True
resp = _estimate(model_path = "org/model", mlx_kv_bits = 4)
assert (asked["kv_bits"], seen["kv_bits"], resp.context_fitted) == (None, None, 24_576)
resp = _estimate(model_path = "org/model", max_seq_length = 8192, mlx_kv_bits = 4)
assert (seen["n_ctx"], seen["kv_bits"], resp.context_fitted) == (8192, None, None)
refuses["answer"] = False
asked.clear()
resp = _estimate(model_path = "org/model", max_seq_length = 8192, mlx_kv_bits = 4)
assert (seen["n_ctx"], seen["kv_bits"], resp.context_fitted, asked) == (8192, 4, None, {})
widths.clear()
_estimate(model_path = "org/model")
assert (widths, asked["kv_bits"], seen["kv_bits"]) == ([], None, None)
fit["answer"] = None
write(json.dumps({"max_position_embeddings": MAX_REQUESTABLE_CONTEXT * 4}))
resp = _estimate(model_path = "org/model", mlx_kv_bits = 4)
assert resp.context_fitted is None
assert (seen["n_ctx"], seen["kv_bits"]) == (MAX_REQUESTABLE_CONTEXT, 4)
write(json.dumps({"model_type": "llama"}))
seen.clear()
assert _estimate(model_path = "org/model").reason == "unsizable"
assert seen == {}
write(json.dumps({"max_position_embeddings": 262_144}))
self._mlx_target(monkeypatch, str(tmp_path), is_vision = True)
for available in (True, False):
monkeypatch.setattr(
mlx_inference, "mlx_vlm_snapshot_store_available", lambda a = available: a
)
_estimate(model_path = "org/model")
assert asked["retains_history"] is available
def test_gguf_not_on_disk_is_not_downloaded(self, monkeypatch):
monkeypatch.setattr(
ri,
"_cached_estimate_config",
lambda *a, **kw: SimpleNamespace(is_gguf = True, identifier = "org/model"),
)
monkeypatch.setattr(ri, "_local_gguf_main_path", lambda config: None)
resp = _estimate(model_path = "org/model", gguf_variant = "Q4_K_M")
assert resp.available is False
assert resp.reason == "not_downloaded"
def test_a_repo_not_on_this_disk_is_refused_before_it_is_resolved(self, monkeypatch):
# Driven for real: an uncached remote id must not reach the Hub.
def boom(*a, **kw):
raise AssertionError("must not resolve a model that is not on this disk")
monkeypatch.setattr(ri.ModelConfig, "from_identifier", staticmethod(boom))
monkeypatch.setattr(ri, "_estimate_hf_cache_roots", lambda: [Path(os.devnull).parent])
resp = _estimate(model_path = "org/definitely-not-cached")
assert resp.available is False
assert resp.reason == "not_downloaded"
def test_the_on_disk_check_reads_every_cache_root(self, tmp_path, monkeypatch):
# Must iterate all cache roots (configured, legacy, default), not just the configured one.
primary, secondary = tmp_path / "a", tmp_path / "b"
for d in (primary, secondary):
d.mkdir()
snap = secondary / "models--org--Model-GGUF" / "snapshots" / "rev1"
snap.mkdir(parents = True)
(snap / "model.gguf").write_bytes(b"\0" * 8)
monkeypatch.setattr(ri, "_estimate_hf_cache_roots", lambda: [primary, secondary])
assert ri._estimate_target_is_on_this_disk("org/Model-GGUF") is True
assert ri._estimate_target_is_on_this_disk("ORG/model-gguf") is True
assert ri._estimate_target_is_on_this_disk("org/Other-GGUF") is False
def test_the_on_disk_check_fails_open(self, monkeypatch, gqa_gguf):
def boom(*a, **kw):
raise OSError("cache root unreadable")
monkeypatch.setattr(ri, "_estimate_hf_cache_roots", boom)
assert ri._estimate_target_is_on_this_disk("org/anything") is True
monkeypatch.setattr(ri, "_estimate_hf_cache_roots", lambda: [])
assert ri._estimate_target_is_on_this_disk("org/anything") is True
assert ri._estimate_target_is_on_this_disk(gqa_gguf) is True
def test_the_cache_evictions_are_serialised(self):
# Runs in to_thread workers: concurrent eviction must not raise during min().
for source in (
inspect.getsource(ri._gguf_resident_file_gb),
inspect.getsource(ri._cached_estimate_config),
):
evict = source[source.index("_CACHE_MAX") :]
assert "_ESTIMATE_CACHE_LOCK" in source
assert source.index("_ESTIMATE_CACHE_LOCK") < source.index("_CACHE_MAX"), (
"the lock must be taken BEFORE the length check, or the check-then-"
"insert it protects is still torn"
)
assert "min(" in evict
def test_the_route_never_reaps_processes_or_leaks_an_atexit_handler(
self, monkeypatch, gqa_gguf
):
# Sizing backends must not reap llama-servers or register atexit handlers.
import atexit
from core.inference.llama_cpp import LlamaCppBackend
reaps = []
raw = inspect.getattr_static(LlamaCppBackend, "_kill_orphaned_servers").__func__
monkeypatch.setattr(
LlamaCppBackend,
"_kill_orphaned_servers",
staticmethod(lambda: reaps.append(1)),
)
assert callable(raw)
config = SimpleNamespace(
identifier = "local/model",
gguf_file = gqa_gguf,
is_gguf = True,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: config)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 2.0)
before = atexit._ncallbacks()
for i in range(5):
ri._estimate_files_cache.clear()
ri._estimate_config_cache.clear()
assert _estimate(model_path = gqa_gguf, n_ctx = 4096 + i).available is True
assert reaps == [], f"the estimate reaped orphaned servers {len(reaps)}x"
assert atexit._ncallbacks() == before, (
f"the estimate leaked {atexit._ncallbacks() - before} atexit handlers over "
f"five requests; each one retains a whole backend"
)
def test_the_inert_probe_mode_is_opt_in(self):
from core.inference.llama_cpp import LlamaCppBackend
sig = inspect.signature(LlamaCppBackend.__init__)
param = sig.parameters["manages_processes"]
assert param.default is True
assert param.kind is inspect.Parameter.KEYWORD_ONLY
def test_expert_offload_from_the_extras_is_declared_unmodelled(self, monkeypatch, gqa_gguf):
config = SimpleNamespace(identifier = "local/model", gguf_file = gqa_gguf, is_gguf = True)
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: config)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 2.0)
def flagged(**kw):
return _estimate(model_path = gqa_gguf, n_ctx = 8192, **kw).moe_offload_unmodelled
assert flagged(gpu_memory_mode = "auto", llama_extra_args = ["--n-cpu-moe", "8"]) is True
assert flagged(gpu_memory_mode = "auto", llama_extra_args = ["-cmoe"]) is True
assert flagged(gpu_memory_mode = "auto", llama_extra_args = ["-ot", "exps=CPU"]) is True
assert flagged(gpu_memory_mode = "auto", llama_extra_args = ["-ncmoe", "0"]) is False
assert flagged(gpu_memory_mode = "auto", llama_extra_args = ["--top-k", "40"]) is False
assert flagged(gpu_memory_mode = "manual", llama_extra_args = ["-ncmoe", "8"]) is False
assert flagged(gpu_memory_mode = "manual", n_cpu_moe = 8) is True
def test_the_device_count_uses_the_effective_split_mode(self, monkeypatch, gqa_gguf):
config = SimpleNamespace(identifier = "local/model", gguf_file = gqa_gguf, is_gguf = True)
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: config)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 2.0)
asked: list[bool] = []
def _count(
gpu_ids,
_unused,
*,
tensor_parallel = False,
):
asked.append(tensor_parallel)
return 2
monkeypatch.setattr(ri, "_guard_device_count", _count)
_estimate(
model_path = gqa_gguf,
n_ctx = 8192,
tensor_parallel = False,
llama_extra_args = ["--split-mode", "tensor"],
selected_gpu_ids = [0, 1],
)
assert asked == [True]
_estimate(
model_path = gqa_gguf,
n_ctx = 8192,
tensor_parallel = True,
llama_extra_args = ["--split-mode", "tensor"],
selected_gpu_ids = [0],
)
assert asked == [True, False]
def test_unresolvable_config_is_unsizable(self, monkeypatch):
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: None)
resp = _estimate(model_path = "org/does-not-exist")
assert resp.available is False
assert resp.reason == "unsizable"
def test_local_gguf_is_priced_end_to_end(self, monkeypatch, gqa_gguf):
config = SimpleNamespace(identifier = "local/model", gguf_file = gqa_gguf, is_gguf = True)
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: config)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 2.0)
resp = _estimate(
model_path = gqa_gguf,
n_ctx = 8192,
cache_type_kv = "q8_0",
gpu_memory_mode = "manual",
gpu_layers = 6,
n_cpu_moe = 2,
)
assert resp.available is True
assert resp.reason is None
assert resp.weights_bytes == int(round(2.0 * _GIB))
assert resp.total_bytes == resp.weights_bytes + resp.kv_bytes + resp.compute_bytes
assert resp.kv_estimable is True
assert resp.n_ctx == 8192
assert resp.cache_type_kv == "q8_0"
assert resp.layer_count == _GQA_FIELDS["block_count"]
assert resp.gpu_layers == 6
assert resp.moe_offload_unmodelled is True
def test_cached_repo_is_priced_from_disk_not_from_a_listing(self, monkeypatch, gqa_gguf):
# Both remote-arm helpers raise, so reaching either fails the test.
def _network(*a, **kw):
raise AssertionError("estimate-memory must not price a cached repo remotely")
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: _repo_config())
monkeypatch.setattr(ri, "_local_gguf_main_path", lambda config: gqa_gguf)
monkeypatch.setattr(ri, "_remote_gguf_compute_reserve_gb", _network)
resp = _estimate(model_path = "org/model-GGUF", gguf_variant = "Q4_K_M", n_ctx = 4096)
assert resp.available is True
assert resp.kv_estimable is True
assert resp.layer_count == _GQA_FIELDS["block_count"]
assert resp.weights_bytes > 0
assert resp.n_ctx == 4096
class TestPrefillOptionAvailabilityRoute:
@pytest.mark.parametrize(
"option,scope,takes",
[
("int8", "nax_quantized_linear", lambda m, int8_prefill = None: m),
("ane", "ane_prefill_mlp", lambda m, ane_prefill = None: m),
],
)
@pytest.mark.parametrize(
"downloaded,takes_option,answer",
[
(True, True, (True, None)),
(False, True, (False, "not_downloaded")),
(True, False, (False, "unsupported_zoo")),
],
)
def test_the_answer(
self, monkeypatch, tmp_path, option, scope, takes, downloaded, takes_option, answer
):
from core.inference import mlx_inference
import models.inference as models
asked = []
zoo = _types.ModuleType("unsloth_zoo.mlx.inference")
setattr(zoo, scope, takes if takes_option else len)
verdict = SimpleNamespace(available = True, reason = "")
setattr(
zoo,
f"{option}_prefill_checkpoint_available",
lambda path: asked.append(path) or verdict,
)
monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.inference", zoo)
monkeypatch.setattr(mlx_inference, f"_{option}_prefill_checkpoint_cache", {})
model_dir = str(tmp_path) if downloaded else None
TestEstimateMemoryRoute._mlx_target(monkeypatch, model_dir)
request = getattr(models, f"{option.capitalize()}PrefillAvailabilityRequest")(
model_path = "org/model"
)
route = getattr(ri, f"{option}_prefill_availability")
resp = asyncio.run(route(request, current_subject = "test"))
assert (resp.available, resp.reason) == answer
if answer[0]:
status = getattr(mlx_inference, f"mlx_{option}_prefill_checkpoint_status")
assert status(model_dir) == (True, "")
assert asked == ([model_dir] if answer[0] else [])
class TestFastVerifyAvailabilityRoute:
@pytest.mark.parametrize(
"capable,verdict,answer",
[
(True, (True, ""), (True, None)),
(True, (False, "no_lane_kernel"), (False, "no_lane_kernel")),
(False, (True, ""), (False, "unsupported_zoo")),
],
)
def test_the_answer(self, monkeypatch, tmp_path, capable, verdict, answer):
import models.inference as models
zoo = _types.ModuleType("unsloth_zoo.mlx.inference")
zoo.inference_scopes = (lambda m, fast_verify = None: m) if capable else (lambda m: m)
zoo.fast_verify_checkpoint_available = lambda path: SimpleNamespace(
available = verdict[0], reason = verdict[1]
)
monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.inference", zoo)
TestEstimateMemoryRoute._mlx_target(monkeypatch, str(tmp_path))
request = models.FastVerifyAvailabilityRequest(model_path = "org/model")
resp = asyncio.run(ri.fast_verify_availability(request, current_subject = "test"))
assert (resp.available, resp.reason) == answer
class TestParallelSlotResolution:
"""Blank Parallel Slots means the server default (four in a standard launch), so price it, not one."""
@pytest.fixture(autouse = True)
def _local_model(self, monkeypatch, gqa_gguf):
config = SimpleNamespace(
identifier = "local/model",
gguf_file = gqa_gguf,
is_gguf = True,
gguf_mmproj_file = None,
)
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: config)
monkeypatch.setattr(ri, "_local_gguf_main_path", lambda cfg: gqa_gguf)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 1.0)
def test_omitted_slots_inherit_the_server_default(self):
resp = _estimate(
model_path = "local/model",
n_ctx = 8192,
fastapi_request = _request_with_slots(4),
)
assert resp.n_parallel == 4
def test_an_explicit_count_still_wins(self):
resp = _estimate(
model_path = "local/model",
n_ctx = 8192,
n_parallel = 1,
fastapi_request = _request_with_slots(4),
)
assert resp.n_parallel == 1
def test_more_slots_cost_more(self):
one = _estimate(
model_path = "local/model",
n_ctx = 8192,
fastapi_request = _request_with_slots(1),
)
four = _estimate(
model_path = "local/model",
n_ctx = 8192,
fastapi_request = _request_with_slots(4),
)
# Under unified KV a GQA model shares cells, so the cache assertion is >=.
assert four.compute_bytes > one.compute_bytes
assert four.kv_bytes >= one.kv_bytes
assert four.total_bytes > one.total_bytes
def test_a_missing_app_state_falls_back_to_one(self):
resp = _estimate(model_path = "local/model", n_ctx = 8192)
assert resp.n_parallel == 1
class TestNativeLeaseOperation:
"""Leases are signed per operation and must be verified under a mintable name, not estimate-memory."""
def test_the_route_verifies_the_validate_model_grant(self, monkeypatch):
seen = {}
def _resolve(
request,
*,
operation,
resolved_ollama_path = None,
):
seen["operation"] = operation
return ("/tmp/dropped.gguf", "dropped.gguf", True)
monkeypatch.setattr(ri, "_resolve_model_identifier_for_request", _resolve)
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: None)
_estimate(model_path = "dropped.gguf", native_path_lease = "signed")
assert seen["operation"] == "validate-model"
def test_the_operation_is_one_the_client_can_mint(self):
types_ts = (
Path(__file__).resolve().parents[2] / "frontend/src/features/native-intents/types.ts"
).read_text(encoding = "utf-8")
assert '"validate-model"' in types_ts
assert "estimate-memory" not in types_ts
from core.inference.llama_cpp import ( # noqa: E402
_extra_args_draft_offloaded_to_cpu as _draft_on_cpu,
)
class TestDrafterAccounting:
"""A drafter costs its KV cache and rollback state too, which grow with context, not only its file."""
@pytest.fixture
def config(self, gqa_gguf, tmp_path):
drafter = tmp_path / "mtp-model.gguf"
drafter.write_bytes(
_make_gguf_bytes(
"qwen3",
{
"general.architecture": "qwen3",
**{f"qwen3.{k}": v for k, v in _GQA_FIELDS.items()},
},
)
)
return SimpleNamespace(
identifier = "local/model",
gguf_file = gqa_gguf,
is_gguf = True,
gguf_mmproj_file = None,
gguf_mtp_file = str(drafter),
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def test_the_charged_drafter_is_found_by_the_bytes_it_added(self, config, tmp_path):
size = os.path.getsize(config.gguf_mtp_file)
assert ri._charged_drafter_path(config, size) == (config.gguf_mtp_file, "mtp")
assert ri._charged_drafter_path(config, size + 1) is None
assert ri._charged_drafter_path(config, 0) is None
def test_drafter_runtime_grows_with_context(self, config, gqa_gguf, monkeypatch):
drafter_bytes = os.path.getsize(config.gguf_mtp_file)
main_gb = 1.0
def _files(
cfg,
*,
llama_extra_args = None,
**kw,
):
pinned = _draft_on_cpu(list(llama_extra_args or ()))
return main_gb + (0.0 if pinned else drafter_bytes / 1024**3)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
small = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 2048)
large = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 32768)
assert small.weights_bytes == large.weights_bytes
drafter_runtime_small = small.total_bytes - small.weights_bytes - small.kv_bytes
drafter_runtime_large = large.total_bytes - large.weights_bytes - large.kv_bytes
assert drafter_runtime_large > drafter_runtime_small
def test_a_cpu_pinned_drafter_stays_in_the_total(self, config, gqa_gguf, monkeypatch):
drafter_bytes = os.path.getsize(config.gguf_mtp_file)
main_gb = 1.0
def _files(
cfg,
*,
llama_extra_args = None,
**kw,
):
pinned = _draft_on_cpu(list(llama_extra_args or ()))
return main_gb + (0.0 if pinned else drafter_bytes / 1024**3)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
on_gpu = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 8192)
on_cpu = ri._gguf_memory_breakdown(
config, gqa_gguf, n_ctx = 8192, llama_extra_args = ["--spec-draft-ngl", "0"]
)
assert on_cpu.total_bytes == on_gpu.total_bytes
assert on_cpu.weights_bytes == on_gpu.weights_bytes
assert on_cpu.gpu_bytes < on_gpu.gpu_bytes
def test_the_drafter_runtime_reports_its_own_gpu_share(self, config, gqa_gguf, monkeypatch):
drafter_bytes = os.path.getsize(config.gguf_mtp_file)
def _files(
cfg,
*,
llama_extra_args = None,
**kw,
):
pinned = _draft_on_cpu(list(llama_extra_args or ()))
return 1.0 + (0.0 if pinned else drafter_bytes / 1024**3)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
verify_rows = 7 * 1024 * 1024
monkeypatch.setattr(
ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: verify_rows
)
on_gpu = ri._gguf_memory_breakdown(config, gqa_gguf, n_ctx = 8192)
assert on_gpu.drafter_runtime_bytes > 0
assert on_gpu.drafter_runtime_gpu_bytes == on_gpu.drafter_runtime_bytes
on_cpu = ri._gguf_memory_breakdown(
config, gqa_gguf, n_ctx = 8192, llama_extra_args = ["--spec-draft-ngl", "0"]
)
assert on_cpu.drafter_runtime_gpu_bytes == verify_rows
nkvo = ri._gguf_memory_breakdown(
config, gqa_gguf, n_ctx = 8192, llama_extra_args = ["--no-kv-offload"]
)
assert nkvo.kv_on_gpu is False
assert nkvo.drafter_runtime_gpu_bytes > 0
for b in (on_gpu, on_cpu, nkvo):
assert 0 <= b.drafter_runtime_gpu_bytes <= b.drafter_runtime_bytes
class TestDeviceCount:
"""A pinned layer split pays per-device overhead, not just tensor mode."""
def test_a_pinned_layer_split_counts_its_cards(self):
assert ri._guard_device_count([0, 1], None, tensor_parallel = False) == 2
assert ri._guard_device_count([0, 1, 2], None, tensor_parallel = False) == 3
assert ri._guard_device_count(None, None, tensor_parallel = False) == 1
_HYBRID_FIELDS = {
**_GQA_FIELDS,
"block_count": 24,
"full_attention_interval": 4,
"ssm.inner_size": 4096,
"ssm.state_size": 128,
"ssm.group_count": 8,
"ssm.conv_kernel": 4,
}
class TestBlankDraftDepth:
"""Blank Draft Tokens is the default depth; pricing it as zero drops the rollback state."""
@pytest.fixture
def hybrid(self, tmp_path) -> str:
return _write_gguf(tmp_path, "qwen3next", _HYBRID_FIELDS, name = "hybrid.gguf")
@pytest.fixture
def config(self, hybrid, tmp_path):
drafter = tmp_path / "mtp-hybrid.gguf"
drafter.write_bytes(
_make_gguf_bytes(
"qwen3",
{
"general.architecture": "qwen3",
**{f"qwen3.{k}": v for k, v in _GQA_FIELDS.items()},
},
)
)
return SimpleNamespace(
identifier = "local/hybrid",
gguf_file = hybrid,
is_gguf = True,
gguf_mmproj_file = None,
gguf_mtp_file = str(drafter),
gguf_dspark_file = None,
gguf_dflash_file = None,
)
@pytest.fixture(autouse = True)
def _fixed_files(self, config, monkeypatch):
drafter_bytes = os.path.getsize(config.gguf_mtp_file)
def _files(
cfg,
*,
llama_extra_args = None,
**kw,
):
pinned = _draft_on_cpu(list(llama_extra_args or ()))
return 1.0 + (0.0 if pinned else drafter_bytes / 1024**3)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
def _priced(self, config, hybrid, **kw):
return ri._gguf_memory_breakdown(config, hybrid, n_ctx = 8192, n_parallel = 4, **kw)
def test_blank_prices_the_launch_default_not_zero(self, config, hybrid):
zero = self._priced(config, hybrid, spec_draft_n_max = 0)
one = self._priced(config, hybrid, spec_draft_n_max = 1)
blank = self._priced(config, hybrid, spec_draft_n_max = None)
per_copy = one.total_bytes - zero.total_bytes
assert per_copy > 0, "fixture is not hybrid-Mamba; the test would prove nothing"
assert blank.total_bytes > zero.total_bytes
# 2 on a GPU box, 3 without; not pinned so it passes with or without a card.
assert (blank.total_bytes - zero.total_bytes) // per_copy in (2, 3)
assert blank.gpu_bytes > zero.gpu_bytes
def test_an_explicit_depth_still_wins(self, config, hybrid):
zero = self._priced(config, hybrid, spec_draft_n_max = 0)
one = self._priced(config, hybrid, spec_draft_n_max = 1)
per_copy = one.total_bytes - zero.total_bytes
five = self._priced(config, hybrid, spec_draft_n_max = 5)
assert five.total_bytes - zero.total_bytes == 5 * per_copy
def test_an_extras_depth_wins_over_a_blank_field(self, config, hybrid):
zero = self._priced(config, hybrid, spec_draft_n_max = 0)
one = self._priced(config, hybrid, spec_draft_n_max = 1)
per_copy = one.total_bytes - zero.total_bytes
pinned = self._priced(
config,
hybrid,
spec_draft_n_max = None,
llama_extra_args = ["--spec-draft-n-max", "5"],
)
assert pinned.total_bytes - zero.total_bytes == 5 * per_copy
def test_depth_resolution_follows_the_launch_precedence(self, config):
drafter = config.gguf_mtp_file
assert ri._estimate_draft_n_max(config, drafter, requested = 7, extras = []) == 7
assert (
ri._estimate_draft_n_max(config, drafter, requested = 7, extras = ["--draft-max", "4"]) == 4
)
assert ri._estimate_draft_n_max(config, drafter, requested = 0, extras = []) == 0
assert ri._estimate_draft_n_max(config, drafter, requested = None, extras = []) in (2, 3)
dspark = SimpleNamespace(gguf_dspark_file = "/tmp/dspark-model.gguf")
assert (
ri._estimate_draft_n_max(dspark, "/tmp/dspark-model.gguf", requested = None, extras = [])
== 3
)
class TestQuantSubdirCompanions:
"""Projector and drafter sit at the snapshot root above the quant dir, so search must walk up to it."""
def test_a_projector_at_the_snapshot_root_is_found(self, tmp_path):
quant_dir = tmp_path / "UD-Q4_K_XL"
quant_dir.mkdir()
weight = _write_gguf(
quant_dir, "qwen3", _GQA_FIELDS, name = "model-UD-Q4_K_XL-00001-of-00002.gguf"
)
projector = _write_gguf(tmp_path, "clip", {"block_count": 2}, name = "mmproj-F16.gguf")
localized = ri._localized_estimate_config(_repo_config(is_vision = True), weight)
assert localized.gguf_mmproj_file == projector
def test_a_drafter_at_the_snapshot_root_is_found(self, tmp_path):
quant_dir = tmp_path / "Q4_K_M"
quant_dir.mkdir()
weight = _write_gguf(quant_dir, "qwen3", _GQA_FIELDS, name = "model-Q4_K_M.gguf")
drafter = _write_gguf(tmp_path, "qwen3", _GQA_FIELDS, name = "mtp-model.gguf")
localized = ri._localized_estimate_config(_repo_config(), weight)
assert localized.gguf_mtp_file == drafter
def test_a_flat_layout_still_scans_only_the_weights_directory(self, tmp_path):
model_dir = tmp_path / "some-model"
model_dir.mkdir()
weight = _write_gguf(model_dir, "qwen3", _GQA_FIELDS, name = "model-Q4_K_M.gguf")
_write_gguf(tmp_path, "clip", {"block_count": 2}, name = "mmproj-F16.gguf")
localized = ri._localized_estimate_config(_repo_config(is_vision = True), weight)
assert localized.gguf_mmproj_file is None
class TestDrafterEdgeCases:
"""The narrow drafter paths, each of which produced a wrong answer or a 500."""
@pytest.fixture
def bare_config(self, gqa_gguf):
"""A model shipping no sidecar of any kind."""
return SimpleNamespace(
identifier = "local/model",
gguf_file = gqa_gguf,
is_gguf = True,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def test_a_cpu_pin_without_a_drafter_does_not_crash(self, bare_config, gqa_gguf, monkeypatch):
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 1.0)
breakdown = ri._gguf_memory_breakdown(
bare_config, gqa_gguf, n_ctx = 4096, llama_extra_args = ["--spec-draft-ngl", "0"]
)
assert breakdown is not None
assert breakdown.drafter_runtime_bytes == 0
assert breakdown.total_bytes > 0
def test_a_custom_model_draft_is_carried_into_runtime_sizing(
self, bare_config, gqa_gguf, tmp_path
):
custom = tmp_path / "my-drafter.gguf"
custom.write_bytes(Path(gqa_gguf).read_bytes())
found = ri._charged_drafter_path(bare_config, 4096, extras = ["--model-draft", str(custom)])
assert found == (str(custom), "extras")
assert (
ri._charged_drafter_path(
bare_config, 4096, extras = ["--spec-draft-hf", "org/drafter-GGUF"]
)
is None
)
assert (
ri._charged_drafter_path(bare_config, 0, extras = ["--model-draft", str(custom)]) is None
)
def test_draft_cache_overrides_come_from_the_extras(self):
from core.inference.llama_cpp import _extra_args_draft_cache_types
assert _extra_args_draft_cache_types(["--cache-type-k-draft", "f32"]) == ("f32", None)
assert _extra_args_draft_cache_types(["--cache-type-v-draft", "q8_0"]) == (None, "q8_0")
class TestSpeculativeModeTerms:
"""Target terms follow the engaged mode, or DSpark/DFlash would bill a second KV copy."""
@pytest.fixture
def mla(self, tmp_path) -> str:
return _write_gguf(
tmp_path,
"deepseek2",
{**_GQA_FIELDS, "attention.kv_lora_rank": 512},
name = "mla.gguf",
)
def _config(self, mla, tmp_path, kind: str):
drafter = tmp_path / f"{kind}-drafter.gguf"
drafter.write_bytes(Path(mla).read_bytes())
return SimpleNamespace(
identifier = "local/mla",
gguf_file = mla,
is_gguf = True,
gguf_mmproj_file = None,
gguf_mtp_file = str(drafter) if kind == "mtp" else None,
gguf_dspark_file = str(drafter) if kind == "dspark" else None,
gguf_dflash_file = None,
)
@pytest.fixture
def priced_files(self, mla, tmp_path, monkeypatch):
"""The files term as the real one behaves: drafter charged unless pinned."""
drafter_bytes = os.path.getsize(mla)
def _files(
cfg,
*,
llama_extra_args = None,
**kw,
):
pinned = _draft_on_cpu(list(llama_extra_args or ()))
return 1.0 + (0.0 if pinned else drafter_bytes / 1024**3)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
def _target_ctx_copy(self, mla, n_ctx: int) -> int:
probe = ri.LlamaCppBackend()
probe._read_gguf_metadata(mla)
return probe._estimate_kv_cache_bytes(n_ctx, "f16")
def test_a_dspark_drafter_is_not_charged_the_targets_context(
self, mla, tmp_path, priced_files, monkeypatch
):
monkeypatch.setattr(ri.LlamaCppBackend, "_mtp_draft_compute_bytes", lambda *a, **k: 0)
ctx = 131072
mtp = ri._gguf_memory_breakdown(self._config(mla, tmp_path, "mtp"), mla, n_ctx = ctx)
dspark = ri._gguf_memory_breakdown(self._config(mla, tmp_path, "dspark"), mla, n_ctx = ctx)
assert mtp.drafter_runtime_bytes - dspark.drafter_runtime_bytes == self._target_ctx_copy(
mla, ctx
)
assert dspark.total_bytes < mtp.total_bytes
def test_a_gpu_drafter_is_charged_when_the_target_keeps_no_layers(
self, mla, tmp_path, priced_files, monkeypatch
):
# A separate drafter ignores --gpu-layers (draft placement defaults to auto).
monkeypatch.setattr(ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: 0)
config = self._config(mla, tmp_path, "dspark")
manual = dict(gpu_memory_mode = "manual", gpu_layers = 0, n_ctx = 131072)
on_gpu = ri._gguf_memory_breakdown(config, mla, **manual)
pinned = ri._gguf_memory_breakdown(
config, mla, llama_extra_args = ["--spec-draft-ngl", "0"], **manual
)
assert on_gpu.drafter_runtime_bytes > 0
assert (
on_gpu.gpu_bytes - pinned.gpu_bytes
== on_gpu.drafter_runtime_bytes + os.path.getsize(mla)
)
def test_the_target_side_of_the_reserve_stays_with_the_target(
self, mla, tmp_path, priced_files, monkeypatch
):
monkeypatch.setattr(ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: 0)
ctx = 131072
config = self._config(mla, tmp_path, "mtp")
copy_bytes = self._target_ctx_copy(mla, ctx)
manual = dict(gpu_memory_mode = "manual", gpu_layers = 0, n_ctx = ctx)
no_layers = ri._gguf_memory_breakdown(config, mla, **manual)
no_layers_pinned = ri._gguf_memory_breakdown(
config, mla, llama_extra_args = ["--spec-draft-ngl", "0"], **manual
)
assert copy_bytes > 0
assert no_layers.gpu_bytes - no_layers_pinned.gpu_bytes == (
no_layers.drafter_runtime_bytes - copy_bytes + os.path.getsize(mla)
)
pinned = ri._gguf_memory_breakdown(
config, mla, n_ctx = ctx, llama_extra_args = ["--spec-draft-ngl", "0"]
)
unpinned = ri._gguf_memory_breakdown(config, mla, n_ctx = ctx)
assert unpinned.gpu_bytes - pinned.gpu_bytes == (
unpinned.drafter_runtime_bytes - copy_bytes + os.path.getsize(mla)
)
assert pinned.gpu_bytes - no_layers_pinned.gpu_bytes > copy_bytes
def test_verification_rows_follow_the_targets_layers(
self, mla, tmp_path, priced_files, monkeypatch
):
"""Target compute runs on the GPU with the target's layers, whichever device the drafter uses."""
verify_rows = 7 * 1024 * 1024
monkeypatch.setattr(
ri.LlamaCppBackend, "_spec_verify_rows_bytes", lambda *a, **k: verify_rows
)
config = self._config(mla, tmp_path, "dspark")
pin = ["--spec-draft-ngl", "0"]
no_layers = dict(gpu_memory_mode = "manual", gpu_layers = 0)
target_gpu = ri._gguf_memory_breakdown(config, mla, n_ctx = 8192, llama_extra_args = pin)
target_cpu = ri._gguf_memory_breakdown(config, mla, n_ctx = 8192, **no_layers)
assert target_gpu.drafter_runtime_gpu_bytes == verify_rows
assert target_cpu.drafter_runtime_gpu_bytes == (
target_cpu.drafter_runtime_bytes - verify_rows
)
def test_extras_owning_the_spec_block_price_the_builds_depth(self, mla):
# Once extras name --spec-type no depth is emitted; the build default applies.
config = SimpleNamespace(gguf_dspark_file = None)
owned = ["--spec-type", "draft-mtp"]
assert ri._estimate_draft_n_max(config, mla, requested = None, extras = owned) == 16
assert ri._estimate_draft_n_max(config, mla, requested = 2, extras = owned) == 16
assert (
ri._estimate_draft_n_max(
config, mla, requested = 2, extras = [*owned, "--spec-draft-n-max", "5"]
)
== 5
)
assert ri._estimate_draft_n_max(config, mla, requested = 2, extras = []) == 2
def test_each_mode_charges_only_the_terms_it_allocates(self):
assert ri._estimate_spec_mode_terms("mtp", []) == (True, True)
assert ri._estimate_spec_mode_terms("dspark", []) == (False, True)
assert ri._estimate_spec_mode_terms("dflash", []) == (False, True)
assert ri._estimate_spec_mode_terms("extras", []) == (False, False)
assert ri._estimate_spec_mode_terms("extras", [], studio_emits_mtp = True) == (True, True)
assert ri._estimate_spec_mode_terms(
"extras", ["--spec-type", "draft-simple"], studio_emits_mtp = True
) == (False, False)
assert ri._estimate_spec_mode_terms("extras", ["--spec-type", "draft-simple"]) == (
False,
False,
)
assert ri._estimate_spec_mode_terms("extras", ["--spec-type", "draft-mtp"]) == (True, True)
assert ri._estimate_spec_mode_terms("extras", ["--spec-type", "draft-eagle3"]) == (
False,
True,
)
class TestLaunchShapedPricing:
"""Each case is a setting the panel resolves differently from load_model, enough to flip the verdict."""
@pytest.fixture
def swa(self, tmp_path) -> str:
return _write_gguf(
tmp_path,
"gemma3",
{**_GQA_FIELDS, "attention.sliding_window": 4096, "context_length": 262144},
name = "swa.gguf",
)
@pytest.fixture
def spec_config(self, swa, tmp_path, monkeypatch):
drafter = tmp_path / "mtp-swa.gguf"
drafter.write_bytes(Path(swa).read_bytes())
drafter_bytes = drafter.stat().st_size
def _files(
cfg,
*,
llama_extra_args = None,
**kw,
):
pinned = _draft_on_cpu(list(llama_extra_args or ()))
return 1.0 + (0.0 if pinned else drafter_bytes / 1024**3)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
return SimpleNamespace(
identifier = "local/swa",
gguf_file = swa,
is_gguf = True,
gguf_mmproj_file = None,
gguf_mtp_file = str(drafter),
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def test_the_drafter_cache_is_priced_at_the_targets_layout(self, spec_config, swa):
windowed = ri._gguf_memory_breakdown(spec_config, swa, n_ctx = 131072)
full = ri._gguf_memory_breakdown(
spec_config, swa, n_ctx = 131072, llama_extra_args = ["--swa-full"]
)
assert full.kv_bytes - full.kv_checkpoint_bytes > (
windowed.kv_bytes - windowed.kv_checkpoint_bytes
)
assert full.kv_checkpoint_bytes == 0
assert windowed.kv_checkpoint_bytes > 0
# Compared by difference: the draft compute floor does not follow --swa-full.
assert full.drafter_runtime_bytes - windowed.drafter_runtime_bytes == (
(full.kv_bytes - full.kv_checkpoint_bytes)
- (windowed.kv_bytes - windowed.kv_checkpoint_bytes)
)
assert windowed.drafter_runtime_bytes > windowed.kv_bytes - windowed.kv_checkpoint_bytes
def test_a_cpu_device_selection_takes_the_weights_off_the_gpu(self, spec_config, swa):
"""--device none keeps weights off the GPU whatever -ngl says, like the loader's gate."""
priced = dict(n_ctx = 32768, cache_type_kv = "f16")
on_gpu = ri._gguf_memory_breakdown(spec_config, swa, **priced)
for flag in (["--device", "none"], ["--device", "cpu"], ["-dev", "none"]):
cpu_only = ri._gguf_memory_breakdown(spec_config, swa, llama_extra_args = flag, **priced)
assert cpu_only.gpu_bytes == 0, flag
assert cpu_only.total_bytes > 0
assert on_gpu.gpu_bytes > 0
def test_context_checkpoints_are_charged_to_the_host_not_the_gpu(self, spec_config, swa):
"""Context checkpoints live in host RAM (--cache-ram): count them in total_bytes, not gpu_bytes."""
priced = dict(n_ctx = 131072, cache_type_kv = "f16")
none = ri._gguf_memory_breakdown(spec_config, swa, ctx_checkpoints = 0, **priced)
many = ri._gguf_memory_breakdown(spec_config, swa, ctx_checkpoints = 8, **priced)
assert many.kv_bytes > none.kv_bytes
assert many.total_bytes > none.total_bytes
assert many.gpu_bytes == none.gpu_bytes
def test_the_panel_route_reports_the_checkpoint_share(self, swa, monkeypatch):
config = SimpleNamespace(
identifier = "local/swa",
gguf_file = swa,
is_gguf = True,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: config)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 1.0)
priced = dict(model_path = swa, n_ctx = 131072, cache_type_kv = "f16")
none = _estimate(ctx_checkpoints = 0, **priced)
many = _estimate(ctx_checkpoints = 8, **priced)
assert none.kv_checkpoint_bytes == 0
assert many.kv_checkpoint_bytes > 0
assert many.kv_checkpoint_bytes == many.kv_bytes - none.kv_bytes
assert (many.total_bytes - many.gpu_bytes) - (
none.total_bytes - none.gpu_bytes
) == many.kv_checkpoint_bytes
def test_one_card_is_priced_as_the_layer_load_it_launches(self, spec_config, swa):
# One card launches a layer load. KV stays q4_0: equality catches a reintroduced f16 gate.
priced = dict(n_ctx = 32768, cache_type_kv = "q4_0", tensor_parallel = True, n_devices = 1)
downgraded = ri._gguf_memory_breakdown(
spec_config, swa, tensor_split_possible = False, **priced
)
as_tensor = ri._gguf_memory_breakdown(
spec_config, swa, tensor_split_possible = True, **priced
)
assert downgraded.cache_type_kv == "q4_0"
assert as_tensor.cache_type_kv == "q4_0"
assert downgraded.kv_bytes == as_tensor.kv_bytes
assert downgraded.compute_bytes == as_tensor.compute_bytes
assert downgraded.total_bytes == as_tensor.total_bytes
def test_manual_auto_layers_is_priced_as_the_layer_load_it_launches(self, spec_config, swa):
# Auto layers use --fit, which drops tensor parallelism, so this prices a layer split.
priced = dict(
n_ctx = 32768,
cache_type_kv = "q4_0",
tensor_parallel = True,
n_devices = 2,
gpu_memory_mode = "manual",
tensor_split_possible = True,
)
auto_layers = ri._gguf_memory_breakdown(spec_config, swa, gpu_layers = None, **priced)
explicit = ri._gguf_memory_breakdown(spec_config, swa, gpu_layers = 40, **priced)
assert auto_layers.compute_bytes != explicit.compute_bytes
as_layers = ri._gguf_memory_breakdown(
spec_config, swa, gpu_layers = None, **{**priced, "tensor_parallel": False}
)
assert auto_layers.compute_bytes == as_layers.compute_bytes
def test_manual_zero_layers_is_priced_as_the_cpu_load_it_launches(self, spec_config, swa):
priced = dict(
n_ctx = 32768,
cache_type_kv = "q4_0",
n_devices = 2,
gpu_memory_mode = "manual",
gpu_layers = 0,
tensor_split_possible = True,
)
as_tensor = ri._gguf_memory_breakdown(spec_config, swa, tensor_parallel = True, **priced)
as_layers = ri._gguf_memory_breakdown(spec_config, swa, tensor_parallel = False, **priced)
assert as_tensor.compute_bytes == as_layers.compute_bytes
assert as_tensor.total_bytes == as_layers.total_bytes
def test_an_ngl_in_the_extras_keeps_the_manual_split(self, spec_config, swa):
assert ri._manual_keeps_tensor_split("manual", 0, ["-ngl", "40"]) is True
assert ri._manual_keeps_tensor_split("manual", 40, ["-ngl", "0"]) is False
assert ri._manual_keeps_tensor_split("manual", None, None) is False
assert ri._manual_keeps_tensor_split("manual", 0, None) is False
assert ri._manual_keeps_tensor_split("manual", 1, None) is True
assert ri._manual_keeps_tensor_split("auto", 0, None) is True
assert ri._manual_keeps_tensor_split(None, None, None) is True
assert ri._manual_keeps_tensor_split("manual", 8, ["-ngl", "banana"]) is True
def test_a_one_card_pin_cannot_tensor_split(self):
assert ri._tensor_split_possible([0]) is False
assert ri._tensor_split_possible([0, 1]) is True
assert ri._tensor_split_possible([2, 3, 5]) is True
assert isinstance(ri._tensor_split_possible(None), bool)
def test_the_vision_encoder_costs_more_than_its_projector_file(
self, swa, tmp_path, monkeypatch
):
# Encoder buffers run ~1.3x the projector file (_MMPROJ_VRAM_SAFETY - 1).
projector = tmp_path / "mmproj-swa.gguf"
projector.write_bytes(b"\0" * 1024)
config = SimpleNamespace(
identifier = "local/vision",
gguf_file = swa,
is_gguf = True,
gguf_mmproj_file = str(projector),
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 4.0)
real_size = ri.LlamaCppBackend._get_gguf_size_bytes
monkeypatch.setattr(
ri.LlamaCppBackend,
"_get_gguf_size_bytes",
staticmethod(
lambda path: 3 * _GIB
if os.path.basename(str(path)) == "swa.gguf"
else real_size(path)
),
)
expected = int(1024 * (ri.LlamaCppBackend._MMPROJ_VRAM_SAFETY - 1.0))
resident = ri._gguf_memory_breakdown(config, swa, n_ctx = 8192)
assert resident.projector_runtime_bytes == expected
assert resident.total_bytes == (
resident.weights_bytes + resident.kv_bytes + resident.compute_bytes + expected
)
pinned = ri._gguf_memory_breakdown(
config, swa, n_ctx = 8192, llama_extra_args = ["--no-mmproj-offload"]
)
assert pinned.total_bytes == resident.total_bytes
assert resident.gpu_bytes - pinned.gpu_bytes == expected + 1024
def test_a_model_with_no_projector_is_charged_nothing_for_one(self, spec_config, swa):
assert ri._gguf_memory_breakdown(spec_config, swa, n_ctx = 8192).projector_runtime_bytes == 0
class TestInheritedEnvironment:
"""LLAMA_ARG_SPEC_* is scrubbed if Unsloth owns the spec; the projector path never scrubs."""
@pytest.fixture
def bare(self, gqa_gguf):
return SimpleNamespace(
identifier = "local/bare",
gguf_file = gqa_gguf,
is_gguf = True,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
@pytest.fixture
def fixed_files(self, monkeypatch):
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 1.0)
def test_an_inherited_drafter_is_charged_when_the_extras_own_speculation(
self, bare, gqa_gguf, tmp_path, monkeypatch, fixed_files
):
drafter = _write_gguf(tmp_path, "qwen3", _GQA_FIELDS, name = "inherited-draft.gguf")
monkeypatch.setenv("LLAMA_ARG_SPEC_DRAFT_MODEL", drafter)
owned = ri._gguf_memory_breakdown(
bare, gqa_gguf, n_ctx = 32768, llama_extra_args = ["--spec-type", "draft-mtp"]
)
assert owned.drafter_runtime_bytes > 0
assert owned.weights_bytes == _GIB + os.path.getsize(drafter)
scrubbed = ri._gguf_memory_breakdown(bare, gqa_gguf, n_ctx = 32768)
assert scrubbed.drafter_runtime_bytes == 0
assert scrubbed.weights_bytes == _GIB
def test_an_inherited_draft_depth_is_honoured_only_when_the_child_keeps_it(
self, bare, gqa_gguf, monkeypatch
):
monkeypatch.setenv("LLAMA_ARG_SPEC_DRAFT_N_MAX", "7")
owned = ["--spec-type", "draft-mtp"]
assert ri._estimate_draft_n_max(bare, gqa_gguf, requested = None, extras = owned) == 7
assert ri._estimate_draft_n_max(bare, gqa_gguf, requested = None, extras = []) in (2, 3)
assert (
ri._estimate_draft_n_max(
bare, gqa_gguf, requested = None, extras = [*owned, "--spec-draft-n-max", "5"]
)
== 5
)
def test_an_inherited_projector_is_charged_like_a_configured_one(
self, bare, gqa_gguf, tmp_path, monkeypatch
):
projector = tmp_path / "inherited-mmproj.gguf"
projector.write_bytes(b"\0" * 4096)
monkeypatch.setenv("LLAMA_ARG_MMPROJ", str(projector))
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 4.0)
real_size = ri.LlamaCppBackend._get_gguf_size_bytes
monkeypatch.setattr(
ri.LlamaCppBackend,
"_get_gguf_size_bytes",
staticmethod(
lambda path: 3 * _GIB
if os.path.basename(str(path)) == "model.gguf"
else real_size(path)
),
)
expected = int(4096 * (ri.LlamaCppBackend._MMPROJ_VRAM_SAFETY - 1.0))
resident = ri._gguf_memory_breakdown(bare, gqa_gguf, n_ctx = 8192)
assert resident.projector_runtime_bytes == expected
by_flag = ri._gguf_memory_breakdown(
bare, gqa_gguf, n_ctx = 8192, llama_extra_args = ["--no-mmproj-offload"]
)
assert resident.gpu_bytes - by_flag.gpu_bytes == expected + 4096
monkeypatch.setenv("LLAMA_ARG_MMPROJ_OFFLOAD", "0")
assert ri._gguf_memory_breakdown(bare, gqa_gguf, n_ctx = 8192).gpu_bytes == by_flag.gpu_bytes
def test_the_vulkan_pool_comes_from_the_probed_snapshot(self, monkeypatch):
# Vulkan: CUDA count sees no devices; use the cached /api/system inventory, never re-probe.
fake_main = _types.ModuleType("main")
monkeypatch.setitem(sys.modules, "main", fake_main)
monkeypatch.setattr(ri.LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda *a: True))
def snapshot(count: int):
fake_main._system_gpu_cache = (
0.0,
({}, {"devices": [{"index": i} for i in range(count)]}),
)
snapshot(3)
assert len(ri._cached_inference_devices()) == 3
assert (
ri._guard_device_count(None, ri._cached_inference_devices(), tensor_parallel = True) == 3
)
assert ri._tensor_split_possible(None) is True
snapshot(1)
assert (
ri._guard_device_count(None, ri._cached_inference_devices(), tensor_parallel = True) == 1
)
assert ri._tensor_split_possible(None) is False
fake_main._system_gpu_cache = None
assert ri._cached_inference_devices() is None
assert ri._tensor_split_possible(None) is True
class TestManualNormalizationAndRemoteDrafters:
"""Two more places the panel described a command the loader would not send."""
@pytest.fixture
def bare(self, gqa_gguf):
return SimpleNamespace(
identifier = "local/bare",
gguf_file = gqa_gguf,
is_gguf = True,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def test_manual_prices_the_stripped_extras(self, bare, gqa_gguf, monkeypatch):
# -ncmoe disables pipeline parallelism; Manual strips it pre-launch, Auto keeps it.
monkeypatch.setattr(ri, "_gguf_resident_file_gb", lambda cfg, **kw: 1.0)
priced = dict(n_ctx = 32768, gpu_layers = 99, n_devices = 2)
def compute(mode: str, extras):
return ri._gguf_memory_breakdown(
bare, gqa_gguf, gpu_memory_mode = mode, llama_extra_args = extras, **priced
).compute_bytes
moe = ["--n-cpu-moe", "8"]
assert compute("manual", moe) == compute("manual", None)
assert compute("auto", moe) < compute("auto", None)
def test_a_remote_drafter_is_declared_unsized_rather_than_priced_at_zero(
self, bare, gqa_gguf, monkeypatch
):
from core.inference.llama_cpp import _extra_args_draft_offloaded_to_cpu as pinned
def _files(
cfg,
*,
llama_extra_args = None,
**kw,
):
args = list(llama_extra_args or ())
remote = "--spec-draft-hf" in args
return 1.0 + (2.0 if remote and not pinned(args) else 0.0)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
remote = ri._gguf_memory_breakdown(
bare, gqa_gguf, n_ctx = 32768, llama_extra_args = ["--spec-draft-hf", "org/drafter-GGUF"]
)
assert remote.drafter_kv_unsized is True
assert remote.weights_bytes == 3 * _GIB
assert remote.drafter_runtime_bytes == 0
assert ri._gguf_memory_breakdown(bare, gqa_gguf, n_ctx = 32768).drafter_kv_unsized is False
def test_the_route_carries_the_unsized_flag(self, bare, gqa_gguf, monkeypatch):
from core.inference.llama_cpp import _extra_args_draft_offloaded_to_cpu as pinned
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: bare)
def _files(
cfg,
*,
llama_extra_args = None,
**kw,
):
args = list(llama_extra_args or ())
return 1.0 + (2.0 if "--spec-draft-hf" in args and not pinned(args) else 0.0)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
resp = _estimate(
model_path = gqa_gguf,
n_ctx = 8192,
llama_extra_args = ["--spec-draft-hf", "org/drafter-GGUF"],
)
assert resp.available is True
assert resp.drafter_kv_unsized is True
class TestSpeculationOffChargesNoDrafter:
"""Modes that never emit --model-draft (OFF, ngram-simple, ngram) must not be charged a drafter file."""
@pytest.fixture
def config_with_a_sidecar(self, tmp_path):
target = _write_gguf_with_embeddings(tmp_path, "qwen3", _GQA_FIELDS, name = "target.gguf")
sidecar = tmp_path / "mtp.gguf"
sidecar.write_bytes(Path(target).read_bytes())
return SimpleNamespace(
identifier = "local/target",
gguf_file = target,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = str(sidecar),
gguf_dspark_file = None,
gguf_dflash_file = None,
)
@pytest.mark.parametrize("mode", ["off", "ngram", "ngram-simple"])
def test_a_drafterless_mode_is_not_charged_for_the_sidecar(self, config_with_a_sidecar, mode):
charged = ri._gguf_resident_file_gb(config_with_a_sidecar, speculative_type = "mtp")
quiet = ri._gguf_resident_file_gb(config_with_a_sidecar, speculative_type = mode)
assert charged is not None and quiet is not None
assert quiet < charged
assert quiet == pytest.approx(charged / 2, rel = 0.05)
def test_a_drafting_mode_still_charges_it(self, config_with_a_sidecar):
"""The guard against over-correcting: MTP must keep paying for its drafter."""
with_mtp = ri._gguf_resident_file_gb(config_with_a_sidecar, speculative_type = "mtp")
auto = ri._gguf_resident_file_gb(config_with_a_sidecar, speculative_type = "auto")
assert with_mtp is not None and auto is not None
assert with_mtp == auto
class TestAnUnloadableSidecarIsNotCharged:
"""An mtp-*.gguf lacking token_embd.weight and nextn_shared_target_tensors is dropped, not priced."""
def _config(self, tmp_path, sidecar):
home = tmp_path / sidecar.stem
home.mkdir()
return SimpleNamespace(
identifier = "local/target",
gguf_file = _write_gguf_with_embeddings(home, "qwen3", _GQA_FIELDS, name = "target.gguf"),
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = str(sidecar),
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def test_a_head_only_sidecar_is_uncharged_and_a_complete_one_is_not(self, tmp_path):
import numpy as np
from gguf import GGUFWriter
def drafter(name, tensors):
path = tmp_path / name
writer = GGUFWriter(str(path), "qwen3")
for tensor in tensors:
writer.add_tensor(tensor, np.zeros((512, 512), dtype = np.float32))
writer.write_header_to_file()
writer.write_kv_data_to_file()
writer.write_tensors_to_file()
writer.close()
return path
heads = ["output.weight", "blk.64.nextn.eh_proj.weight"]
head_only = drafter("mtp-head.gguf", heads)
complete = drafter("mtp-full.gguf", ["token_embd.weight", *heads])
assert head_only.stat().st_size > 1024 * 1024
without = ri._gguf_resident_file_gb(
self._config(tmp_path, head_only), speculative_type = "mtp"
)
with_it = ri._gguf_resident_file_gb(
self._config(tmp_path, complete), speculative_type = "mtp"
)
assert without is not None and with_it is not None
assert with_it > without
assert without == pytest.approx(with_it - complete.stat().st_size / (1024**3), rel = 0.02)
class TestAProjectorOverrideIsTheOneCharged:
"""A --mmproj in Advanced Arguments overrides Studio's projector (last wins), so price the one used."""
@pytest.fixture
def vision_config(self, tmp_path):
target = _write_gguf(tmp_path, "qwen3", _GQA_FIELDS, name = "target.gguf")
small = tmp_path / "mmproj-small.gguf"
small.write_bytes(b"s" * 4096)
big = tmp_path / "mmproj-big.gguf"
big.write_bytes(b"b" * (4096 * 8))
config = SimpleNamespace(
identifier = "local/vision",
gguf_file = target,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = str(small),
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
return config, str(small), str(big)
def test_the_override_replaces_the_configured_projector_in_the_files(self, vision_config):
config, small, big = vision_config
configured = ri._gguf_resident_file_gb(config)
overridden = ri._gguf_resident_file_gb(config, llama_extra_args = ["--mmproj", big])
assert configured is not None and overridden is not None
assert overridden > configured
delta = (overridden - configured) * 1024**3
assert delta == pytest.approx(Path(big).stat().st_size - Path(small).stat().st_size, abs = 8)
def test_the_overrides_encoder_runtime_is_charged_too(self, vision_config):
config, _small, big = vision_config
charged = ri._charged_projector_bytes(config, ["--mmproj", big], False)
assert charged == Path(big).stat().st_size
def test_the_short_flag_is_read_the_same_way(self, vision_config):
config, _small, big = vision_config
assert ri._charged_projector_bytes(config, ["-mm", big], False) == Path(big).stat().st_size
def test_no_override_still_charges_the_configured_projector(self, vision_config):
config, small, _big = vision_config
assert ri._charged_projector_bytes(config, [], False) == Path(small).stat().st_size
class TestAnEmbeddedMtpHeadIsPriced:
"""An embedded NextN head has no file but allocates a draft cache, sized from nextn_predict_layers."""
@pytest.fixture
def nextn_model(self, tmp_path):
gguf = _write_gguf(
tmp_path,
"qwen3",
{**_GQA_FIELDS, "context_length": 262144, "nextn_predict_layers": 2},
name = "nextn.gguf",
)
return gguf, SimpleNamespace(
identifier = "local/nextn",
gguf_file = gguf,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def test_the_head_is_charged_when_speculation_can_engage(self, nextn_model):
gguf, config = nextn_model
priced = ri._gguf_memory_breakdown(config, gguf, n_ctx = 131072)
assert priced is not None
assert priced.drafter_runtime_bytes > 0
assert priced.total_bytes > priced.weights_bytes + priced.kv_bytes
def test_turning_speculation_off_charges_no_head(self, nextn_model):
gguf, config = nextn_model
off = ri._gguf_memory_breakdown(config, gguf, n_ctx = 131072, speculative_type = "off")
assert off is not None
assert off.drafter_runtime_bytes == 0
def test_a_model_without_a_head_is_unaffected(self, tmp_path):
"""The guard against charging every model: no NextN key, nothing priced."""
plain = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 262144})
config = SimpleNamespace(
identifier = "local/plain",
gguf_file = plain,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
priced = ri._gguf_memory_breakdown(config, plain, n_ctx = 131072)
assert priced is not None
assert priced.drafter_runtime_bytes == 0
def test_a_draft_pin_does_not_move_the_embedded_head(self, nextn_model):
"""--spec-draft-ngl 0 neither disables nor moves an embedded head, keeping the target's
placement."""
gguf, config = nextn_model
unpinned = ri._gguf_memory_breakdown(config, gguf, n_ctx = 131072)
assert unpinned is not None
assert unpinned.drafter_runtime_gpu_bytes > 0
for pin in (["--spec-draft-ngl", "0"], ["--spec-draft-device", "cpu"]):
pinned = ri._gguf_memory_breakdown(config, gguf, n_ctx = 131072, llama_extra_args = pin)
assert pinned is not None, pin
assert pinned.drafter_runtime_bytes > 0, pin
assert pinned.drafter_runtime_gpu_bytes == unpinned.drafter_runtime_gpu_bytes, pin
assert pinned.drafter_runtime_bytes == unpinned.drafter_runtime_bytes, pin
def test_a_cpu_placed_target_takes_the_embedded_head_with_it(self, nextn_model):
"""An embedded head follows a CPU-placed target: at --gpu-layers 0 its cache belongs in host RAM."""
gguf, config = nextn_model
on_cpu = ri._gguf_memory_breakdown(
config, gguf, n_ctx = 131072, gpu_memory_mode = "manual", gpu_layers = 0
)
assert on_cpu is not None
assert on_cpu.drafter_runtime_bytes > 0
assert on_cpu.drafter_runtime_gpu_bytes == 0
class TestACpuOnlyHostShowsNoGpuFootprint:
"""CPU placement needs a probe that ran; an unfilled snapshot or a zero CUDA count is not evidence."""
@pytest.fixture
def priced(self, tmp_path):
gguf = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 262144})
return gguf, SimpleNamespace(
identifier = "local/cpu",
gguf_file = gguf,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def test_a_probed_empty_inventory_puts_everything_in_host_ram(self, priced, monkeypatch):
gguf, config = priced
monkeypatch.setattr(ri, "_cached_inference_devices", lambda: [])
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 32768)
assert out is not None
assert out.gpu_bytes == 0
assert out.total_bytes > 0
assert out.kv_on_gpu is False
def test_an_unprobed_snapshot_is_not_evidence_of_absence(self, priced, monkeypatch):
gguf, config = priced
monkeypatch.setattr(ri, "_cached_inference_devices", lambda: None)
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 32768)
assert out is not None
assert out.gpu_bytes > 0
def test_a_probed_inventory_with_a_card_is_unaffected(self, priced, monkeypatch):
gguf, config = priced
monkeypatch.setattr(ri, "_cached_inference_devices", lambda: [(0, 0, 0)])
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 32768)
assert out is not None
assert out.gpu_bytes > 0
class TestAnInheritedContextIsPriced:
"""A positive inherited LLAMA_ARG_CTX_SIZE is the child's length, so the estimate must read it."""
@pytest.fixture
def wide(self, tmp_path):
gguf = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 262144})
return gguf, SimpleNamespace(
identifier = "local/wide",
gguf_file = gguf,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def test_the_environment_length_is_priced_when_nothing_else_sets_one(self, wide, monkeypatch):
gguf, config = wide
monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "4096")
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 0)
assert out is not None
assert out.n_ctx == 4096
def test_the_native_context_is_still_the_last_resort(self, wide, monkeypatch):
gguf, config = wide
monkeypatch.delenv("LLAMA_ARG_CTX_SIZE", raising = False)
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 0)
assert out is not None
assert out.n_ctx == 262144
@pytest.mark.parametrize("value", ["0", "", "not-a-number", "-5"])
def test_a_useless_environment_value_falls_through(self, wide, monkeypatch, value):
"""Zero is the one the launch itself drops, and the rest are not lengths."""
gguf, config = wide
monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", value)
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 0)
assert out is not None
assert out.n_ctx == 262144
@pytest.mark.parametrize("flag", [["-c", "0"], ["--ctx-size", "0"], ["--ctx-size=0"]])
def test_an_explicit_zero_asks_for_native_and_beats_the_environment(
self, wide, monkeypatch, flag
):
"""An explicit -c 0 requests native context and beats an inherited LLAMA_ARG_CTX_SIZE."""
gguf, config = wide
monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "4096")
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 0, llama_extra_args = flag)
assert out is not None
assert out.n_ctx == 262144, flag
def test_without_that_zero_the_environment_still_answers(self, wide, monkeypatch):
"""The guard against over-correcting: only an explicit zero bypasses it."""
gguf, config = wide
monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "4096")
assert ri._gguf_memory_breakdown(config, gguf, n_ctx = 0).n_ctx == 4096
def test_an_explicit_panel_length_still_wins(self, wide, monkeypatch):
gguf, config = wide
monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "4096")
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 8192)
assert out is not None
assert out.n_ctx == 8192
class TestAnAutoContextIsMarkedShrinkable:
"""Only contexts the loader can shrink should use a floor for memory warnings."""
@pytest.fixture(autouse = True)
def wide(self, tmp_path, monkeypatch):
import utils.hardware as hardware
# The layer-split floor (8192). Metal reprices down to 256 instead; that path is
# TestTheContextFloorFollowsThePlacementPath's, and on a Mac runner it would be taken here.
monkeypatch.setattr(hardware, "is_apple_silicon", lambda: False)
gguf = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 262144})
config = SimpleNamespace(
identifier = "local/wide",
gguf_file = gguf,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: config)
monkeypatch.setattr(ri, "_cached_inference_devices", lambda: [(0, 0, 0)])
monkeypatch.delenv("LLAMA_ARG_CTX_SIZE", raising = False)
monkeypatch.delenv("LLAMA_ARG_FIT_CTX", raising = False)
self.fit_ctx_supported(monkeypatch, True)
return gguf
@staticmethod
def fit_ctx_supported(monkeypatch, supported):
from core.inference.llama_cpp import LlamaCppBackend
real = LlamaCppBackend.probe_server_capabilities
monkeypatch.setattr(
LlamaCppBackend,
"probe_server_capabilities",
classmethod(lambda cls, binary = None: {**real(binary), "supports_fit_ctx": supported}),
)
@pytest.mark.parametrize(
("request_fields", "fit_ctx_supported", "floor_ctx"),
[
({}, True, 8192),
({"gpu_memory_mode": "auto"}, True, 8192),
({"gpu_memory_mode": "manual"}, True, 8192),
({"gpu_memory_mode": "manual"}, False, 4096),
({"gpu_memory_mode": "manual", "llama_extra_args": ["-fitc", "16384"]}, True, 16384),
({"gpu_memory_mode": "manual", "llama_extra_args": ["--fit", "off"]}, True, 8192),
],
)
def test_an_unnamed_context_is_priced_at_the_fit_floor(
self, wide, monkeypatch, request_fields, fit_ctx_supported, floor_ctx
):
"""Use the loader's fit floor, not the smallest possible context."""
self.fit_ctx_supported(monkeypatch, fit_ctx_supported)
resp = _estimate(model_path = wide, **request_fields)
assert resp.n_ctx == 262144
assert resp.context_is_pinned is False
config = ri._cached_estimate_config()
floor = ri._gguf_memory_breakdown(config, wide, n_ctx = floor_ctx, **request_fields)
shorter = ri._gguf_memory_breakdown(config, wide, n_ctx = 256, **request_fields)
assert shorter.gpu_bytes < resp.gpu_floor_bytes == floor.gpu_bytes < resp.gpu_bytes
def test_a_native_context_under_the_fit_floor_has_nothing_to_shrink(
self, tmp_path, monkeypatch
):
gguf = _write_gguf(
tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 4096}, name = "short.gguf"
)
config = SimpleNamespace(**{**vars(ri._cached_estimate_config()), "gguf_file": gguf})
monkeypatch.setattr(ri, "_cached_estimate_config", lambda *a, **kw: config)
resp = _estimate(model_path = gguf)
assert resp.context_is_pinned is False
assert resp.gpu_floor_bytes == resp.gpu_bytes
@pytest.mark.parametrize(
("request_fields", "env_ctx", "fit_ctx_supported"),
[
({"n_ctx": 8192}, None, True),
({"llama_extra_args": ["-c", "0"]}, None, True),
({"llama_extra_args": ["--ctx-size=4096"]}, None, True),
({"llama_extra_args": ["-nkvo"]}, None, True),
({"gpu_memory_mode": "manual", "gpu_layers": 0}, None, True),
({"gpu_memory_mode": "manual", "gpu_layers": 99}, None, True),
({"gpu_memory_mode": "manual", "llama_extra_args": ["-ngl", "99"]}, None, True),
({}, "4096", True),
({"gpu_memory_mode": "manual"}, "0", False),
# llama.cpp stores --fit-ctx unsigned, so -1 disables the reduction.
({"gpu_memory_mode": "manual", "llama_extra_args": ["--fit-ctx", "-1"]}, None, True),
],
)
def test_a_context_the_launch_opens_verbatim_stays_pinned(
self, wide, monkeypatch, request_fields, env_ctx, fit_ctx_supported
):
self.fit_ctx_supported(monkeypatch, fit_ctx_supported)
if env_ctx is not None:
monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", env_ctx)
resp = _estimate(model_path = wide, **request_fields)
assert resp.available is True
assert resp.context_is_pinned is True, (request_fields, env_ctx)
assert resp.gpu_floor_bytes is None
@pytest.mark.parametrize(("fit_ctx_supported", "floor_ctx"), [(True, 8192), (False, 4096)])
def test_manual_ignores_the_fit_variables_its_launch_clears(
self, wide, monkeypatch, fit_ctx_supported, floor_ctx
):
monkeypatch.setenv("LLAMA_ARG_FIT_CTX", "16384")
monkeypatch.setenv("LLAMA_ARG_N_GPU_LAYERS", "20")
self.fit_ctx_supported(monkeypatch, fit_ctx_supported)
manual = {"gpu_memory_mode": "manual"}
floor = ri._gguf_memory_breakdown(
ri._cached_estimate_config(), wide, n_ctx = floor_ctx, **manual
)
resp = _estimate(model_path = wide, **manual)
assert resp.gpu_floor_bytes == floor.gpu_bytes
assert resp.floor_can_offload is True
@pytest.mark.parametrize("request_fields", [{}, {"gpu_memory_mode": "manual"}])
def test_an_inherited_zero_is_overruled_by_the_launch(self, wide, monkeypatch, request_fields):
monkeypatch.setenv("LLAMA_ARG_CTX_SIZE", "0")
assert _estimate(model_path = wide, **request_fields).context_is_pinned is False
@pytest.mark.parametrize(
("request_fields", "env", "can_offload"),
[
({}, {}, True),
({"gpu_memory_mode": "manual"}, {}, True),
({"llama_extra_args": ["--fit", "off"]}, {}, False),
({"llama_extra_args": ["-ngl", "99"]}, {}, False),
({}, {"LLAMA_ARG_N_GPU_LAYERS": "20"}, False),
({"llama_extra_args": ["-ngl", "-1"]}, {}, True),
({"llama_extra_args": ["-ngl", "99", "-ngl", "-1"]}, {}, True),
({"llama_extra_args": ["-ngl", "-1"]}, {"LLAMA_ARG_N_GPU_LAYERS": "20"}, True),
({"llama_extra_args": ["-ngl", "auto"]}, {}, True),
({"llama_extra_args": ["-ngl", "all"]}, {}, False),
],
)
def test_the_floor_says_whether_layers_can_move_past_it(
self, wide, monkeypatch, request_fields, env, can_offload
):
for key, value in env.items():
monkeypatch.setenv(key, value)
resp = _estimate(model_path = wide, **request_fields)
assert resp.context_is_pinned is False
assert resp.floor_can_offload is can_offload, (request_fields, env)
@pytest.mark.parametrize(
"request_fields",
[
{"llama_extra_args": ["--fit"]},
{"gpu_memory_mode": "manual", "llama_extra_args": ["--fit-ctx"]},
{"gpu_memory_mode": "manual", "llama_extra_args": ["-fitc="]},
],
)
def test_a_half_typed_fit_flag_still_prices(self, wide, request_fields):
resp = _estimate(model_path = wide, **request_fields)
assert resp.available is True
assert resp.context_is_pinned is True
assert resp.gpu_floor_bytes is None
def test_a_zero_fit_ctx_can_land_on_the_native_length(self, wide):
resp = _estimate(model_path = wide, gpu_memory_mode = "manual", llama_extra_args = ["-fitc", "0"])
assert resp.context_is_pinned is True
@pytest.mark.parametrize("selected_gpu_ids", [None, [0, 1]])
def test_tensor_mode_takes_the_layer_fallback_floor(self, wide, monkeypatch, selected_gpu_ids):
"""Tensor mode falls back to a layer split; floor priced as that split when larger."""
devices = [(0, 0, 0), (1, 0, 0)]
monkeypatch.setattr(ri, "_cached_inference_devices", lambda: devices)
monkeypatch.setattr(ri, "_tensor_split_possible", lambda ids: True)
config = ri._cached_estimate_config()
pinned = bool(selected_gpu_ids)
cards = ri._guard_device_count(selected_gpu_ids, devices, tensor_parallel = True)
assert cards == 2
common = dict(
n_ctx = 8192, n_devices = cards, tensor_split_possible = True, device_pin_governs = pinned
)
as_tensor = ri._gguf_memory_breakdown(config, wide, tensor_parallel = True, **common)
as_layer = ri._gguf_memory_breakdown(
config,
wide,
tensor_parallel = False,
llama_extra_args = ["--split-mode", "layer"],
**common,
)
assert as_tensor.gpu_bytes < as_layer.gpu_bytes
resp = _estimate(model_path = wide, tensor_parallel = True, selected_gpu_ids = selected_gpu_ids)
assert resp.context_is_pinned is False
assert resp.gpu_floor_bytes == as_layer.gpu_bytes
class TestTheContextFloorFollowsThePlacementPath:
"""Auto placement stops shrinking at a floor that depends on the path the load takes."""
@staticmethod
def floor(monkeypatch, *, apple = False):
import utils.hardware as hardware
monkeypatch.setattr(hardware, "is_apple_silicon", lambda: apple)
monkeypatch.delenv("LLAMA_ARG_CTX_SIZE", raising = False)
monkeypatch.delenv("LLAMA_ARG_N_GPU_LAYERS", raising = False)
breakdown = SimpleNamespace(kv_on_gpu = True, kv_estimable = True)
return ri._estimate_context_floor(None, [], "auto", None, breakdown)
def test_a_layer_split_stops_at_the_fit_floor(self, monkeypatch):
from core.inference.llama_cpp import _FIT_MIN_CTX
assert self.floor(monkeypatch).ctx == _FIT_MIN_CTX == 8192
def test_metal_reprices_down_to_the_smallest_context_it_searches(self, monkeypatch):
from core.inference.llama_cpp import _FIT_FLOOR_MIN_CTX
assert self.floor(monkeypatch, apple = True).ctx == _FIT_FLOOR_MIN_CTX == 256
class TestTheResolutionTriesOfflineFirst:
"""A repo already on disk must be priced offline, so resolution tries forced-offline first."""
def test_the_first_attempt_runs_under_forced_offline(self, tmp_path, monkeypatch):
gguf = _write_gguf(tmp_path, "qwen3", _GQA_FIELDS)
seen = []
import contextlib
@contextlib.contextmanager
def _forced():
seen.append("offline")
yield True
monkeypatch.setattr(ri, "_estimate_target_is_on_this_disk", lambda ident: True)
monkeypatch.setitem(
__import__("sys").modules,
"utils.utils",
SimpleNamespace(force_hf_offline = _forced),
)
cfg = SimpleNamespace(identifier = "org/cached", gguf_file = gguf, is_gguf = True)
monkeypatch.setattr(ri.ModelConfig, "from_identifier", staticmethod(lambda **kw: cfg))
ri._estimate_config_cache.clear()
out = ri._cached_estimate_config("org/cached", None, None, False)
assert out is cfg
assert seen == ["offline"], "the offline window was not entered"
def test_an_offline_failure_falls_back_rather_than_blanking_the_row(
self, tmp_path, monkeypatch
):
"""A slow right answer beats a fast blank row."""
gguf = _write_gguf(tmp_path, "qwen3", _GQA_FIELDS)
calls = []
cfg = SimpleNamespace(identifier = "org/cached", gguf_file = gguf, is_gguf = True)
def _from_identifier(**kw):
calls.append(1)
if len(calls) == 1:
raise RuntimeError("needs the hub")
return cfg
import contextlib
@contextlib.contextmanager
def _forced():
yield True
monkeypatch.setattr(ri, "_estimate_target_is_on_this_disk", lambda ident: True)
monkeypatch.setitem(
__import__("sys").modules,
"utils.utils",
SimpleNamespace(force_hf_offline = _forced),
)
monkeypatch.setattr(ri.ModelConfig, "from_identifier", staticmethod(_from_identifier))
ri._estimate_config_cache.clear()
out = ri._cached_estimate_config("org/cached", None, None, False)
assert out is cfg
assert len(calls) == 2, "the online retry did not run"
class TestTheEmbeddedHeadIsChargedOnlyWhenItEngages:
"""An embedded head has no file, so an over-charge is invisible; no-MTP launches must allocate none."""
def _config(self, gguf, identifier):
return SimpleNamespace(
identifier = identifier,
gguf_file = gguf,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
@pytest.fixture
def head(self, tmp_path):
return _write_gguf(
tmp_path,
"qwen3",
{**_GQA_FIELDS, "context_length": 262144, "nextn_predict_layers": 2},
name = "head.gguf",
)
def test_a_sub_3b_head_is_dropped_by_auto(self, head):
"""Auto falls back to ngram-mod there, because MTP regresses below 3B."""
big = ri._gguf_memory_breakdown(self._config(head, "org/Qwen3-8B"), head, n_ctx = 131072)
small = ri._gguf_memory_breakdown(self._config(head, "org/Qwen3-1.7B"), head, n_ctx = 131072)
assert big.drafter_runtime_bytes > 0
assert small.drafter_runtime_bytes == 0
def test_forcing_mtp_overrides_the_size_drop(self, head):
"""The drop is Auto's judgement; an explicit request is not overruled."""
forced = ri._gguf_memory_breakdown(
self._config(head, "org/Qwen3-1.7B"), head, n_ctx = 131072, speculative_type = "mtp"
)
assert forced.drafter_runtime_bytes > 0
@pytest.mark.parametrize("mode", ["dspark", "dflash"])
def test_a_sidecar_mode_never_charges_the_embedded_head(self, head, mode):
"""Those either open their own sidecar, charged as a file, or --spec-default."""
out = ri._gguf_memory_breakdown(
self._config(head, "org/Qwen3-8B"), head, n_ctx = 131072, speculative_type = mode
)
assert out.drafter_runtime_bytes == 0
@pytest.mark.parametrize("mode", [None, "mtp"])
def test_a_binary_that_cannot_run_mtp_is_not_charged_for_it(self, head, monkeypatch, mode):
"""No mtp_token means a --spec-default launch, whatever the mode asked, even when forced."""
monkeypatch.setattr(
ri.LlamaCppBackend,
"probe_server_capabilities",
classmethod(lambda cls, *a, **k: {"found": True, "mtp_token": None}),
)
out = ri._gguf_memory_breakdown(
self._config(head, "org/Qwen3-8B"), head, n_ctx = 131072, speculative_type = mode
)
assert out.drafter_runtime_bytes == 0
def test_an_unanswerable_probe_keeps_the_charge(self, head, monkeypatch):
"""A probe that could not run is not evidence the build lacks the flag."""
monkeypatch.setattr(
ri.LlamaCppBackend,
"probe_server_capabilities",
classmethod(lambda cls, *a, **k: {"found": False, "mtp_token": None}),
)
out = ri._gguf_memory_breakdown(self._config(head, "org/Qwen3-8B"), head, n_ctx = 131072)
assert out.drafter_runtime_bytes > 0
def test_extras_asking_for_another_mode_are_left_alone(self, head):
"""Studio emits no spec block of its own once --spec-type is in the extras."""
out = ri._gguf_memory_breakdown(
self._config(head, "org/Qwen3-8B"),
head,
n_ctx = 131072,
llama_extra_args = ["--spec-type", "ngram-mod"],
)
assert out.drafter_runtime_bytes == 0
@pytest.mark.parametrize("spec", ["draft-mtp", "mtp"])
def test_extras_asking_for_mtp_engage_the_embedded_head(self, head, spec):
"""An MTP extra engages the embedded NextN head, whose cache must count in the total."""
out = ri._gguf_memory_breakdown(
self._config(head, "org/Qwen3-8B"),
head,
n_ctx = 131072,
llama_extra_args = ["--spec-type", spec],
)
assert out.drafter_runtime_bytes > 0
def test_extras_asking_for_mtp_on_a_sub_3b_model_still_engage_it(self, head):
"""An explicit request is not second-guessed by Auto's size rule."""
out = ri._gguf_memory_breakdown(
self._config(head, "org/Qwen3-1.7B"),
head,
n_ctx = 131072,
llama_extra_args = ["--spec-type", "draft-mtp"],
)
assert out.drafter_runtime_bytes > 0
class TestTheCpuOnlyCheckIsActuallyReachable:
"""Uses the real snapshot: a trailing or None made an empty CPU-only probe look unfilled."""
@pytest.fixture
def snapshot(self, monkeypatch):
import sys as _sys
def _set(devices):
main = SimpleNamespace(_system_gpu_cache = (0.0, (None, {"devices": devices})))
monkeypatch.setitem(_sys.modules, "main", main)
return _set
def test_a_probed_cpu_only_host_reads_as_empty_not_unknown(self, snapshot):
snapshot([])
assert ri._cached_inference_devices() == []
assert ri._estimate_host_has_no_gpu() is True
def test_a_probed_host_with_cards_reads_as_those_cards(self, snapshot):
snapshot([{"index": 0}, {"index": 1}])
assert len(ri._cached_inference_devices()) == 2
assert ri._estimate_host_has_no_gpu() is False
def test_an_unfilled_snapshot_is_still_unknown(self, monkeypatch):
import sys as _sys
monkeypatch.setitem(_sys.modules, "main", SimpleNamespace())
assert ri._cached_inference_devices() is None
assert ri._estimate_host_has_no_gpu() is False
def test_a_cpu_only_host_cannot_take_a_tensor_split(self, snapshot, monkeypatch):
"""The other caller of the same snapshot, which was also reading True here."""
snapshot([])
monkeypatch.setattr(ri.LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda *a: True))
assert ri._tensor_split_possible(None) is False
def test_the_whole_footprint_lands_in_host_ram_end_to_end(self, snapshot, tmp_path):
gguf = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 262144})
config = SimpleNamespace(
identifier = "local/cpu",
gguf_file = gguf,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
snapshot([])
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 32768)
assert out is not None
assert out.gpu_bytes == 0
assert out.total_bytes > 0
class TestDraftCacheTypePrecedence:
"""Launch order decides: extras beat the panel field, which beats the inherited environment."""
@pytest.fixture
def spec(self, tmp_path, monkeypatch):
target = _write_gguf(
tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 262144}, name = "t.gguf"
)
drafter = tmp_path / "mtp.gguf"
drafter.write_bytes(Path(target).read_bytes())
drafter_bytes = drafter.stat().st_size
# Must vary with the draft pin or every drafter assertion compares 0 with 0.
def _files(
cfg,
*,
llama_extra_args = None,
**kw,
):
pinned = _draft_on_cpu(list(llama_extra_args or ()))
return 1.0 + (0.0 if pinned else drafter_bytes / 1024**3)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
return target, SimpleNamespace(
identifier = "org/Qwen3-8B",
gguf_file = target,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = str(drafter),
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def _bytes(
self,
spec,
monkeypatch,
*,
env = None,
field = None,
extras = None,
):
target, config = spec
for key in ("LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K", "LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V"):
monkeypatch.delenv(key, raising = False)
if env:
monkeypatch.setenv("LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K", env)
monkeypatch.setenv("LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V", env)
out = ri._gguf_memory_breakdown(
config,
target,
n_ctx = 131072,
spec_draft_cache_type = field,
llama_extra_args = extras,
)
assert out is not None
return out.drafter_runtime_bytes
def test_the_panel_field_beats_an_inherited_value(self, spec, monkeypatch):
inherited_only = self._bytes(spec, monkeypatch, env = "q4_0")
field_wins = self._bytes(spec, monkeypatch, env = "q4_0", field = "f16")
assert field_wins > inherited_only
def test_the_inherited_value_is_used_when_the_field_is_blank(self, spec, monkeypatch):
plain = self._bytes(spec, monkeypatch)
inherited = self._bytes(spec, monkeypatch, env = "q4_0")
assert inherited < plain
def test_the_extras_beat_both(self, spec, monkeypatch):
field_only = self._bytes(spec, monkeypatch, env = "q4_0", field = "f16")
extras_win = self._bytes(
spec,
monkeypatch,
env = "q4_0",
field = "f16",
extras = ["--cache-type-k-draft", "q4_0", "--cache-type-v-draft", "q4_0"],
)
assert extras_win < field_only
class TestTheTensorSplitLatchesAreHonoured:
"""load_model consults two in-process latches before planning, so the price must consult them too."""
@pytest.fixture
def two_card(self, tmp_path, monkeypatch):
gguf = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 262144})
config = SimpleNamespace(
identifier = "local/tensor",
gguf_file = gguf,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
monkeypatch.setattr(
ri.LlamaCppBackend,
"_find_llama_server_binary",
staticmethod(lambda **kw: "/opt/llama/llama-server"),
)
return gguf, config
def _priced(self, two_card, **kw):
gguf, config = two_card
return ri._gguf_memory_breakdown(
config,
gguf,
n_ctx = 32768,
tensor_parallel = True,
n_devices = 2,
tensor_split_possible = True,
**kw,
)
def test_a_binary_that_refused_a_quantized_tensor_cache_is_priced_as_layer(
self, two_card, monkeypatch
):
as_tensor = self._priced(two_card, cache_type_kv = "q4_0")
monkeypatch.setattr(
ri.LlamaCppBackend,
"_tensor_quant_kv_unsupported_binary",
classmethod(lambda cls, *a, **k: True),
)
downgraded = self._priced(two_card, cache_type_kv = "q4_0")
assert downgraded.compute_bytes != as_tensor.compute_bytes
def test_a_model_that_aborted_on_tensor_this_session_is_priced_as_layer(
self, two_card, monkeypatch
):
as_tensor = self._priced(two_card, cache_type_kv = "f16")
monkeypatch.setattr(
ri.LlamaCppBackend,
"_tensor_split_aborts",
classmethod(lambda cls, *a, **k: True),
)
downgraded = self._priced(two_card, cache_type_kv = "f16")
assert downgraded.compute_bytes != as_tensor.compute_bytes
def test_no_latch_leaves_tensor_pricing_alone(self, two_card, monkeypatch):
"""The guard against the check firing on every load."""
monkeypatch.setattr(
ri.LlamaCppBackend,
"_tensor_quant_kv_unsupported_binary",
classmethod(lambda cls, *a, **k: False),
)
monkeypatch.setattr(
ri.LlamaCppBackend,
"_tensor_split_aborts",
classmethod(lambda cls, *a, **k: False),
)
assert ri._tensor_latches_allow_a_split(two_card[0], two_card[1], "f16", None) is True
def test_an_unresolvable_binary_fails_open(self, two_card, monkeypatch):
monkeypatch.setattr(
ri.LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda **kw: None)
)
assert ri._tensor_latches_allow_a_split(two_card[0], two_card[1], "f16", None) is True
class TestInheritedRemoteFilesAreMarkedUnsized:
"""Inherited remote files cannot be sized without a network call, so report a marked lower bound."""
@pytest.fixture
def plain(self, tmp_path, monkeypatch):
for key in (
"LLAMA_ARG_SPEC_DRAFT_HF_REPO",
"LLAMA_ARG_HFD_REPO",
"LLAMA_ARG_MMPROJ_URL",
):
monkeypatch.delenv(key, raising = False)
gguf = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 262144})
return gguf, SimpleNamespace(
identifier = "local/plain",
gguf_file = gguf,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def test_nothing_inherited_is_not_marked(self, plain):
gguf, config = plain
assert ri._gguf_memory_breakdown(config, gguf, n_ctx = 8192).drafter_kv_unsized is False
@pytest.mark.parametrize("key", ["LLAMA_ARG_SPEC_DRAFT_HF_REPO", "LLAMA_ARG_HFD_REPO"])
def test_an_inherited_remote_drafter_repo_is_marked(self, plain, monkeypatch, key):
gguf, config = plain
monkeypatch.setenv(key, "org/drafter-GGUF")
out = ri._gguf_memory_breakdown(
config, gguf, n_ctx = 8192, llama_extra_args = ["--spec-type", "draft-mtp"]
)
assert out.drafter_kv_unsized is True
assert out.total_bytes > 0
def test_an_inherited_projector_url_is_marked(self, plain, monkeypatch):
gguf, config = plain
monkeypatch.setenv("LLAMA_ARG_MMPROJ_URL", "https://example.invalid/mmproj.gguf")
assert ri._gguf_memory_breakdown(config, gguf, n_ctx = 8192).drafter_kv_unsized is True
def test_a_projector_url_is_irrelevant_with_vision_off(self, plain, monkeypatch):
gguf, config = plain
monkeypatch.setenv("LLAMA_ARG_MMPROJ_URL", "https://example.invalid/mmproj.gguf")
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 8192, disable_vision = True)
assert out.drafter_kv_unsized is False
class TestFourMoreLaunchNormalizations:
"""Each is a rule load_model applies before it sizes anything."""
@pytest.fixture
def basic(self, tmp_path):
gguf = _write_gguf(tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 262144})
return gguf, SimpleNamespace(
identifier = "local/basic",
gguf_file = gguf,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = None,
gguf_dspark_file = None,
gguf_dflash_file = None,
)
def test_a_gpu_pin_beats_a_stale_device_flag(self, basic):
"""A GPU pin overrides a stale --device flag, which the launch strips as contradictory."""
gguf, config = basic
cpu_flag = ["--device", "none"]
unpinned = ri._gguf_memory_breakdown(config, gguf, n_ctx = 8192, llama_extra_args = cpu_flag)
pinned = ri._gguf_memory_breakdown(
config, gguf, n_ctx = 8192, llama_extra_args = cpu_flag, device_pin_governs = True
)
assert unpinned.gpu_bytes == 0
assert pinned.gpu_bytes > 0
def test_a_remote_drafter_repo_is_not_listed_over_the_network(self, basic, monkeypatch):
"""Sizing it means an hf_model_info per settings change; the marker replaces it."""
gguf, config = basic
called = []
monkeypatch.setattr(
ri,
"_remote_drafter_repo_bytes",
lambda *a, **k: (called.append(1), 1 << 30)[1],
)
out = ri._gguf_memory_breakdown(
config,
gguf,
n_ctx = 8192,
llama_extra_args = ["--spec-draft-hf", "org/drafter-GGUF"],
)
assert called == [], "the Hub listing ran behind the panel"
assert out.drafter_kv_unsized is True
@pytest.mark.parametrize(
"caps,managed",
[
(
{
"found": True,
"spec_draft_cache_k_flag": "-ctkd",
"spec_draft_cache_v_flag": "-ctvd",
},
True,
),
(
{
"found": True,
"spec_draft_cache_k_flag": "-ctkd",
"spec_draft_cache_v_flag": None,
},
False,
),
(
{
"found": True,
"spec_draft_cache_k_flag": None,
"spec_draft_cache_v_flag": "-ctvd",
},
False,
),
(
{"found": False, "spec_draft_cache_k_flag": None, "spec_draft_cache_v_flag": None},
True,
),
],
)
def test_the_draft_cache_field_needs_both_flags(self, tmp_path, monkeypatch, caps, managed):
"""The launcher emits the pair or neither, so one flag is not enough."""
target = _write_gguf(
tmp_path, "qwen3", {**_GQA_FIELDS, "context_length": 262144}, name = "t2.gguf"
)
drafter = tmp_path / "mtp2.gguf"
drafter.write_bytes(Path(target).read_bytes())
drafter_bytes = drafter.stat().st_size
def _files(
cfg,
*,
llama_extra_args = None,
**kw,
):
pinned = _draft_on_cpu(list(llama_extra_args or ()))
return 1.0 + (0.0 if pinned else drafter_bytes / 1024**3)
monkeypatch.setattr(ri, "_gguf_resident_file_gb", _files)
monkeypatch.setattr(
ri.LlamaCppBackend,
"probe_server_capabilities",
classmethod(lambda cls, *a, **k: {**caps, "mtp_token": "draft-mtp"}),
)
config = SimpleNamespace(
identifier = "org/Qwen3-8B",
gguf_file = target,
is_gguf = True,
gguf_variant = None,
gguf_mmproj_file = None,
gguf_mtp_file = str(drafter),
gguf_dspark_file = None,
gguf_dflash_file = None,
)
priced = ri._gguf_memory_breakdown(
config, target, n_ctx = 131072, spec_draft_cache_type = "q4_0"
)
default = ri._gguf_memory_breakdown(config, target, n_ctx = 131072)
if managed:
assert priced.drafter_runtime_bytes < default.drafter_runtime_bytes
else:
assert priced.drafter_runtime_bytes == default.drafter_runtime_bytes
def test_slots_are_clamped_on_a_build_without_kv_unified(self, basic, monkeypatch):
"""load_model drops to one slot there, before it sizes anything."""
gguf, config = basic
monkeypatch.setattr(
ri.LlamaCppBackend,
"probe_server_capabilities",
classmethod(lambda cls, *a, **k: {"found": True, "supports_kv_unified": False}),
)
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 8192, n_parallel = 4)
assert out.n_parallel == 1
def test_an_unprobed_build_keeps_the_requested_slots(self, basic, monkeypatch):
"""A probe that could not run is not evidence the flag is missing."""
gguf, config = basic
monkeypatch.setattr(
ri.LlamaCppBackend,
"probe_server_capabilities",
classmethod(lambda cls, *a, **k: {"found": False, "supports_kv_unified": False}),
)
out = ri._gguf_memory_breakdown(config, gguf, n_ctx = 8192, n_parallel = 4)
assert out.n_parallel == 4
class TestMlxEstimateKvBits:
"""Which requested KV widths reach the MLX planner, and which are dropped."""
@pytest.mark.parametrize(
"quant, bits, expected",
[
(None, 4, 4),
(None, 8, 8),
(None, 7, None),
(None, 0, None),
(None, None, None),
(None, True, None),
("4", None, 4),
(" 8 ", 4, 8),
("auto", 8, None),
("tq-4", 4, ("turboquant", 4)),
("tq-3.5", None, ("turboquant", 3.5)),
("bogus", 4, None),
],
)
def test_kv_bits_resolution(self, quant, bits, expected):
if isinstance(expected, tuple):
expected = mm.MlxTurboQuant(expected[1])
assert ri._mlx_estimate_kv_bits(quant, bits) == expected
import glob # noqa: E402
from dataclasses import replace # noqa: E402
from core.inference import mlx_memory as mm # noqa: E402
@pytest.fixture(autouse = True)
def _fresh_sizing_cache():
from core.inference import mlx_inference
mm._clear_sizing_cache()
mlx_inference._kv_refusal_cache.clear()
yield
mm._clear_sizing_cache()
mlx_inference._kv_refusal_cache.clear()
def test_a_load_is_sized_once_until_its_files_change(monkeypatch, tmp_path):
(tmp_path / "config.json").write_text("{}")
shard = tmp_path / "model.safetensors"
shard.write_bytes(b"x")
built = []
monkeypatch.setattr(
mm, "_size_load_uncached", lambda *a: built.append(a) or f"sizing{len(built)}"
)
first = mm._size_load(str(tmp_path), 4, None, True)
assert mm._size_load(str(tmp_path), 4, None, True) is first and len(built) == 1
assert mm._size_load(str(tmp_path), None, None, True) != first and len(built) == 2
shard.write_bytes(b"xy")
assert mm._size_load(str(tmp_path), 4, None, True) != first and len(built) == 3
(drafter := tmp_path / "drafter").mkdir()
sized = lambda: mm._size_load(str(tmp_path), 4, None, True, True, (str(drafter), False))
first = sized()
(drafter / "config.json").write_text("{}")
assert sized() != first # a changed local drafter is re-sized too
offloaded = lambda fraction: mm._size_load(
str(tmp_path), 4, None, True, None, None, (None, fraction)
)
assert offloaded(0.5) is offloaded(0.5) and offloaded(0.5) != offloaded(0.75)
def test_offload_is_priced_by_what_unsloth_zoos_offload_takes(tmp_path, monkeypatch):
offload = pytest.importorskip("unsloth_zoo.mlx.offload")
if not _HAVE_MLX or not hasattr(offload, "plan_offload"):
pytest.skip("needs MLX and an unsloth_zoo that offloads")
mm.mlx_offload_planner.cache_clear()
planner = mm.mlx_offload_planner()
assert planner is offload
import mlx.core as mx
import mlx.nn as nn
from mlx.utils import tree_flatten
from mlx_lm.utils import _get_classes
config = {
"model_type": "qwen3_moe",
"hidden_size": 64,
"num_hidden_layers": 1,
"intermediate_size": 64,
"moe_intermediate_size": 64,
"num_attention_heads": 2,
"num_key_value_heads": 1,
"head_dim": 32,
"vocab_size": 64,
"num_experts": 16,
"num_experts_per_tok": 2,
"decoder_sparse_step": 1,
"mlp_only_layers": [],
"rms_norm_eps": 1e-6,
"rope_theta": 10000.0,
"tie_word_embeddings": True,
"max_position_embeddings": 512,
"norm_topk_prob": True,
}
model_class, args_class = _get_classes(config)
model = model_class(args_class.from_dict(config))
nn.quantize(
model,
group_size = 32,
bits = 4,
class_predicate = lambda path, m: hasattr(m, "to_quantized") and "switch_mlp" in path,
)
weights = dict(tree_flatten(model.parameters()))
# A tensor the model does not instantiate, as Gemma 4 checkpoints carry and unsloth_zoo loads past.
mx.save_safetensors(
str(tmp_path / "model.safetensors"), {**weights, "model.unused.scales": mx.zeros(4)}
)
config["quantization"] = {"group_size": 32, "bits": 4}
(tmp_path / "config.json").write_text(json.dumps(config))
experts = sum(v.nbytes for k, v in weights.items() if ".switch_mlp." in k)
found = offload.offloadable(mm._lazy_checkpoint(str(tmp_path), config), str(tmp_path))
assert (found["expert_bytes"], found["experts"]) == (experts, None)
assert mm.mlx_offloadable_parts(str(tmp_path)) == ("experts",)
for fractions in ((None, 0.5), (1.0, 1.0)): # no PLE table: its knob applies nothing
applied = (None, fractions[1])
delta = planner.offload_footprint(found, *applied) - experts
assert mm._offloaded(str(tmp_path), config, fractions) == (delta, applied)
# Auto: the 50/50 rule on this load's own pricing; when nothing fits, all of it goes to disk.
plan = lambda **kw: mm.mlx_offload_plan(str(tmp_path), max_ctx = 512, **kw)
asked, decide = {}, planner.plan_offload
monkeypatch.setattr(planner, "plan_offload", lambda **kw: asked.update(kw) or decide(**kw))
assert plan(budget_bytes = 1 << 40) == (None, None, None)
priced = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 512).total_bytes
assert asked["resident"] + experts + asked["context_cost"](512, None) == priced
wide = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 512, prefill_chunk = 8192).total_bytes
assert wide > priced and asked["resident"] + experts + asked["context_cost"](512, 8192) == wide
assert plan(budget_bytes = 1) == (None, 1.0, None)
assert (
plan(budget_bytes = 1 << 40, experts = 0.5) == (None, 0.5, 2048)
and asked["prefill_step"] is None
)
assert (
plan(budget_bytes = 1 << 40, experts = None) == (None, None, None)
and asked["prefill_step"] == 2048
)
mm.mlx_offload_planner.cache_clear()
with monkeypatch.context() as patch: # an unsloth_zoo that would not hand "auto" to Studio
patch.setattr(sys.modules["unsloth_zoo.mlx.loader"].FastMLXModel, "from_pretrained", print)
assert mm.mlx_offload_planner() is None
mm.mlx_offload_planner.cache_clear()
mm._clear_sizing_cache()
seen = []
kept = {
**found,
"expert_bytes": 0,
"expert_floor_bytes": 0,
"experts": "its routing top-k is unknown",
}
monkeypatch.setattr(
offload, "offloadable", lambda model, d, cast_dtype: seen.append(cast_dtype) or kept
)
assert mm._offloaded(str(tmp_path), config, (None, 1.0)) == (0, (None, None)) and seen == [None]
mm._offloaded(str(tmp_path), {**config, "quantization": None}, (None, 1.0))
assert seen[-1] == mm._runtime_dtype()
monkeypatch.setattr(mm, "mlx_offload_planner", lambda: None)
assert mm._offloaded(str(tmp_path), config, (1.0, 1.0)) == (0, (None, None))
assert plan(budget_bytes = 1) is None
(tmp_path / "own.py").write_text(f"open({str(tmp_path / 'ran')!r}, 'w')")
(tmp_path / "config.json").write_text(json.dumps({**config, "model_file": "own.py"}))
monkeypatch.setattr(mm, "_loads_as_vision", lambda config: True)
with pytest.raises(ValueError):
mm._lazy_checkpoint(str(tmp_path), mm._snapshot_config(str(tmp_path)))
assert not (tmp_path / "ran").exists()
def test_auto_offload_is_planned_on_the_route_the_load_takes(monkeypatch):
routes, costs = [], []
decide = lambda inventory, context_cost, **kw: {
"context": costs.append(context_cost(512, 8192)) or costs[-1] and 512,
"ple": None if inventory["ple"] else 1.0,
"experts": 0.5,
}
# A drafter's weights and cache are priced with the context, so a plan making room for one sees them there.
monkeypatch.setattr(
mm,
"_priced_at",
lambda sizing, tokens: SimpleNamespace(total_bytes = 11 + 5 * bool(sizing.drafter)),
)
monkeypatch.setattr(mm, "mlx_offload_planner", lambda: SimpleNamespace(plan_offload = decide))
monkeypatch.setattr(
mm, "_snapshot_config", lambda d: {"model_type": "qwen3_moe", "quantization": {"bits": 4}}
)
monkeypatch.setattr(mm, "_checkpoint_fingerprint", lambda d: ())
drafters = []
monkeypatch.setattr(
mm,
"_size_load",
lambda d, kv, chunk, four, vision = None, drafter = None: routes.append(("sized", vision))
or drafters.append(drafter)
or (
None
if drafter == ("/unsizable", False)
else SimpleNamespace(weights = 10, chunk = 512, drafter = drafter)
),
)
monkeypatch.setattr(mm, "_lazy_checkpoint", lambda d, config: mm._builds_on_mlx_vlm(config))
found = lambda tree: {
"ple_bytes": 0,
"expert_bytes": 4,
"ple": None if tree else "no",
"experts": None,
}
offload = SimpleNamespace(
offloadable = lambda tree, d, cast_dtype: routes.append(("inventory", tree)) or found(tree)
)
monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.offload", offload)
mm._clear_sizing_cache()
for vision in (None, True): # a speculative text load goes through mlx-vlm
planned = mm.mlx_offload_plan("/x", budget_bytes = 1, max_ctx = 512, vision = vision)
assert planned == ((1.0, 0.5, None) if vision else (None, 0.5, None))
parts = mm.mlx_offloadable_parts("/x", vision)
assert parts == (("ple", "experts") if vision else ("experts",))
assert routes == [("sized", None), ("inventory", False), ("sized", None)] + [
("sized", True),
("inventory", True),
("sized", True),
]
drafters.clear(), costs.clear()
mm._clear_sizing_cache()
mm.mlx_offload_plan("/x", budget_bytes = 1, max_ctx = 512, vision = True, drafter = ("/x", True))
assert drafters == [("/x", True)] * 2 and costs == [6] # the target alone costs 1
# One that cannot be sized is skipped by the load too: the target is still planned, alone.
drafters.clear(), costs.clear()
mm._clear_sizing_cache()
unsizable = ("/unsizable", False)
planned = mm.mlx_offload_plan("/x", budget_bytes = 1, max_ctx = 512, vision = True, drafter = unsizable)
assert planned == (1.0, 0.5, None) and drafters == [unsizable, None, None] and costs == [1]
def test_the_offload_probe_maps_a_text_model_on_the_route_unsloth_zoo_builds_it(monkeypatch):
if not _HAVE_MLX:
pytest.skip("resolves the loaders")
import mlx_lm.utils
import mlx_vlm.utils
monkeypatch.setattr(
mm, "_builds_on_mlx_vlm", lambda config: config["model_type"] == "deepseek_v4"
)
monkeypatch.setattr(mlx_vlm.utils, "load_model", lambda path, **kw: "mlx-vlm")
monkeypatch.setattr(mlx_lm.utils, "load_model", lambda path, **kw: ("mlx-lm", None))
assert mm._lazy_checkpoint("/x", {"model_type": "deepseek_v4"}) == "mlx-vlm"
assert mm._lazy_checkpoint("/x", {"model_type": "llama"}) == "mlx-lm"
def test_the_offload_probe_never_evaluates_a_converted_checkpoint(tmp_path, monkeypatch):
if not _HAVE_MLX:
pytest.skip("needs MLX to build and load a checkpoint")
checkpoint = pytest.importorskip("mlx_vlm.quantization.checkpoint")
if not hasattr(checkpoint, "transform_quantized_weights"):
pytest.skip("this mlx-vlm converts no ModelOpt checkpoint")
import mlx.core as mx
from mlx.utils import tree_flatten
from mlx_lm.models import qwen3
config = {
"model_type": "qwen3",
"hidden_size": 128,
"intermediate_size": 256,
"num_hidden_layers": 1,
"num_attention_heads": 2,
"num_key_value_heads": 1,
"head_dim": 64,
"vocab_size": 64,
"rms_norm_eps": 1e-6,
"rope_theta": 1e6,
"max_position_embeddings": 512,
"tie_word_embeddings": True,
# mlx-vlm decodes ModelOpt's lone-scale FP8 linears to dense weights, evaluating them.
"quantization_config": {"quant_method": "modelopt", "quant_algo": "MIXED_PRECISION"},
}
(tmp_path / "config.json").write_text(json.dumps(config))
weights = {}
for name, value in tree_flatten(qwen3.Model(qwen3.ModelArgs.from_dict(config)).parameters()):
if name.endswith("proj.weight"):
weights[name] = mx.zeros(value.shape, dtype = mx.uint8)
weights[name[: -len("weight")] + "weight_scale"] = mx.array(0.5)
else:
weights[name] = value
mx.save_safetensors(str(tmp_path / "model.safetensors"), weights)
evaluated = []
real_eval = mx.eval
monkeypatch.setattr(mx, "eval", lambda *a, **k: evaluated.append(1) or real_eval(*a, **k))
with mm._on_route(True):
mm._lazy_checkpoint(str(tmp_path), json.loads((tmp_path / "config.json").read_text()))
assert evaluated == []
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"lm_head.weight": "../outside.safetensors"}})
)
with pytest.raises(ValueError, match = "outside the snapshot"):
mm._lazy_checkpoint(str(tmp_path), config)
def test_a_deepseek_v41_oq_quant_is_priced_as_unsloth_zoo_loads_it(tmp_path, monkeypatch):
if not _HAVE_MLX:
pytest.skip("needs MLX to build and load a checkpoint")
v41 = pytest.importorskip("mlx_vlm.models.deepseek_v41.deepseek_v41")
loader = pytest.importorskip("unsloth_zoo.mlx.loader")
if not hasattr(loader, "_ensure_deepseek_v41_omlx_layout"):
pytest.skip("this unsloth_zoo loads no oQ quant")
import gc
import re
import mlx.core as mx
import mlx.nn as nn
from mlx.utils import tree_flatten
from mlx_vlm.models.deepseek_v41.config import ModelConfig
text = dict(
vocab_size = 512,
hidden_size = 256,
moe_intermediate_size = 128,
num_hidden_layers = 4,
num_attention_heads = 4,
head_dim = 64,
qk_rope_head_dim = 32,
q_lora_rank = 128,
o_lora_rank = 64,
o_groups = 2,
n_routed_experts = 16,
num_experts_per_tok = 2,
sliding_window = 16,
compress_ratios = [0, 2, 1, 1],
kv_source_layer_ids = [1, 2],
index_source_layer_ids = [1, 2, 3],
index_n_heads = 2,
index_head_dim = 64,
candidate_source_layer_id = 2,
engram_layer_ids = [1],
engram_num_embeddings = [1024],
engram_n_heads = 2,
engram_head_dim = 64,
)
config = {
"model_type": "deepseek_v41",
"text_config": text,
"vision_config": {"num_hidden_layers": 1, "hidden_size": 64},
}
model = v41.Model(ModelConfig.from_dict(json.loads(json.dumps(config))))
def oq2e(path, module):
if (
not hasattr(module, "to_quantized")
or path.startswith(("vision", "aligner"))
or path.endswith(("head", "embed_tokens", "ffn.gate", "wo_a"))
):
return False
if path.endswith("engram.wkv"):
return {"group_size": 32, "bits": 8, "mode": "mxfp8"}
if path.endswith("engram.embed"):
return {"group_size": 32, "bits": 4, "mode": "affine"}
return {"group_size": 64, "bits": 2 if ".switch_mlp." in path else 8, "mode": "affine"}
specs = {}
nn.quantize(
model, class_predicate = lambda path, module: specs.setdefault(path, oq2e(path, module))
)
names = {"gate_proj": "w1", "down_proj": "w2", "up_proj": "w3"}
def omlx(key):
key = key.replace("language_model.embed_tokens.", "language_model.embed.").replace(
".switch_mlp.", ".experts."
)
key = re.sub(r"(experts)\.(\w+_proj)\.", lambda m: f"{m[1]}.{names[m[2]]}.", key)
return re.sub(r"^(vision\.blocks\.\d+)\.mlp\.", r"\1.ffn.", key)
# bf16 as stored, beside the float32 the model is built in: a name sanitize misses keeps the latter.
params = {
k: v.astype(mx.bfloat16)
if v.dtype == mx.float32 and "hc_" not in k and "gate.bias" not in k
else v
for k, v in tree_flatten(model.parameters())
}
stored = {
omlx(k): v.reshape(-1, v.shape[-1]) if k.endswith("wo_a.weight") else v
for k, v in params.items()
}
stored["language_model.mtp.0.ffn.experts.w1.weight"] = mx.zeros(
(16, 128, 32), mx.uint32
) # a DSpark stage
mx.save_safetensors(str(tmp_path / "model.safetensors"), stored)
# No "quantization": only unsloth_zoo's reading of the config supplies it.
quantized = {
omlx(f"{p}.weight").removesuffix(".weight"): s
if s["mode"] == "affine"
else {"bits": s["bits"], "mode": s["mode"]}
for p, s in specs.items()
if s and not p.endswith("engram.embed")
}
quantized["language_model.mtp.0.ffn.experts.w1"] = {"bits": 4, "mode": "mxfp4"}
table = "language_model.layers.1.engram.embed"
keys = {
"weight_key": f"{table}.weight",
"scale_key": f"{table}.scales",
"bias_key": f"{table}.biases",
}
config["omlx_deepseek_v41"] = {
"version": 1,
"quantized_modules": quantized,
"engram_tables": {table: {**keys, "bits": 4, "group_size": 32, "mode": "affine"}},
}
(tmp_path / "config.json").write_text(json.dumps(config))
model_dir = str(tmp_path)
priced = mm._snapshot_config(model_dir)
# Installed process-wide: reset between checks so each must install it itself.
sanitize = v41.Model.sanitize
if getattr(sanitize, "_unsloth_omlx_layout", False):
pytest.skip("an earlier load in this process installed the oQ layout")
monkeypatch.setattr(v41.Model, "sanitize", sanitize)
assert mm._load_is_refused(model_dir, priced, load_in_4bit = True) is None
v41.Model.sanitize = sanitize
experts = sum(v.nbytes for k, v in params.items() if ".switch_mlp." in k)
found = mm._offloadable(model_dir, priced)
assert (found["expert_bytes"], found["experts"]) == (experts, None)
assert mm.mlx_offloadable_parts(model_dir) == ("experts",)
# The view carrying that config lives exactly as long as the model mapping its shards.
lazy = mm._lazy_checkpoint(model_dir, priced)
(view,) = lazy._unsloth_mlx_config_view_paths
assert os.path.isfile(os.path.join(view, "model.safetensors"))
del lazy
gc.collect()
assert not os.path.exists(view)
@pytest.mark.parametrize("name", ["_ensure_omlx_layout", "_ensure_deepseek_v41_omlx_layout"])
def test_unsloth_zoo_installs_its_oq_layout_by_either_name(monkeypatch, name):
loader = pytest.importorskip("unsloth_zoo.mlx.loader")
installed = []
for other in ("_ensure_omlx_layout", "_ensure_deepseek_v41_omlx_layout"):
monkeypatch.delattr(loader, other, raising = False)
monkeypatch.setattr(loader, name, installed.append, raising = False)
mm._install_oq_renames({"model_type": "glm5_next"})
assert installed == ["glm5_next"]
def test_a_glm5_next_oq_quant_is_priced_as_unsloth_zoo_loads_it(tmp_path, monkeypatch):
if not _HAVE_MLX:
pytest.skip("needs MLX to build and load a checkpoint")
glm = pytest.importorskip("mlx_vlm.models.glm5_next.glm5_next")
loader = pytest.importorskip("unsloth_zoo.mlx.loader")
if "glm5_next" not in getattr(loader, "_OMLX_LAYOUTS", {}):
pytest.skip("this unsloth_zoo loads no GLM-5.3 oQ quant")
import re
import mlx.core as mx
import mlx.nn as nn
from mlx.utils import tree_flatten
from mlx_vlm.models.glm5_next.config import ModelConfig
linear = {
"num_heads": 2,
"head_dim": 64,
"short_conv_kernel_size": 4,
"kda_layers": [0],
"full_attn_layers": [1],
}
text = dict(
model_type = "glm5_next_text",
vocab_size = 512,
hidden_size = 256,
num_hidden_layers = 2,
intermediate_size = 256,
moe_intermediate_size = 128,
n_routed_experts = 16,
num_experts_per_tok = 2,
first_k_dense_replace = 1,
num_attention_heads = 2,
num_key_value_heads = 2,
q_lora_rank = 128,
kv_lora_rank = 128,
qk_head_dim = 64,
qk_nope_head_dim = 64,
qk_rope_head_dim = 0,
v_head_dim = 64,
index_n_heads = 2,
index_head_dim = 64,
linear_attn_config = linear,
layer_types = ["linear_attention", "deepseek_sparse_attention"],
mlp_layer_types = ["dense", "sparse"],
)
vision = dict(depth = 1, hidden_size = 64, intermediate_size = 128, num_heads = 1, out_hidden_size = 256)
config = {"model_type": "glm5_next", "text_config": text, "vision_config": vision}
model = glm.Model(ModelConfig.from_dict(json.loads(json.dumps(config))))
nn.quantize(
model,
64,
4,
class_predicate = lambda path, module: hasattr(module, "to_quantized")
and model.quant_predicate(path, module),
)
config["quantization"] = {"group_size": 64, "bits": 4, "mode": "affine"}
params = dict(tree_flatten(model.parameters()))
# The layout omlx writes: its forget gate module, the fused convolution as `conv1d`, `vision_model`.
def omlx(key):
key = key.replace("vision_tower.", "vision_model.").replace(".qkv_conv.conv.", ".conv1d.")
return re.sub(r"\.self_attn\.(f_b_proj|A_log|dt_bias)", r".self_attn.forget_gate.\1", key)
fbg = "language_model.model.layers.0.self_attn.fbg_a_proj."
stored = {omlx(k): v for k, v in params.items() if not k.startswith(fbg)}
for leaf in ("weight", "scales", "biases"):
for name, part in zip(
("forget_gate.f_a_proj", "b_proj", "g_a_proj"),
mx.split(params[fbg + leaf], [64, 66], axis = 0),
):
stored[f"language_model.model.layers.0.self_attn.{name}.{leaf}"] = part
stored["language_model.mtp.0.eh_proj.weight"] = mx.zeros((256, 512))
mx.save_safetensors(str(tmp_path / "model.safetensors"), stored)
(tmp_path / "config.json").write_text(json.dumps(config))
model_dir = str(tmp_path)
priced = mm._snapshot_config(model_dir)
sanitize = glm.Model.sanitize
if getattr(sanitize, "_unsloth_omlx_layout", False):
pytest.skip("an earlier load in this process installed the oQ layout")
monkeypatch.setattr(glm.Model, "sanitize", sanitize)
assert mm._load_is_refused(model_dir, priced, load_in_4bit = False) is None
glm.Model.sanitize = sanitize
experts = sum(v.nbytes for k, v in params.items() if ".switch_mlp." in k)
found = mm._offloadable(model_dir, priced)
assert (found["expert_bytes"], found["experts"]) == (experts, None)
try:
import importlib
importlib.import_module("mlx.core")
importlib.import_module("mlx_lm")
_HAVE_MLX = True
except Exception:
_HAVE_MLX = False
_NEEDS_MLX = pytest.mark.skipif(not _HAVE_MLX, reason = "needs a working MLX install")
def _mlx_raise():
raise NotImplementedError
def _plan_entry(**over):
"""One cache plan entry: a growing key/value cache unless overridden."""
return {
"const": 0.0,
"slope": 4096.0,
"quant_const": 0.0,
"quant_slope": 1152.0,
"bound_spec": None,
"conv_width": 0,
"block": 256,
"converts": True,
**over,
}
_GROWING = _plan_entry()
_RECURRENT = _plan_entry(
const = 2_195_456.0,
slope = 0.0,
quant_const = 2_195_456.0,
quant_slope = 0.0,
conv_width = 8192,
block = 1,
converts = False,
)
_UNQUANTIZED = {"quant_start": None, "prefill_chunk": mm.MLX_PREFILL_CHUNK}
def _caption(
plan,
bits = None,
converted = False,
n_ctx = 32768,
):
return mm._cache_width_name(plan, bits, converted, "bf16", n_ctx, mm.MLX_PREFILL_CHUNK)
def _local_snapshot(repo):
"""This repo's snapshot directory, skipping the test when it is not here."""
hub = os.path.expanduser("~/.cache/huggingface/hub")
hits = glob.glob(
os.path.join(hub, "models--" + repo.replace("/", "--"), "snapshots", "*", "config.json")
)
if not hits:
pytest.skip(f"{repo} not on this disk")
return os.path.dirname(hits[0])
@_NEEDS_MLX
@pytest.mark.parametrize("module", ["mlx_lm.models.cache", "mlx_vlm.models.cache"])
def test_a_linear_attention_cache_reports_its_convolution_width(module):
import importlib
import mlx.core as mx
cache = importlib.import_module(module).ArraysCache(2)
cache[0], cache[1] = mx.zeros((1, 3, 8192)), mx.zeros((1, 32, 128, 128))
assert mm._conv_width(cache) == 8192
@_NEEDS_MLX
def test_a_runtime_4bit_load_is_priced_with_its_quantized_embeddings(tmp_path):
import mlx.core as mx
from mlx.utils import tree_flatten, tree_map
from mlx_lm.models import llama
from mlx_lm.utils import save_model
from tokenizers import Tokenizer, models, pre_tokenizers
from unsloth_zoo.mlx.loader import FastMLXModel, _resolve_mlx_quantization_spec
if "quantize_embeddings" not in inspect.signature(_resolve_mlx_quantization_spec).parameters:
pytest.skip("this unsloth_zoo keeps embeddings full width")
_on_bfloat16_chip()
config = dict(
model_type = "llama",
hidden_size = 128,
num_hidden_layers = 2,
intermediate_size = 256,
num_attention_heads = 4,
num_key_value_heads = 4,
rms_norm_eps = 1e-5,
vocab_size = 512,
tie_word_embeddings = False,
)
model = llama.Model(llama.ModelArgs(**config))
model.update(tree_map(lambda p: p.astype(mx.bfloat16), model.parameters()))
mx.eval(model.parameters())
save_model(str(tmp_path), model)
config = {**config, "architectures": ["LlamaForCausalLM"], "torch_dtype": "bfloat16"}
(tmp_path / "config.json").write_text(json.dumps(config))
vocab = {f"t{i}": i for i in range(511)}
vocab["[UNK]"] = 511
tokenizer = Tokenizer(models.WordLevel(vocab, unk_token = "[UNK]"))
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
tokenizer.save(str(tmp_path / "tokenizer.json"))
(tmp_path / "tokenizer_config.json").write_text(
json.dumps(
{"tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "[UNK]", "eos_token": "t0"}
)
)
loaded, _ = FastMLXModel.from_pretrained(
str(tmp_path), load_in_4bit = True, quantize_embeddings = True, text_only = True
)
held = sum(value.nbytes for _, value in tree_flatten(loaded.parameters()))
assert mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = True) == held
def _on_bfloat16_chip():
import mlx.core as mx
if mm._runtime_dtype() is not mx.bfloat16:
pytest.skip("measured on a chip the loader gives bfloat16")
class TestCacheWidthName:
def test_only_the_entries_that_grow_name_the_cache(self):
hybrid = [_RECURRENT] * 30 + [_GROWING] * 10
assert _caption(hybrid) == "bf16"
assert _caption(hybrid, 4, True) == "4-bit"
assert _caption([_RECURRENT]) == "bf16"
def test_a_converted_run_is_named_as_its_own_control_is_labelled(self):
assert [_caption([_GROWING] * 10, b, True) for b in (8, 6, 5, 4, 3, 2)] == [
"8-bit",
"6-bit",
"5-bit",
"4-bit",
"3-bit",
"2-bit",
]
assert _caption([_GROWING] * 10, 4, False) == "bf16"
assert _caption([_GROWING] * 10, mm.MlxTurboQuant(3.5), True) == "TurboQuant 3.5-bit"
def test_an_entry_that_cannot_convert_keeps_its_width_through_a_converting_run(self):
staying = _plan_entry(converts = False, quant_slope = _GROWING["slope"])
assert _caption([staying] * 57 + [_GROWING] * 3, 4, True) == "bf16/4-bit"
heavy, light = (
_plan_entry(quant_slope = 100_000.0),
_plan_entry(converts = False, slope = 1.0, quant_slope = 1.0),
)
assert _caption([light] * 50 + [heavy] * 2, 4, True) == "4-bit/bf16"
@_NEEDS_MLX
def test_a_bounded_entry_is_weighed_by_what_it_holds_and_not_by_its_slope(self):
from mlx_lm.models.cache import ChunkedKVCache
bounded = _plan_entry(converts = False, bound_spec = (ChunkedKVCache, "chunk_size", 8192))
plan = [bounded] * 3 + [_plan_entry(quant_slope = 1152.0)]
assert _caption(plan, 4, True, 8192).startswith("bf16")
assert _caption(plan, 4, True, 131_072).startswith("4-bit")
@_NEEDS_MLX
def test_every_width_a_cache_is_built_at_has_a_caption(self):
import mlx.core as mx
assert [mm._width_name(d) for d in (mx.bfloat16, mx.float16, mx.float32)] == [
"bf16",
"f16",
"f32",
]
@_NEEDS_MLX
def test_a_weightless_probe_never_runs_the_evaluations_its_model_asks_for(monkeypatch):
import threading
import mlx.core as mx
from mlx_lm.models.cache import KVCache
elsewhere = []
def evaluates():
before, other = mx.get_active_memory(), mx.ones((16, 1024, 1024))
mx.eval(other)
elsewhere.append(mx.get_active_memory() - before >= other.nbytes)
class Tower:
def __call__(self, inputs, cache):
weights = mx.ones((256, 1024, 1024)) # 1 GiB once evaluated
mx.async_eval(weights)
mx.eval(weights)
thread = threading.Thread(target = evaluates)
thread.start()
thread.join()
keys = mx.zeros((1, 2, inputs.shape[1], 8)) + weights[0, 0, 0]
return cache[0].update_and_fetch(keys, keys)[0]
monkeypatch.setattr(
mm, "_probe_models", lambda c, d: iter([(Tower, lambda m: [KVCache()], Tower)])
)
mx.reset_peak_memory()
assert len(mm._cache_plan({}, mx.bfloat16, None)[0]) == 1
assert mx.get_peak_memory() < 2**28
from core.inference.mlx_inference import mlx_kv_quant_is_refused
monkeypatch.setattr(mm, "_snapshot_config", lambda d: {})
mlx_kv_quant_is_refused("weightless", 8)
assert mx.get_peak_memory() < 2**28
assert elsewhere and all(elsewhere)
mx.eval(mx.ones((64, 1024, 1024)))
assert mx.get_peak_memory() >= 2**28
@pytest.mark.skipif(not _HAVE_MLX, reason = "resolves an mlx-vlm architecture")
def test_a_text_model_only_mlx_vlm_builds_is_sized_on_mlx_vlm_and_served_as_text(monkeypatch):
zoo, asked = _types.ModuleType("unsloth_zoo.mlx.loader"), []
zoo._mlx_vlm_only_text_model = lambda config, model_type: (
asked.append(model_type) or "engram_config" in config
)
monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.loader", zoo)
refused = {"model_type": "deepseek_v4", "engram_config": {}}
assert mm._builds_on_mlx_vlm(refused)
assert not mm._builds_on_mlx_vlm({"model_type": "deepseek_v4"})
assert asked == ["deepseek_v4", "deepseek_v4"]
assert not mm._loads_as_vision(refused)
_, make_cache, _ = next(mm._probe_models(refused, None))
assert make_cache.__module__ == "mlx_vlm.models.cache"
@pytest.mark.skipif(not _HAVE_MLX, reason = "runs a lazy forward")
def test_a_tower_that_cannot_stand_alone_leaves_the_whole_model_to_size(monkeypatch):
import mlx.core as mx
from mlx_lm.models.cache import KVCache
class Whole:
def __call__(self, inputs, cache):
keys = mx.zeros((1, 2, inputs.shape[1], 8))
return cache[0].update_and_fetch(keys, keys)[0]
def tower():
raise ValueError("this architecture declares no text config")
monkeypatch.setattr(
mm,
"_probe_models",
lambda c, d: iter(
[(tower, lambda m: [KVCache()], Whole), (Whole, lambda m: [KVCache()], Whole)]
),
)
assert len(mm._cache_plan({}, mx.bfloat16, None)[0]) == 1
class TestKvBytes:
def test_a_hybrid_plan_is_far_cheaper_than_charging_every_layer_as_attention(self):
hybrid = [_RECURRENT] * 30 + [_GROWING] * 10
assert (
mm._kv_bytes(hybrid, 131_072, **_UNQUANTIZED)[0] * 3
< mm._kv_bytes([_GROWING] * 40, 131_072, **_UNQUANTIZED)[0]
)
def test_a_runtime_that_declines_to_chunk_is_recognised_on_either_half(self):
assert mm._declines_to_chunk(type("M", (), {"no_chunked_prefill": True}), object())
assert mm._declines_to_chunk(object, type("T", (), {"no_chunked_prefill": True})())
assert not mm._declines_to_chunk(object, object())
def test_a_cache_converted_before_prefill_is_never_charged_at_full_width(self):
plan = [_GROWING, _RECURRENT]
quantized = mm._line_bytes(plan, 32768, 2048, quantized = True)
assert mm._kv_bytes(plan, 32768, 0, 2048) == (quantized, 0)
assert mm._kv_bytes(plan, 2048, 0, 2048)[0] == mm._line_bytes(plan, 2048, 2048, True)
def test_one_failed_conversion_refuses_the_whole_request(self):
def _entry(converts):
return type(
"Entry",
(),
{
"max_size": None,
"window_size": None,
"to_quantized": lambda self, **kw: "converted" if converts else _mlx_raise(),
},
)()
assert mm._quantize_like_runtime([_entry(True), _entry(False)], 4) is None
def test_a_windowed_entry_keeps_its_width_while_the_rest_convert(self):
from core.inference.mlx_inference import MLX_KV_GROUP_SIZE
def _entry(**window):
attrs = {"max_size": None, "window_size": None, **window}
attrs["to_quantized"] = lambda self, **kw: ("converted", kw)
return type("Entry", (), attrs)()
full, ring, window = _entry(), _entry(max_size = 512), _entry(window_size = 512)
quantized = ("converted", {"group_size": MLX_KV_GROUP_SIZE, "bits": 4})
assert mm._quantize_like_runtime([full, ring], 4) == [quantized, ring]
last = _entry()
assert mm._quantize_like_runtime([full, last, ring, window], 4) == [
quantized,
last,
ring,
window,
]
assert mm._quantize_like_runtime([full, ring, window], 4) is None
assert mm._quantize_like_runtime([ring, window], 4) is None
assert mm._quantize_like_runtime([full], None) is None
@pytest.mark.skipif(not _HAVE_MLX, reason = "drives a real cache class")
def test_a_bounded_window_leaves_the_request_full_width(self):
from mlx_vlm.models.unlimited_ocr.language import RingSlidingKVCache
entry = RingSlidingKVCache(512)
assert entry.to_quantized(group_size = 64, bits = 4) is not None
assert mm._quantize_like_runtime([entry], 4) is None
class TestComputeBytes:
def test_an_unreadable_width_is_not_guessed(self):
assert mm._compute_bytes(mm._config_widths({}), 2, 2048, [_GROWING], 8192, None) == 0
def test_the_calibrated_term_matches_the_measurement_it_came_from(self):
measured = 610 * 1024**2
priced = mm._compute_bytes(
(1024, 3072, 16), 2, mm.MLX_PREFILL_CHUNK, [_GROWING] * 28, 8192, None
)
assert 0.7 * measured < priced < 1.5 * measured, (
f"prices {priced / 1024**2:.0f} MiB where the load measured "
f"{measured / 1024**2:.0f} MiB"
)
def _unfused_route(monkeypatch, fuses):
"""mlx and zoo stubs, so the unquantized route runs where MLX does not."""
asked = []
mx = SimpleNamespace(
float32 = SimpleNamespace(size = 4),
bfloat16 = SimpleNamespace(size = 2),
zeros = lambda shape, dtype: SimpleNamespace(shape = shape, dtype = dtype),
)
def fused_kernel_exists(
queries,
q_keys,
q_values,
group_size,
mask = None,
):
L, S = queries.shape[-2], q_keys[0].shape[-2]
assert q_keys[1].shape[-1] * group_size == q_values[1].shape[-1] * group_size == 256
assert queries.shape[1] == 24 and q_keys[0].shape[1] == 4 and mask == "causal"
asked.append((L, S))
return fuses(L, S)
monkeypatch.setitem(sys.modules, "mlx", SimpleNamespace(core = mx))
monkeypatch.setitem(sys.modules, "mlx.core", mx)
monkeypatch.setitem(
sys.modules,
"unsloth_zoo.mlx.attention",
SimpleNamespace(fused_kernel_exists = fused_kernel_exists),
)
return asked
_BONSAI = (5120, 17408, 24, 4, 256, 0)
_SCORE = 24 * 2 * mm._SCORE_LIVE
@pytest.mark.parametrize(
"fuses, cells",
[
(lambda L, S: True, 0),
(lambda L, S: L <= 8, 2048 * 32768),
# NAX: the split kernel needs 1024 queries, so only the 1023-token tail is unfused.
(lambda L, S: L <= 8 or L >= 1024, 1023 * 33791),
],
)
def test_a_full_width_cache_pays_for_the_widest_call_mlx_does_not_fuse(monkeypatch, fuses, cells):
n_ctx = 33792
floor = mm._compute_bytes(_BONSAI[:3], 2, 2048, [_GROWING], n_ctx, None)
asked = _unfused_route(monkeypatch, fuses)
priced = mm._compute_bytes(_BONSAI, 2, 2048, [_GROWING], n_ctx, None)
assert priced - floor == pytest.approx(cells * _SCORE, abs = 1)
assert (2048, 2 * 2048) in asked
def test_a_tail_unfused_past_any_fixed_scan_is_still_priced(monkeypatch):
# NAX d512 with few heads: tails fuse only past 8160 queries, longer than a fixed scan reaches.
_unfused_route(monkeypatch, lambda L, S: L <= 8 or L > 8160)
floor = mm._compute_bytes(_BONSAI[:3], 2, 16384, [_GROWING], 32768, None)
priced = mm._compute_bytes(_BONSAI, 2, 16384, [_GROWING], 32768, None)
assert priced - floor == pytest.approx(8160 * (16384 + 8160) * _SCORE, abs = 1)
def test_the_scores_charged_never_fall_as_the_context_grows(monkeypatch):
_unfused_route(monkeypatch, lambda L, S: L <= 8 or L == S >= 2048)
steps = range(256, 12_289, 256)
priced = [mm._compute_bytes(_BONSAI, 2, 2048, [_GROWING], n, None) for n in steps]
assert priced == sorted(priced)
def test_a_windowed_model_and_an_unreachable_zoo_keep_the_old_price(monkeypatch):
widths = _BONSAI
floor = mm._compute_bytes(widths[:3], 2, 2048, [_GROWING], 262144, None)
asked = _unfused_route(monkeypatch, lambda L, S: False)
windowed = [_GROWING, _plan_entry(bound_spec = (object, "max_size", 1024))]
assert mm._compute_bytes(widths, 2, 2048, windowed, 262144, None) == floor
assert asked == []
monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.attention", None)
assert mm._compute_bytes(widths, 2, 2048, [_GROWING], 262144, None) == floor
def _longest_unfused_d256_tail():
from unsloth_zoo.mlx.nax import nax_available
# NAX fuses head_dim 256 from 1024 queries.
return 1023 if nax_available() else 2047
@_NEEDS_MLX
class TestTheQuantizedAttentionRouteIsTheOneZooTakes:
"""unsloth-zoo routes each call between a score matrix and a dequantized copy of the cache."""
GQA = (3584, 18944, 32, 8, 128, 0)
MHA = (3584, 18944, 32, 32, 128, 0)
WIDE = (3584, 18944, 16, 2, 256, 0)
@pytest.mark.parametrize(
"widths, chunk, copies",
[
(GQA, 2048, True),
(GQA, 256, True),
(MHA, 2048, True),
(MHA, 256, False),
],
)
def test_both_halves_of_the_wrappers_choice_are_asked(self, widths, chunk, copies):
assert (mm._dequantized_row_bytes(widths, 2, 4, 64, chunk) is not None) is copies
def test_the_copy_charged_is_the_cache_the_wrapper_would_make(self):
import mlx.core as mx
from unsloth_zoo.mlx.attention import _row_bytes
cache = mx.quantize(mx.zeros((1, 8, 1, 128), dtype = mx.bfloat16), group_size = 64, bits = 4)
assert mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048) == (
2 * 8 * _row_bytes(cache, 64, mx.bfloat16)
)
def test_a_copied_route_is_charged_the_copy_and_not_the_scores(self):
row = mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)
floor = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, None)
priced = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, 2048, 4, 64)
scores = mm._widest_quantized_scores(32768, 2048, 2048) * 32 * 2 * mm._SCORE_LIVE
assert priced - floor == row * 32768 < scores
def test_a_short_call_still_scores_where_that_is_the_wider_of_the_two(self):
# 256:1 is past the vector kernel's limit, so the eight-query tail is unfused.
widths = (3584, 18944, 256, 1, 64, 0)
row = mm._dequantized_row_bytes(widths, 2, 8, 64, 2048)
floor = mm._compute_bytes(widths[:3], 2, 2048, [], 32768, None)
charged = mm._compute_bytes(widths, 2, 2048, [], 32768, 2048, 8, 64) - floor
assert row is not None
assert charged == pytest.approx(8 * (15 * 2048 + 8) * 256 * 2 * mm._SCORE_LIVE, abs = 1)
assert charged > row * 32768
def test_a_full_width_cache_is_charged_only_the_tail_the_runtime_leaves_unfused(self):
from unsloth_zoo.mlx import attention
if not hasattr(attention, "runtime_fuses"):
pytest.skip("this zoo keeps head_dim 256 on mlx's dispatch")
assert mm._dequantized_row_bytes(self.WIDE, 2, 4, 64, 2048) is not None
plan = [_GROWING] * 4
unrouted = mm._compute_bytes(self.GQA[:3], 2, 2048, plan, 32768, None)
assert mm._compute_bytes(self.GQA, 2, 2048, plan, 32768, None) == unrouted
wide = mm._compute_bytes(self.WIDE, 2, 2048, plan, 32768, None)
tail = _longest_unfused_d256_tail()
assert wide - unrouted == pytest.approx(
tail * (3 * 2048 + tail) * 16 * 2 * mm._SCORE_LIVE, abs = 1
)
def test_a_copied_route_is_charged_a_tail_whose_scores_outgrow_the_copy(self):
from unsloth_zoo.mlx import attention
if not hasattr(attention, "runtime_fuses"):
pytest.skip("this zoo keeps head_dim 256 on mlx's dispatch")
widths, n_ctx = (3584, 18944, 32, 2, 256, 0), 262144
row = mm._dequantized_row_bytes(widths, 2, 4, 64, 2048)
floor = mm._compute_bytes(widths[:3], 2, 2048, [], n_ctx, None)
queries = _longest_unfused_d256_tail()
tail = queries * (3 * 2048 + queries) * 32 * 2 * mm._SCORE_LIVE
priced = mm._compute_bytes(widths, 2, 2048, [], n_ctx, 0, 4, 64) - floor
assert priced == int(max(row * n_ctx, tail))
def test_an_older_zoo_keeps_head_dim_256_on_mlxs_dispatch(self, monkeypatch):
from unsloth_zoo.mlx import attention
from unsloth_zoo.mlx.nax import nax_available
if nax_available():
pytest.skip("NAX fuses head_dim 256 prefill")
monkeypatch.delattr(attention, "runtime_fuses", raising = False)
assert mm._dequantized_row_bytes(self.WIDE, 2, 4, 64, 2048) is None
plan = [_GROWING] * 4
unrouted = mm._compute_bytes(self.GQA[:3], 2, 2048, plan, 32768, None)
wide = mm._compute_bytes(self.WIDE, 2, 2048, plan, 32768, None)
assert wide - unrouted == pytest.approx(
mm._widest_quantized_scores(32768, 0, 2048) * 16 * 2 * mm._SCORE_LIVE, abs = 1
)
def test_a_zoo_that_cannot_route_it_keeps_the_scores_whole(self, monkeypatch):
import builtins
real = builtins.__import__
def _no_wrapper(name, *a, **kw):
if name == "unsloth_zoo.mlx.attention":
raise ImportError("this zoo predates the wrapper")
return real(name, *a, **kw)
monkeypatch.setattr(builtins, "__import__", _no_wrapper)
assert mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048) is None
floor = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, None)
priced = mm._compute_bytes(self.GQA, 2, 2048, [], 32768, 2048, 4, 64)
assert priced - floor == pytest.approx(
mm._widest_quantized_scores(32768, 2048, 2048) * 32 * 2 * mm._SCORE_LIVE, abs = 1
)
@pytest.mark.parametrize(
"stated, expected",
[
({"num_key_value_heads": 8, "head_dim": 128}, (8, 128, 0)),
({}, (32, 112, 0)),
({"num_key_value_heads": 4}, (4, 112, 0)),
({"head_dim": 64}, (32, 64, 0)),
({"head_dim": 192, "v_head_dim": 128}, (32, 192, 128)),
],
)
def test_the_geometry_the_route_reads_comes_off_the_checkpoint(self, stated, expected):
from types import SimpleNamespace
config = {"hidden_size": 3584, "num_attention_heads": 32, **stated}
assert mm._config_widths(config)[3:] == expected
assert mm._tower_widths(SimpleNamespace(args = SimpleNamespace(**config)))[3:] == expected
def test_a_tower_whose_values_are_narrower_keeps_the_scores(self):
assert mm._dequantized_row_bytes((3584, 18944, 32, 8, 128, 64), 2, 4, 64, 2048) is None
assert mm._dequantized_row_bytes((3584, 18944, 32, 8, 160, 128), 2, 4, 64, 2048) is None
assert mm._dequantized_row_bytes((3584, 18944, 32, 8, 128, 128), 2, 4, 64, 2048) == (
mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)
)
def test_a_tower_that_states_nothing_is_still_priced_from_its_checkpoint(self):
assert len(mm._tower_widths(object())) == len(mm._config_widths({}))
config = {
"hidden_size": 3584,
"intermediate_size": 18944,
"num_attention_heads": 32,
"num_key_value_heads": 8,
"head_dim": 128,
}
merged = tuple(
tower or checkpoint
for tower, checkpoint in zip(mm._tower_widths(object()), mm._config_widths(config))
)
assert mm._dequantized_row_bytes(merged, 2, 4, 64, 2048) == (
mm._dequantized_row_bytes(self.GQA, 2, 4, 64, 2048)
)
def test_an_unquantized_load_never_reaches_the_route(self):
assert mm._dequantized_row_bytes(self.GQA, 2, None, 64, 2048) is None
assert mm._dequantized_row_bytes((3584, 18944, 32, 0, 0, 0), 2, 4, 64, 2048) is None
@pytest.mark.parametrize("chunk", [256, 2048])
def test_the_route_is_asked_about_a_cache_longer_than_the_chunk(monkeypatch, chunk):
"""zoo never fuses a causal chunk over fewer keys than queries (#15935)."""
asked = []
def quantize(w, group_size, bits):
*lead, width = w.shape
packed = SimpleNamespace(shape = (*lead, width * bits // 32), dtype = SimpleNamespace(size = 4))
scales = SimpleNamespace(shape = (*lead, width // group_size), dtype = w.dtype)
return packed, scales, scales
mx = SimpleNamespace(
float32 = SimpleNamespace(size = 4),
bfloat16 = SimpleNamespace(size = 2),
zeros = lambda shape, dtype: SimpleNamespace(shape = shape, dtype = dtype),
quantize = quantize,
)
def fused_kernel_exists(
queries,
q_keys,
q_values,
group_size,
mask = None,
):
asked.append((queries.shape[-2], q_keys[0].shape[-2], q_values[0].shape[-2], mask))
return q_keys[0].shape[-2] >= queries.shape[-2]
zoo = SimpleNamespace(
_row_bytes = lambda cache, group_size, dtype: 7,
dequantizing_is_smaller = lambda *args: True,
fused_kernel_exists = fused_kernel_exists,
)
monkeypatch.setitem(sys.modules, "mlx", SimpleNamespace(core = mx))
monkeypatch.setitem(sys.modules, "mlx.core", mx)
monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.attention", zoo)
widths = (3584, 18944, 32, 8, 128, 0)
assert mm._dequantized_row_bytes(widths, 2, 4, 64, chunk) == 8 * 2 * 7
((queries, keys, values, mask),) = asked
assert queries == chunk and keys == values > chunk
assert mask == "causal"
def test_chunks_below_a_forced_routes_key_floor_still_charge_their_scores(monkeypatch):
_unfused_route(monkeypatch, lambda L, S: L > 15 and S >= 8192)
assert mm._widest_unfused_scores(_BONSAI, 2, 65536, 2048) == 2047 * (3 * 2048 + 2047)
assert mm._widest_unfused_scores(_BONSAI, 2, 4352, 2048) == 2048 * 2 * 2048
assert mm._widest_unfused_scores(_BONSAI, 2, 32768, 4096) == 4095 * 8191
assert mm._widest_unfused_scores(_BONSAI, 2, 65536, 16384) == 8191 * 8191
@pytest.mark.parametrize(
"fuses",
[
lambda L, S: L > 15 and S >= 256,
lambda L, S: L >= 32 or (L > 15 and S >= 256),
lambda L, S: L > 8,
lambda L, S: L > 8 and L == S,
lambda L, S: False,
],
)
def test_the_widest_call_matches_walking_every_prompt_the_context_admits(monkeypatch, fuses):
_unfused_route(monkeypatch, fuses)
for chunk in (16, 48, 64, 80, 96, 384):
for n_ctx in range(2, 900, 37):
widest = 0
for prompt in range(1, n_ctx + 1):
whole, rest = divmod(prompt - 1, chunk)
calls = [(chunk, held * chunk) for held in range(1, whole + 1)]
calls += [(rest, whole * chunk + rest)] if rest else []
calls += [(1, prompt)]
widest = max([widest] + [L * S for L, S in calls if not fuses(L, S)])
charged = mm._widest_unfused_scores(_BONSAI, 2, n_ctx, chunk)
assert widest <= charged <= max(widest, n_ctx + 1), (chunk, n_ctx)
def test_tails_below_a_key_floor_are_charged_though_every_full_chunk_fuses(monkeypatch):
# NAX head_dim 256: mlx fuses from 1024 queries, and shorter calls only past the floor.
_unfused_route(monkeypatch, lambda L, S: L >= 1024 or (L > 15 and S >= 8192))
assert mm._widest_unfused_scores(_BONSAI, 2, 65536, 2048) == 1023 * (3 * 2048 + 1023)
@_NEEDS_MLX
@pytest.mark.parametrize("on_nax, row", [(False, 4 * 192 * 2), (True, 4 * 512 * 2)])
def test_a_padded_route_is_charged_its_copies_where_they_outgrow_the_scores(
monkeypatch, on_nax, row
):
from unsloth_zoo.mlx import attention, nax
if not hasattr(attention, "forced_padding_row_bytes"):
pytest.skip("needs a zoo with the padded route")
monkeypatch.setattr(nax, "nax_available", lambda: on_nax)
widths = (4096, 16384, 64, 4, 192, 128)
n_ctx = 2**22
floor = mm._compute_bytes(widths[:3], 2, 2048, [], n_ctx, None)
assert mm._compute_bytes(widths, 2, 2048, [], n_ctx, None) - floor >= row * n_ctx
assert mm._padding_row_bytes(widths, 2, 8191) == 0
@_NEEDS_MLX
def test_the_score_multiple_is_what_an_unfused_call_really_holds():
import mlx.core as mx
q = mx.zeros((1, 8, 512, 112), dtype = mx.bfloat16) + 1
k = mx.zeros((1, 8, 8192, 112), dtype = mx.bfloat16) + 1
mx.eval(q, k)
mx.synchronize()
mx.clear_cache()
base = mx.get_active_memory()
mx.reset_peak_memory()
mx.eval(mx.fast.scaled_dot_product_attention(q, k, k, scale = 0.1, mask = "causal"))
held = (mx.get_peak_memory() - base) / (8 * 512 * 8192 * 2)
assert held <= mm._SCORE_LIVE < held * 1.1
@_NEEDS_MLX
class TestATurboQuantCacheIsSizedAsMlxVlmConvertsIt:
@pytest.fixture(autouse = True)
def _off_nax(self, monkeypatch):
from unsloth_zoo.mlx import nax
monkeypatch.setattr(nax, "nax_available", lambda: False)
@pytest.mark.parametrize(
"widths, scored_cells",
[
((3584, 18944, 16, 4, 256, 256), 2047 * (3 * 2048 + 2047)),
# mlx's own kernel for this width starts above 8 queries.
((3584, 18944, 32, 4, 64, 64), 8 * (16 * 2048 + 8)),
],
)
def test_a_prefill_is_charged_its_dequantized_cache_and_any_scores(self, widths, scored_cells):
from unsloth_zoo.mlx import attention
if scored_cells and not hasattr(attention, "runtime_fuses"):
pytest.skip("needs a zoo with the head_dim 256 fused route")
n_ctx = 16 * 2048 + 1024
floor = mm._compute_bytes((*widths[:3], 0, 0, 0), 2, 2048, [], n_ctx, None)
priced = mm._compute_bytes(widths, 2, 2048, [], n_ctx, 0, mm.MlxTurboQuant(4)) - floor
copied = widths[3] * (widths[4] + widths[5]) * 10 * (n_ctx - 1)
assert priced == pytest.approx(
copied + scored_cells * widths[2] * 2 * mm._SCORE_LIVE, abs = 1
)
@staticmethod
def _filled(
entry,
tokens = 40,
width = 64,
):
import mlx.core as mx
entry.update_and_fetch(
mx.zeros((1, 2, tokens, width), dtype = mx.bfloat16),
mx.zeros((1, 2, tokens, width), dtype = mx.bfloat16),
)
return entry
def test_every_layer_but_the_last_converts_and_the_probed_cache_is_left_whole(self):
from mlx_vlm.models.cache import CacheList, KVCache
from mlx_vlm.turboquant import TurboQuantKVCache
cache = [
CacheList(self._filled(KVCache())),
self._filled(KVCache()),
self._filled(KVCache()),
]
converted = mm._quantize_like_runtime(cache, mm.MlxTurboQuant(4))
assert [
isinstance(leaf, TurboQuantKVCache)
for entry in converted
for leaf in mm._sub_caches(entry)
] == [True, True, False]
assert converted[2] is cache[2]
assert all(isinstance(leaf, KVCache) for entry in cache for leaf in mm._sub_caches(entry))
@pytest.mark.parametrize("bits", [4, 3.5])
def test_the_bytes_counted_are_the_ones_mlx_vlm_holds(self, bits):
from mlx_vlm.models.cache import KVCache
cache = [self._filled(KVCache(), width = 128) for _ in range(3)]
leaf = mm._quantize_like_runtime(cache, mm.MlxTurboQuant(bits))[0]
assert 0 < mm._entry_bytes(leaf) == leaf.nbytes < mm._entry_bytes(cache[0])
def test_a_cache_mlx_vlm_does_not_convert_stays_full_width(self):
from mlx_lm.models.cache import KVCache
assert mm._quantize_like_runtime([KVCache() for _ in range(3)], mm.MlxTurboQuant(4)) is None
def test_a_model_whose_attention_cannot_read_the_converted_cache_is_priced_full_width(self):
import mlx.core as mx
# moondream2 hands the fetched TurboQuant state straight to mx.fast.
config = {
"model_type": "moondream2",
"text_config": {
"hidden_size": 64,
"num_hidden_layers": 2,
"num_attention_heads": 4,
"num_key_value_heads": 4,
"intermediate_size": 128,
"vocab_size": 512,
},
"vision_config": {
"hidden_size": 32,
"num_hidden_layers": 1,
"num_attention_heads": 2,
"intermediate_size": 64,
},
}
probed = mm._probe(config, mx.bfloat16, 8, None)
with mm.weightless_forward():
build, make_prompt_cache, _ = next(iter(mm._probe_models(config, mx.bfloat16)))
cache = make_prompt_cache(build())
assert mm._quantize_like_runtime(cache, mm.MlxTurboQuant(4)) is not None
assert mm._probe(config, mx.bfloat16, 8, mm.MlxTurboQuant(4)) == probed
@pytest.mark.skipif(not _HAVE_MLX, reason = "imports architecture modules")
class TestProbeFollowsTheLoadersRoute:
"""The estimator must not price a load path that cannot be taken."""
def test_a_vision_model_is_never_priced_from_mlx_lms_copy_of_it(self):
import importlib
config = {"model_type": "kimi_vl", "vision_config": {"depth": 2}}
assert importlib.import_module("mlx_lm.models.kimi_vl")
assert {
model_class.__module__.split(".")[0]
for _, _, model_class in mm._probe_models(config, None)
} == {"mlx_vlm"}
@pytest.mark.parametrize(
"architecture, resolved",
[("DeepseekOCRForCausalLM", "deepseekocr"), ("DeepseekOCR2ForCausalLM", "deepseekocr_2")],
)
def test_a_config_is_read_as_the_loader_resolves_it_not_as_stated(
self, tmp_path, architecture, resolved
):
(tmp_path / "config.json").write_text(
json.dumps({"model_type": "deepseek_vl_v2", "architectures": [architecture]})
)
assert mm._snapshot_config(str(tmp_path))["model_type"] == resolved
def test_a_checkpoint_supplying_its_own_module_is_refused_and_not_imported(self):
with pytest.raises(ValueError, match = "its own model module"):
list(mm._probe_models({"model_type": "qwen3", "model_file": "modeling.py"}, None))
def test_the_probe_hands_the_forward_pass_a_model_in_eval_mode(self):
import mlx.core as mx
config = json.loads(
(Path(_local_snapshot("unsloth/Qwen3-4B-Thinking-2507")) / "config.json").read_text()
)
assert next(iter(mm._probe_models(config, mx.bfloat16)))[0]().training is False
@pytest.mark.parametrize(
"chip, expected", [("Apple M1 Max", "float16"), ("Apple M3 Max", "bfloat16")]
)
def test_the_chip_decides_the_width_the_loader_installs(self, monkeypatch, chip, expected):
import mlx.core as mx
monkeypatch.setattr(mx, "device_info", lambda: {"device_name": chip})
assert mm._runtime_dtype() is getattr(mx, expected)
@pytest.mark.skipif(not _HAVE_MLX, reason = "asks the installed runtime")
class TestGenerationSettingsComeFromTheLoader:
"""What a load prefills at is read off the runtime, except where Studio pins it."""
def test_each_path_is_read_from_the_function_that_would_run_it(self, monkeypatch):
import inspect
from core.inference import mlx_inference as mi
monkeypatch.setattr(mi, "mlx_vlm_snapshot_store_available", lambda: False)
for vision, drafted in ((False, False), (True, False), (False, True), (True, True)):
step = mi._generation_step(vision = vision, drafted = drafted)
assert mi.mlx_prefill_chunk(vision = vision, drafted = drafted) == (
inspect.signature(step).parameters["prefill_step_size"].default
)
assert mi._generation_step(vision = True, drafted = False).__module__.startswith("mlx_vlm")
assert mi._generation_step(vision = False, drafted = False).__module__.startswith("mlx_lm")
assert mi.mlx_prefill_chunk(drafted = True) != mi.mlx_prefill_chunk()
assert mi.mlx_prefill_chunk(vision = True, drafted = True) == mi.mlx_prefill_chunk(vision = True)
def test_a_pinned_vision_step_outranks_the_runtimes_own(self, monkeypatch):
from core.inference import mlx_inference as mi
vlm_default = mi._generation_default("prefill_step_size", 0, vision = True, drafted = False)
monkeypatch.setattr(mi, "vlm_prefill_step", lambda: vlm_default + 256)
media, bare = {"image_token_id": 151655}, {"model_type": "vlm"}
for store, config, expected in (
(True, media, vlm_default + 256),
(True, bare, vlm_default),
(False, media, vlm_default),
(True, None, vlm_default),
):
monkeypatch.setattr(mi, "mlx_vlm_snapshot_store_available", lambda p = store: p)
assert mi.mlx_prefill_chunk(vision = True, config = config) == expected
assert mi.mlx_prefill_chunk(config = media) == mi.MLX_PREFILL_CHUNK_FALLBACK
def test_the_step_priced_and_the_store_that_pins_it_answer_together(self, monkeypatch):
"""The estimate is only right while these two read the same capability."""
import importlib
from core.inference.mlx_inference import (
MLXInferenceBackend,
mlx_vlm_snapshot_store_available,
)
vlm_generate = importlib.import_module("mlx_vlm.generate")
backend = MLXInferenceBackend.__new__(MLXInferenceBackend)
backend._memory_limits_applied = {}
for cached_tokens, built in ((True, True), (False, False)):
backend._vlm_snapshot_store = None
backend._vlm_snapshot_store_unavailable = False
monkeypatch.setattr(
vlm_generate,
"GenerationResult",
type("R", (), {"cached_tokens": 0} if cached_tokens else {}),
)
assert mlx_vlm_snapshot_store_available() is built
assert (backend._vlm_prompt_cache_store() is not None) is built
@pytest.mark.parametrize("chunk", [None, 0, True, "2048"])
def test_a_runtime_that_states_no_usable_value_falls_back(self, monkeypatch, chunk):
from core.inference import mlx_inference as mi
def _stated(**kw):
return lambda prefill_step_size = chunk: None
monkeypatch.setattr(mi, "_generation_step", _stated)
assert mi.mlx_prefill_chunk() == mi.MLX_PREFILL_CHUNK_FALLBACK
def test_the_estimator_tells_the_loader_which_package_would_load_it(self, monkeypatch):
from core.inference import mlx_inference as mi
seen = []
monkeypatch.setattr(mi, "mlx_prefill_chunk", lambda **kw: seen.append(kw["vision"]) or 2048)
vision = mm._generation_settings({"model_type": "kimi_vl", "vision_config": {"depth": 2}})
assert mm._generation_settings({"model_type": "qwen3"}) == vision == (2048, 64)
assert seen == [True, False]
def test_every_loader_quantizes_a_converted_cache_from_the_first_token(self, monkeypatch):
entry = {
"bytes": 1,
"quant_bytes": 1,
"bound_spec": None,
"conv_width": 0,
"block": 1,
"converts": True,
}
probe = {
"entries": [entry],
"quantized": True,
"whole_prompt": False,
"layers": 1,
"widths": (1, 1, 1),
}
monkeypatch.setattr(mm, "_probe", lambda *a, **kw: probe)
for vision in (True, False):
monkeypatch.setattr(mm, "_loads_as_vision", lambda _c, v = vision: v)
assert mm._cache_plan({}, None, 4)[1] == 0
probe["quantized"] = False
assert mm._cache_plan({}, None, 4)[1] is None
def test_a_host_without_the_loader_prices_the_fallback(self, monkeypatch):
import builtins
real = builtins.__import__
def _no_loader(name, *a, **kw):
if name == "core.inference.mlx_inference":
raise ImportError("no loader here")
return real(name, *a, **kw)
monkeypatch.setattr(builtins, "__import__", _no_loader)
assert mm._generation_settings({"model_type": "llama"}) == (mm.MLX_PREFILL_CHUNK, 64)
@_NEEDS_MLX
@pytest.mark.parametrize("reported, compute", [(None, 1_035_321_999), (512, 903_201_423)])
def test_the_estimate_prices_the_chunk_the_loader_reports(monkeypatch, reported, compute):
_on_bfloat16_chip()
if reported is not None:
from core.inference import mlx_inference as mi
monkeypatch.setattr(mi, "mlx_prefill_chunk", lambda **kw: reported)
breakdown = mm.mlx_memory_breakdown(
_local_snapshot("unsloth/Qwen3-4B-Thinking-2507"),
n_ctx = 32768,
load_in_4bit = True,
kv_bits = 4,
)
assert breakdown is not None and breakdown.compute_bytes == compute
override = mm.mlx_memory_breakdown(
_local_snapshot("unsloth/Qwen3-4B-Thinking-2507"),
n_ctx = 32768,
load_in_4bit = True,
kv_bits = 4,
prefill_chunk = 512,
)
assert override.compute_bytes == 903_201_423
@_NEEDS_MLX
def test_a_model_mlx_vlm_would_diffuse_is_refused_rather_than_priced():
snapshot = _local_snapshot("mlx-community/diffusiongemma-26B-A4B-it-4bit")
assert mm._routes_to_diffusion(mm._snapshot_config(snapshot)) is True
assert mm.mlx_memory_breakdown(snapshot, n_ctx = 32768, load_in_4bit = True) is None
@_NEEDS_MLX
def test_an_ordinary_vision_model_is_not_mistaken_for_a_diffusion_one():
config = {"model_type": "kimi_vl", "vision_config": {"depth": 2}}
assert mm._routes_to_diffusion(config) is False
assert mm._generation_settings(config)[0] > 0
@_NEEDS_MLX
class TestDiffusionRouting:
"""The markers gate a build; mlx-vlm's own predicate is the verdict."""
@staticmethod
def _resolving(
monkeypatch,
*,
canvas,
mask,
build = lambda config: object(),
):
monkeypatch.setattr(
mm,
"_loader_config",
lambda arch, config: SimpleNamespace(canvas_length = canvas, mask_token_id = mask),
)
monkeypatch.setattr(
"mlx_vlm.utils.get_model_and_args",
lambda config: (SimpleNamespace(Model = build), None),
)
@pytest.mark.parametrize("verdict", [True, False])
def test_the_predicate_outranks_the_marker(self, monkeypatch, verdict):
from mlx_vlm.generate import diffusion as vlm_diffusion
self._resolving(monkeypatch, canvas = None, mask = 100)
monkeypatch.setattr(vlm_diffusion, "is_diffusion_model", lambda model, kw: verdict)
assert mm._routes_to_diffusion({"model_type": "whatever"}) is verdict
def test_a_diverted_load_refuses_without_needing_a_checkpoint(self, monkeypatch):
from mlx_vlm.generate import diffusion as vlm_diffusion
self._resolving(monkeypatch, canvas = 512, mask = None)
monkeypatch.setattr(vlm_diffusion, "is_diffusion_model", lambda model, kw: True)
monkeypatch.setattr(mm, "_loads_as_vision", lambda config: True)
with pytest.raises(ValueError, match = "diffusion generator"):
mm._generation_settings({"model_type": "whatever"})
def test_an_architecture_carrying_no_marker_is_never_built(self, monkeypatch):
def _explode(config):
raise AssertionError("built a wrapper for a config carrying no marker")
self._resolving(monkeypatch, canvas = None, mask = None, build = _explode)
assert mm._routes_to_diffusion({"model_type": "whatever"}) is False
def test_a_marked_model_that_cannot_be_placed_is_refused_not_assumed(self, monkeypatch):
def _explode(config):
raise RuntimeError("this wrapper cannot be built")
self._resolving(monkeypatch, canvas = 512, mask = None, build = _explode)
with pytest.raises(RuntimeError):
mm._routes_to_diffusion({"model_type": "whatever"})
with pytest.raises(RuntimeError):
mm._generation_settings({"model_type": "whatever", "vision_config": {"depth": 2}})
@pytest.mark.skipif(not _HAVE_MLX, reason = "drives real cache classes")
def test_the_peak_of_a_bounded_cache_is_measured_not_derived():
from mlx_lm.models import cache as C
for bound, attribute, value, n_ctx, chunk, expected in (
(C.RotatingKVCache, "max_size", 512, 100, 2048, 355),
(C.RotatingKVCache, "max_size", 512, 255, 2048, 510),
(C.RotatingKVCache, "max_size", 512, 1024, 2048, 1023),
(C.ChunkedKVCache, "chunk_size", 512, 4096, 2048, 2560),
(C.ChunkedKVCache, "chunk_size", 512, 2560, 2048, 2048),
(C.RotatingKVCache, "max_size", 512, 256, 1, 512),
(C.RotatingKVCache, "max_size", 512, 10**12, 2048, 2559),
):
assert (
mm._bounded_peak(bound, attribute, value, n_ctx, chunk) == expected
), f"{bound.__name__}({value}) at {n_ctx}"
assert mm._bound_spec(type("E", (), {"max_size": 512})())
assert mm._bound_spec(type("E", (), {"chunk_size": 512})())
spec = (C.RotatingKVCache, "max_size", 512)
entry = {"slope": 1.0, "block": mm.MLX_KV_BLOCK, "bound_spec": spec}
for n_ctx, charged in ((253, 508), (100, 355), (1024, 1023), (10**12, 2559)):
assert mm._held_tokens(entry, n_ctx, 2048) == charged
@pytest.mark.skipif(not _HAVE_MLX, reason = "builds a real architecture")
class TestPricingALoad:
_CONFIG = {
"model_type": "llama",
"hidden_size": 256,
"num_hidden_layers": 2,
"intermediate_size": 512,
"num_attention_heads": 4,
"num_key_value_heads": 4,
"rms_norm_eps": 1e-5,
"vocab_size": 512,
"rope_theta": 10000.0,
}
def _checkpoint(
self,
tmp_path,
*,
prefix = "",
):
import mlx.core as mx
from mlx.utils import tree_flatten
config = dict(self._CONFIG, tie_word_embeddings = True)
(tmp_path / "config.json").write_text(json.dumps(config))
built = mm._whole_model(config, mx.bfloat16)
mx.save_safetensors(
str(tmp_path / "model.safetensors"),
{
prefix + name: mx.zeros(value.shape, dtype = mx.bfloat16)
for name, value in tree_flatten(built.parameters())
},
)
return config
def test_a_float32_checkpoint_goes_resident_at_the_width_the_chip_gives_it(self):
if mm._runtime_dtype().size != 2:
pytest.skip("measured on a chip the loader gives a 16-bit width")
snapshot = _local_snapshot("hf-internal-testing/tiny-random-LlamaForCausalLM")
config = json.loads((Path(snapshot) / "config.json").read_text())
assert mm.mlx_weight_bytes(snapshot, config, load_in_4bit = False) == 2_064_544
def test_an_offloaded_load_prices_what_the_offload_holds(self, tmp_path, monkeypatch):
self._checkpoint(tmp_path)
resident = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 256)
monkeypatch.setattr(
mm, "_offloaded", lambda model_dir, config, offload: (-1000 * bool(offload), offload)
)
offloaded = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 256, offload = (None, 0.5))
assert offloaded.weights_bytes == resident.weights_bytes - 1000
assert offloaded.total_bytes == resident.total_bytes - 1000
assert (offloaded.ple_offload, offloaded.expert_offload) == (None, 0.5)
monkeypatch.setattr(
mm, "mlx_offload_planner", lambda: SimpleNamespace(SAFETY_MARGIN = (2 << 30, 0.03))
)
price = mm.mlx_memory_breakdown(str(tmp_path), n_ctx = 8192, offload = (None, 0.5)).total_bytes
fit = lambda offload: mm.mlx_fit_outcome(
str(tmp_path), budget_bytes = price + (2 << 30), max_ctx = 1 << 20, offload = offload
)
assert fit((None, 0.5)) == ("fitted", 8192) != fit(None)
room = lambda offload: mm.mlx_headroom(
str(tmp_path), budget_bytes = price + (2 << 30) + 5, n_ctx = 8192, offload = offload
)
assert room((None, 0.5)) == 5 # the room keeps the same margin free as the fit
def test_a_quantizing_load_is_priced_below_the_shards(self, tmp_path):
config = self._checkpoint(tmp_path)
wide = mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = False)
quantized = mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = True)
from unsloth_zoo.mlx.loader import _resolve_mlx_quantization_spec
embeddings_quantized = (
"quantize_embeddings" in inspect.signature(_resolve_mlx_quantization_spec).parameters
)
assert (wide, quantized) == (2_886_144, 813_568 if embeddings_quantized else 1_001_984)
assert quantized < wide < mm._shard_bytes(str(tmp_path), config) == 2_888_265
def test_engram_mlx_vlm_reads_from_disk_is_not_priced(self, tmp_path, monkeypatch):
import mlx.core as mx
import mlx.nn as nn
from mlx.utils import tree_flatten
def build(*_):
model = nn.Module()
model.layer = nn.Module()
model.layer.engram, model.layer.gate = nn.Module(), nn.Linear(128, 64, bias = False)
model.layer.engram.embed = nn.Embedding(64, 128)
return model
monkeypatch.setattr(mm, "_whole_model", build)
path = str(tmp_path / "model.safetensors")
mx.save_safetensors(path, dict(tree_flatten(build().parameters())))
price = lambda q = False, **change: mm.mlx_weight_bytes(
str(tmp_path), dict(config, **change), q
)
config = {"model_type": "deepseek_v41"}
# Quantized at load, the table becomes a resident one.
held = 64 * 128 * mm._runtime_dtype().size
assert price(quantization = {"bits": 2}) == os.path.getsize(path) - 64 * 128 * 4
assert (price(), price(model_type = "deepseek_v4")) == (held, 2 * held)
assert price(True) == price(True, model_type = "deepseek_v4")
assert price(model_type = "deepseek_v4", text_config = {"model_type": "deepseek_v41"}) == held
monkeypatch.setattr(mm, "_whole_model", lambda *_: 1 / 0)
assert (price(), price(True)) == (
os.path.getsize(path) - 64 * 128 * 4,
os.path.getsize(path),
)
def test_shards_that_name_nothing_the_architecture_has_are_not_priced_from_it(self, tmp_path):
config = self._checkpoint(tmp_path, prefix = "draft.")
with pytest.raises(ValueError, match = "does not supply"):
mm._resident_bytes(str(tmp_path), config, quantize = True)
for flag in (True, False):
assert mm.mlx_weight_bytes(str(tmp_path), config, load_in_4bit = flag) == (
mm._shard_bytes(str(tmp_path), config)
)
@pytest.mark.skipif(not _HAVE_MLX, reason = "asks the installed loader")
class TestALoadWithNoFootprintToQuote:
def test_quantization_metadata_no_load_can_honour_is_refused_not_priced(self, tmp_path):
per_module = {"model.layers.0.self_attn.q_proj": {"bits": 4, "group_size": 64}}
for spelling in ("quantization", "quantization_config"):
stated = {"model_type": "llama", spelling: per_module}
assert mm._load_is_refused(str(tmp_path), stated, True)
assert mm._load_is_refused(str(tmp_path), stated, False) is None
bnb = {"model_type": "llama", spelling: {"quant_method": "bitsandbytes"}}
for flag in (True, False):
assert "bitsandbytes" in (mm._load_is_refused(str(tmp_path), bnb, flag) or "")
@pytest.mark.parametrize(
"repo, names",
[
("OsaurusAI/ZAYA1-VL-8B-MXFP4", "no home for"),
("Qwen/Qwen3-Embedding-0.6B", "QK-norm"),
("z-lab/Qwen3.6-27B-DFlash", "QK-norm"),
("z-lab/gemma-4-31B-it-DFlash", "QK-norm"),
("yuyijiong/Qwen3.5-4B-Eagle3", "no home for"),
],
)
def test_a_checkpoint_no_load_starts_from_is_refused(self, repo, names):
snapshot = _local_snapshot(repo)
config = mm._snapshot_config(snapshot)
for flag in (True, False):
assert names in (mm._load_is_refused(snapshot, config, flag) or "")
assert mm.mlx_memory_breakdown(snapshot, n_ctx = 2048, load_in_4bit = flag) is None
def test_no_checkpoint_on_this_disk_is_refused_for_an_invented_reason(self):
hub = os.path.expanduser("~/.cache/huggingface/hub")
seen = accepted = 0
for path in glob.glob(os.path.join(hub, "models--*", "snapshots", "*", "config.json")):
directory = os.path.dirname(path)
config = mm._snapshot_config(directory)
if not isinstance(config, dict) or not config.get("model_type"):
continue
try:
if not mm.mlx_shard_files(directory, config):
continue
except Exception:
continue
seen += 1
refusals = [mm._load_is_refused(directory, config, flag) for flag in (True, False)]
for refusal in refusals:
assert refusal is None or any(
reason in refusal for reason in ("QK-norm", "no home for", "bitsandbytes")
), f"{path}: {refusal}"
accepted += refusals[0] is None
if not seen:
pytest.skip("no checkpoints on this disk")
assert accepted > seen * 0.9, "the gate has started refusing wholesale"
def test_extras_a_registered_rule_filters_are_not_refused(self):
snapshot = _local_snapshot("google/gemma-4-E2B")
config = mm._snapshot_config(snapshot)
assert mm._load_is_refused(snapshot, config, True) is None
assert mm.mlx_weight_bytes(snapshot, config, load_in_4bit = True) == 7_510_668_230
_FILTERED = [
"language_model.model.per_layer_model_projection.biases",
"language_model.model.per_layer_model_projection.scales",
]
_GEMMA4 = {"model_type": "gemma4", "vision_config": {"model_type": "gemma4"}}
@pytest.mark.parametrize(
"config, extras, names",
[
(_GEMMA4, _FILTERED, None),
(_GEMMA4, _FILTERED + ["mystery.weight"], "mystery.weight"),
({"model_type": "an-architecture-with-no-rule"}, ["d2t", "fc.weight"], "d2t"),
],
)
def test_a_rule_exempts_only_the_keys_it_names(self, config, extras, names):
refusal = mm._extra_tensors_refused(object(), config, extras)
assert refusal is None if names is None else (names in refusal and "no home for" in refusal)
class TestShardsTheLoaderReads:
"""Which safetensors beside a config actually become weights."""
_VISION = {"model_type": "llava", "vision_config": {"hidden_size": 8}}
def _spread(self, tmp_path, config):
for name in (
"model-00001.safetensors",
"adapter_model.safetensors",
"consolidated.safetensors",
):
(tmp_path / name).write_bytes(b"x" * 100)
return sorted(os.path.basename(p) for p in mm.mlx_shard_files(str(tmp_path), config))
def test_the_text_loader_reads_model_shards_only(self, tmp_path):
assert self._spread(tmp_path, {"model_type": "llama"}) == ["model-00001.safetensors"]
adapter_only = tmp_path / "lora"
adapter_only.mkdir()
(adapter_only / "adapter_model.safetensors").write_bytes(b"x" * 100)
assert mm.mlx_shard_files(str(adapter_only), {"model_type": "llama"}) == []
def test_the_vision_loader_reads_the_index_before_the_directory(self, tmp_path):
assert self._spread(tmp_path, self._VISION) == [
"adapter_model.safetensors",
"model-00001.safetensors",
]
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"a.weight": "model-00001.safetensors"}})
)
assert self._spread(tmp_path, self._VISION) == ["model-00001.safetensors"]
@_NEEDS_MLX
@pytest.mark.parametrize("declared", [0, 1_000_000, 1 << 40])
def test_a_shard_is_not_read_past_the_header_it_declares(self, tmp_path, declared):
import mlx.core as mx
shard = tmp_path / "model.safetensors"
body = json.dumps({"a.weight": {"dtype": "F16", "shape": [2], "data_offsets": [0, 4]}})
shard.write_bytes(declared.to_bytes(8, "little") + body.encode() + b"\0" * 4)
with pytest.raises(ValueError, match = "safetensors header"):
mm._checkpoint_tensors(str(tmp_path), {"model_type": "llama"}, mx.bfloat16)
shard.write_bytes(len(body).to_bytes(8, "little") + body.encode() + b"\0" * 4)
assert list(mm._checkpoint_tensors(str(tmp_path), {"model_type": "llama"}, mx.bfloat16))
@_NEEDS_MLX
def test_a_header_is_capped_below_the_shard_it_sits_in(self, tmp_path, monkeypatch):
import mlx.core as mx
body = json.dumps({"a.weight": {"dtype": "F16", "shape": [2], "data_offsets": [0, 4]}})
(tmp_path / "model.safetensors").write_bytes(
len(body).to_bytes(8, "little") + body.encode() + b"\0" * 4
)
read = lambda: mm._checkpoint_tensors(str(tmp_path), {"model_type": "llama"}, mx.bfloat16)
assert list(read())
monkeypatch.setattr(mm, "_MAX_SAFETENSORS_HEADER", len(body) - 1)
with pytest.raises(ValueError, match = "safetensors header"):
read()
@pytest.mark.parametrize(
"shard",
[
"../secret.safetensors",
"a/../../secret.safetensors",
"/etc/passwd",
"..\\secret.safetensors",
"C:/secret.safetensors",
],
)
def test_an_index_naming_a_shard_outside_the_snapshot_is_refused(self, tmp_path, shard):
(tmp_path / "secret.safetensors").write_bytes(b"x" * 100)
snapshot = tmp_path / "snap"
snapshot.mkdir()
(snapshot / "model-00001.safetensors").write_bytes(b"x" * 100)
(snapshot / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"a.weight": "model-00001.safetensors", "b.weight": shard}})
)
with pytest.raises(ValueError, match = "outside the snapshot"):
mm.mlx_shard_files(str(snapshot), self._VISION)
def test_a_nested_shard_inside_the_snapshot_is_still_read(self, tmp_path):
(tmp_path / "sub").mkdir()
(tmp_path / "sub" / "model-00001.safetensors").write_bytes(b"x" * 100)
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"a.weight": "sub/./model-00001.safetensors"}})
)
assert [os.path.basename(p) for p in mm.mlx_shard_files(str(tmp_path), self._VISION)] == [
"model-00001.safetensors"
]
@pytest.mark.parametrize(
"weight_map", [["model-00001.safetensors"], "model-00001.safetensors", 7, True, None]
)
def test_an_index_whose_weight_map_is_not_a_mapping_is_fatal(self, tmp_path, weight_map):
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": weight_map})
)
with pytest.raises(ValueError, match = "no weight map"):
mm.mlx_shard_files(str(tmp_path), self._VISION)
# mlx 0.32.4 leaves head_dim 256 prefill unfused below NAX, and tails under 1024 queries on NAX.
_UNFUSED_HEAD_DIM_256_SCORES = {
"unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit": {True: 1_298_923_560, False: 2_682_961_960}
}
@_NEEDS_MLX
@pytest.mark.parametrize(
"repo, n_ctx, kv_bits, weights, kv, compute, layers, caption",
[
(
"unsloth/Qwen3-4B-Thinking-2507",
32768,
None,
2_262_920_192,
4_869_586_944,
863_355_535,
36,
"bf16",
),
(
"unsloth/Qwen3-4B-Thinking-2507",
32768,
4,
2_262_920_192,
1_369_571_328,
1_035_321_999,
36,
"4-bit",
),
(
"unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit",
32768,
None,
21_634_993_114,
740_720_640,
943_440_527,
40,
"bf16",
),
("unsloth/gemma-3-270m-it", 8192, None, 150_885_632, 65_258_496, 725_729_935, 18, "bf16"),
(
"mlx-community/deepseek-vl2-tiny-4bit",
4096,
None,
2_099_712_075,
267_386_880,
803_717_775,
12,
"bf16",
),
(
"Qwen/Qwen2.5-VL-3B-Instruct",
32768,
4,
3_073_556_480,
342_392_832,
753_582_735,
36,
"4-bit",
),
(
"Qwen/Qwen2.5-VL-3B-Instruct",
5001,
4,
3_073_556_480,
56_852_480,
717_152_431,
36,
"4-bit",
),
(
"Qwen/Qwen2.5-VL-3B-Instruct",
5000,
4,
3_073_556_480,
56_852_480,
717_151_119,
36,
"4-bit",
),
(
"mlx-community/gemma-3n-E2B-it-4bit",
4096,
None,
4_462_976_625,
60_784_640,
705_544_847,
30,
"bf16",
),
(
"mlx-community/Molmo-7B-D-0924-4bit",
4096,
None,
5_298_715_127,
249_561_088,
1_282_888_719,
28,
"bf16",
),
(
"hf-internal-testing/tiny-random-LlamaForCausalLM",
512,
None,
2_061_600,
98_304,
690_702_180,
2,
"bf16",
),
],
)
def test_a_real_checkpoint_is_priced_to_the_byte(
repo, n_ctx, kv_bits, weights, kv, compute, layers, caption
):
_on_bfloat16_chip()
breakdown = mm.mlx_memory_breakdown(
_local_snapshot(repo), n_ctx = n_ctx, load_in_4bit = True, kv_bits = kv_bits
)
assert breakdown is not None
assert (breakdown.weights_bytes, breakdown.kv_bytes) == (weights, kv)
from unsloth_zoo.mlx.nax import nax_available
unfused = _UNFUSED_HEAD_DIM_256_SCORES.get(repo, {}).get(nax_available(), 0)
assert breakdown.compute_bytes == compute + unfused
assert breakdown.layer_count == layers
assert breakdown.cache_type_kv == caption
@_NEEDS_MLX
@pytest.mark.parametrize("kv_bits, caption", [(6, "6-bit"), (8, "8-bit")])
def test_the_caption_names_a_width_the_real_conversion_reaches(kv_bits, caption):
_on_bfloat16_chip()
breakdown = mm.mlx_memory_breakdown(
_local_snapshot("unsloth/Qwen3-4B-Thinking-2507"),
n_ctx = 32768,
load_in_4bit = True,
kv_bits = kv_bits,
)
assert breakdown is not None and breakdown.cache_type_kv == caption
class TestTheContextSearch:
"""The search itself, priced by a stub so it runs where MLX and the checkpoints do not."""
@staticmethod
def _linear(monkeypatch, unpriceable = ()):
"""A byte per token, so every expected answer below is readable off the budget."""
def _priced(sizing, n_ctx):
return None if n_ctx in unpriceable else SimpleNamespace(total_bytes = n_ctx)
monkeypatch.setattr(
mm, "_size_load", lambda *a, **kw: SimpleNamespace(offload = (None, None))
)
monkeypatch.setattr(mm, "_priced_at", _priced)
def test_the_answer_is_the_largest_whole_block_under_budget(self, monkeypatch):
self._linear(monkeypatch)
assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) == 6912
def test_a_floor_between_blocks_rounds_up_rather_than_under_the_minimum(self, monkeypatch):
self._linear(monkeypatch)
assert mm.mlx_fit_context("x", budget_bytes = 4200, max_ctx = 8192, min_ctx = 4100) is None
assert mm.mlx_fit_context("x", budget_bytes = 5000, max_ctx = 8192, min_ctx = 4100) == 4864
def test_a_context_that_cannot_be_priced_abandons_the_fit(self, monkeypatch):
self._linear(monkeypatch, unpriceable = (6144,))
assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) is None
def test_a_ceiling_that_already_fits_is_left_alone(self, monkeypatch):
self._linear(monkeypatch)
assert mm.mlx_fit_context("x", budget_bytes = 9000, max_ctx = 8192) is None
def test_a_load_that_cannot_be_sized_is_not_fitted(self, monkeypatch):
monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: None)
assert mm.mlx_fit_context("x", budget_bytes = 7000, max_ctx = 8192) is None
def test_the_outcome_says_why_no_context_came_back(self, monkeypatch):
self._linear(monkeypatch)
assert mm.mlx_fit_outcome("x", budget_bytes = 9000, max_ctx = 8192) == ("fits", None)
assert mm.mlx_fit_outcome("x", budget_bytes = 7000, max_ctx = 8192) == ("fitted", 6912)
assert mm.mlx_fit_outcome("x", budget_bytes = 4200, max_ctx = 8192, min_ctx = 4100) == (
"no_fit",
None,
)
assert mm.mlx_headroom("x", budget_bytes = 9000, n_ctx = 8192) == 808
assert mm.mlx_headroom("x", budget_bytes = 9000, n_ctx = 100) == 9000 - 256
assert mm.mlx_headroom("x", budget_bytes = 7000, n_ctx = 8192) == 0
monkeypatch.setattr(mm, "_size_load", lambda *a, **kw: None)
assert mm.mlx_fit_outcome("x", budget_bytes = 7000, max_ctx = 8192) == ("unsizable", None)
assert mm.mlx_headroom("x", budget_bytes = 7000, n_ctx = 8192) is None
@_NEEDS_MLX
@pytest.mark.parametrize("budget_gib, fitted", [(6, 18_432), (12, 61_952), (24, 149_504)])
def test_a_real_checkpoint_is_fitted_to_the_byte(budget_gib, fitted):
_on_bfloat16_chip()
snapshot = _local_snapshot("unsloth/Qwen3-4B-Thinking-2507")
budget = budget_gib * 1024**3
assert (
mm.mlx_fit_context(snapshot, budget_bytes = budget, max_ctx = 262_144, load_in_4bit = True)
== fitted
)
priced = mm.mlx_memory_breakdown(snapshot, n_ctx = fitted, load_in_4bit = True)
over = mm.mlx_memory_breakdown(snapshot, n_ctx = fitted + mm.MLX_KV_BLOCK, load_in_4bit = True)
assert priced.total_bytes <= budget < over.total_bytes
assert mm.mlx_memory_breakdown(snapshot, n_ctx = float("nan"), load_in_4bit = True) is None
@_NEEDS_MLX
def test_a_companion_drafter_adds_its_stored_weights_and_a_cache_to_the_load(monkeypatch):
_on_bfloat16_chip()
# unsloth-zoo otherwise loads a bf16 DFlash drafter at 4 bits, and sizes it so.
monkeypatch.setenv("UNSLOTH_MLX_DRAFTER_BITS", "0")
target = _local_snapshot("mlx-community/Qwen3.5-4B-4bit")
drafter = _local_snapshot("z-lab/Qwen3.5-4B-DFlash")
stored = 0
for shard in glob.glob(os.path.join(drafter, "*.safetensors")):
with open(shard, "rb") as handle:
header = json.loads(handle.read(int.from_bytes(handle.read(8), "little")))
header.pop("__metadata__", None)
stored += sum(end - start for start, end in (t["data_offsets"] for t in header.values()))
alone = mm.mlx_memory_breakdown(target, n_ctx = 8192, vision = True)
paired = mm.mlx_memory_breakdown(target, n_ctx = 8192, vision = True, drafter = (drafter, False))
assert paired.weights_bytes - alone.weights_bytes == stored
assert paired.kv_bytes > alone.kv_bytes and paired.compute_bytes > alone.compute_bytes
@_NEEDS_MLX
@pytest.mark.parametrize(
"repo, kv_bits, whole_prompt",
[
("unsloth/Qwen3-4B-Thinking-2507", 4, False),
("unsloth/Qwen3-4B-Thinking-2507", None, True),
("unsloth/gemma-3-270m-it", None, False),
("unsloth/Qwen3.6-35B-A3B-UD-MLX-4bit", None, False),
],
)
def test_the_total_never_falls_as_the_context_grows(repo, kv_bits, whole_prompt):
sizing = mm._size_load(_local_snapshot(repo), kv_bits, None, True)
if whole_prompt:
forced = replace(sizing, facts = {**sizing.facts, "whole_prompt": True})
assert mm._priced_at(forced, 32_768).total_bytes > mm._priced_at(sizing, 32_768).total_bytes
sizing = forced
steps = range(mm.MLX_KV_BLOCK, 262_145, mm.MLX_KV_BLOCK)
totals = [mm._priced_at(sizing, n).total_bytes for n in steps]
assert totals == sorted(totals)
def test_the_drafter_picker_lists_cached_drafters_of_a_cached_target(tmp_path, monkeypatch):
from core.inference import mlx_speculative
(tmp_path / "config.json").write_text('{"vocab_size": 7}')
found = [
("o/T-DFlash", SimpleNamespace(kind = "dflash"), True),
("o/T-MTP", SimpleNamespace(kind = "mtp"), False),
]
monkeypatch.setattr(
mlx_speculative,
"cached_drafters",
lambda name, config: found if (name, config) == ("o/T", {"vocab_size": 7}) else [],
)
monkeypatch.setattr(
"utils.utils.hf_cache_snapshot_dir", lambda name: tmp_path if name == "o/T" else None
)
assert ri._mlx_cached_drafters("o/T") == [
{"repo_id": "o/T-DFlash", "kind": "dflash", "named": True},
{"repo_id": "o/T-MTP", "kind": "mtp", "named": False},
]
assert ri._mlx_cached_drafters("o/absent") == []
monkeypatch.setattr(ri.account_access, "filter_model_rows", lambda rows: rows[1:])
assert [row["repo_id"] for row in ri._mlx_cached_drafters("o/T")] == ["o/T-MTP"]