Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Auto ctx falls back to 8192 when no GPU subset fits the weights; explicit ctx is never shrunk."""
from __future__ import annotations
import sys
import types as _types
from pathlib import Path
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
_structlog_stub = _types.ModuleType("structlog")
sys.modules.setdefault("structlog", _structlog_stub)
_httpx_stub = _types.ModuleType("httpx")
for _exc_name in (
"ConnectError",
"TimeoutException",
"ReadTimeout",
"ReadError",
"RemoteProtocolError",
"CloseError",
):
setattr(_httpx_stub, _exc_name, type(_exc_name, (Exception,), {}))
class _FakeTimeout:
def __init__(self, *a, **kw):
pass
_httpx_stub.Timeout = _FakeTimeout
_httpx_stub.Client = type(
"Client",
(),
{
"__init__": lambda self, **kw: None,
"__enter__": lambda self: self,
"__exit__": lambda self, *a: None,
},
)
# only when httpx is absent: a stub without Response breaks starlette.testclient later
try:
import httpx # noqa: F401
except ImportError:
sys.modules.setdefault("httpx", _httpx_stub)
from core.inference.llama_cpp import (
_AUTO_OFFLOAD_CTX,
_APPLE_UNIFIED_MEMORY_FRACTION,
_APPLE_WIRED_CEILING_FRACTION,
_CTX_FIT_VRAM_FRACTION,
LlamaCppBackend,
classify_gpu_offload_lines,
parse_gpu_offload_counts,
)
from core.inference.llama_server_args import parse_ctx_override, resolve_requested_ctx
GIB = 1024**3
FIT_MIN_CTX = 4096
def _make_backend(
native_ctx = 131072,
n_layers = 80,
n_kv_heads = 8,
n_heads = 64,
kv_key_length = 128,
kv_value_length = 128,
):
"""LlamaCppBackend with GGUF metadata set and decision helpers stubbed."""
inst = LlamaCppBackend.__new__(LlamaCppBackend)
inst._context_length = native_ctx
inst._n_layers = n_layers
inst._n_kv_heads = n_kv_heads
inst._n_heads = n_heads
inst._embedding_length = 8192
inst._kv_key_length = kv_key_length
inst._kv_value_length = kv_value_length
inst._kv_lora_rank = None
inst._sliding_window = None
inst._sliding_window_pattern = None
inst._ssm_inner_size = None
inst._full_attention_interval = None
inst._key_length_mla = None
inst._n_kv_heads_by_layer = None
inst._kv_key_length_swa = None
inst._kv_value_length_swa = None
return inst
def _drive(
n_ctx,
model_gib,
gpus,
native_ctx = 131072,
kv_per_token_bytes = 325_000,
can_estimate_kv = True,
extra_args = None,
apple_budget_mib = 0,
flat_mtp_reserve = 0.0,
):
"""Mirrors load_model's post-metadata fit with stubs, so the built command can be asserted."""
inst = _make_backend(native_ctx = native_ctx)
model_size = int(model_gib * GIB)
cache_type_kv = None
def fake_estimate(
n_ctx_,
_type = None,
**_kwargs,
):
return 0 if n_ctx_ <= 0 else n_ctx_ * kv_per_token_bytes
inst._estimate_kv_cache_bytes = fake_estimate
inst._can_estimate_kv = lambda: can_estimate_kv
context_length = inst._context_length
ctx_override = parse_ctx_override(extra_args)
requested_ctx = resolve_requested_ctx(extra_args, n_ctx)
effective_ctx = requested_ctx if requested_ctx > 0 else (context_length or 0)
max_available_ctx = context_length or effective_ctx
if requested_ctx > 0:
effective_ctx = requested_ctx
elif context_length is not None:
effective_ctx = context_length
else:
effective_ctx = 0
original_ctx = effective_ctx
max_available_ctx = context_length or effective_ctx
gpu_indices, use_fit = None, True
explicit_ctx = requested_ctx > 0
if gpus and inst._can_estimate_kv() and effective_ctx > 0:
native_ctx_for_cap = context_length or effective_ctx
if native_ctx_for_cap > 0:
ranked_for_cap = sorted(gpus, key = lambda g: g[1], reverse = True)
best_cap = 0
for n_gpus in range(1, len(ranked_for_cap) + 1):
subset = ranked_for_cap[:n_gpus]
pool_mib = sum(free for _, free in subset)
capped = inst._fit_context_to_vram(
native_ctx_for_cap,
pool_mib,
model_size,
cache_type_kv,
)
kv = inst._estimate_kv_cache_bytes(capped, cache_type_kv)
total_mib = (model_size + kv) / (1024 * 1024)
if total_mib <= pool_mib * _CTX_FIT_VRAM_FRACTION:
best_cap = max(best_cap, capped)
if best_cap > 0:
max_available_ctx = best_cap
if explicit_ctx:
requested_total = model_size + inst._estimate_kv_cache_bytes(
effective_ctx, cache_type_kv
)
gpu_indices, use_fit = inst._select_gpus(requested_total, gpus)
else:
ranked = sorted(gpus, key = lambda g: g[1], reverse = True)
matched = False
pin_fraction = LlamaCppBackend._GPU_PIN_VRAM_FRACTION
for n_gpus in range(1, len(ranked) + 1):
subset = ranked[:n_gpus]
pool_mib = sum(free for _, free in subset)
capped = inst._fit_context_to_vram(
effective_ctx,
pool_mib,
model_size,
cache_type_kv,
)
kv = inst._estimate_kv_cache_bytes(capped, cache_type_kv)
total_mib = (model_size + kv) / (1024 * 1024)
if total_mib <= pool_mib * pin_fraction:
effective_ctx = capped
gpu_indices = sorted(idx for idx, _ in subset)
use_fit = False
matched = True
break
if not matched:
effective_ctx = min(_AUTO_OFFLOAD_CTX, effective_ctx)
if effective_ctx > 0:
for n_gpus in range(1, len(ranked) + 1):
subset = ranked[:n_gpus]
pool_mib = sum(free for _, free in subset)
kv = inst._estimate_kv_cache_bytes(effective_ctx, cache_type_kv)
total_mib = (model_size + kv) / (1024 * 1024)
if total_mib <= pool_mib * pin_fraction:
gpu_indices = sorted(idx for idx, _ in subset)
use_fit = False
break
elif gpus:
gpu_indices, use_fit = inst._select_gpus(model_size, gpus)
if use_fit and not explicit_ctx:
effective_ctx = (
min(_AUTO_OFFLOAD_CTX, effective_ctx) if effective_ctx > 0 else _AUTO_OFFLOAD_CTX
)
elif apple_budget_mib > 0 and effective_ctx > 0:
native_ctx_for_cap = context_length or effective_ctx
apple_fit_budget_mib = int(apple_budget_mib * max(0.0, 1.0 - flat_mtp_reserve))
if inst._can_estimate_kv():
cap = inst._fit_context_to_vram(
native_ctx_for_cap,
apple_fit_budget_mib,
model_size,
cache_type_kv,
budget_frac = 1.0,
)
cap_footprint_mib = (model_size + inst._estimate_kv_cache_bytes(cap, cache_type_kv)) / (
1024 * 1024
)
max_available_ctx = (
cap
if cap_footprint_mib <= apple_fit_budget_mib
else min(FIT_MIN_CTX, native_ctx_for_cap)
)
else:
max_available_ctx = min(FIT_MIN_CTX, native_ctx_for_cap)
if not explicit_ctx:
effective_ctx = max_available_ctx
return {
"c_arg": effective_ctx if effective_ctx > 0 else 0,
"use_fit": use_fit,
"gpu_indices": gpu_indices,
"max_available_ctx": max_available_ctx,
"original_ctx": original_ctx,
"ctx_override": ctx_override,
}
def _drive_native(
*args,
n_ctx = 0,
native_ctx = 131072,
**kwargs,
):
"""_drive with n_ctx = 0, which is Auto context: the branch that caps to native_ctx."""
return _drive(*args, n_ctx = n_ctx, native_ctx = native_ctx, **kwargs)
class TestAutoModeWeightsExceedVRAM:
"""``n_ctx == 0`` on a model whose weights don't fit anywhere."""
def test_minimax_like_single_gpu(self):
plan = _drive_native(model_gib = 131, gpus = [(0, 97_000)], native_ctx = 196608)
assert plan["c_arg"] == _AUTO_OFFLOAD_CTX
assert plan["use_fit"] is True
assert plan["gpu_indices"] is None
assert plan["max_available_ctx"] == 196608
def test_multi_gpu_all_subsets_fail(self):
plan = _drive_native(
model_gib = 400,
gpus = [(0, 80_000), (1, 80_000), (2, 80_000), (3, 80_000)],
)
assert plan["c_arg"] == _AUTO_OFFLOAD_CTX
assert plan["use_fit"] is True
assert plan["gpu_indices"] is None
def test_no_kv_metadata_auto(self):
"""File-size-only fallback path uses the same 8192 default."""
plan = _drive_native(
model_gib = 131,
gpus = [(0, 97_000)],
native_ctx = 196608,
can_estimate_kv = False,
)
assert plan["c_arg"] == _AUTO_OFFLOAD_CTX
assert plan["use_fit"] is True
class TestExplicitCtxRespectsUser:
"""``n_ctx > 0`` must never be silently shrunk."""
def test_fittable_weights_oversized_kv(self):
plan = _drive_native(n_ctx = 131072, model_gib = 8, gpus = [(0, 24_000)])
assert plan["c_arg"] == 131072
assert plan["use_fit"] is True
assert plan["gpu_indices"] is None
def test_explicit_that_fits_uses_ngl(self):
plan = _drive_native(n_ctx = 8192, model_gib = 8, gpus = [(0, 24_000)])
assert plan["c_arg"] == 8192
assert plan["use_fit"] is False
assert plan["gpu_indices"] == [0]
def test_explicit_on_weights_exceed_vram(self):
plan = _drive(
n_ctx = 32768,
model_gib = 131,
gpus = [(0, 97_000)],
native_ctx = 196608,
)
assert plan["c_arg"] == 32768
assert plan["use_fit"] is True
def test_explicit_at_fallback_on_too_big(self):
plan = _drive(
n_ctx = _AUTO_OFFLOAD_CTX,
model_gib = 131,
gpus = [(0, 97_000)],
native_ctx = 196608,
)
assert plan["c_arg"] == _AUTO_OFFLOAD_CTX
assert plan["use_fit"] is True
def test_explicit_below_floor_honored(self):
plan = _drive(
n_ctx = 2048,
model_gib = 8,
gpus = [(0, 24_000)],
)
assert plan["c_arg"] == 2048
class TestExtraArgsCtxOverride:
def test_ctx_size_extra_honored_over_auto(self):
plan = _drive_native(
model_gib = 131,
gpus = [(0, 97_000)],
native_ctx = 196608,
extra_args = ["--ctx-size", "128000"],
)
assert plan["ctx_override"] == 128000
assert plan["original_ctx"] == 128000
assert plan["c_arg"] == 128000
assert plan["use_fit"] is True
def test_ctx_size_short_alias_honored_over_auto(self):
plan = _drive_native(
model_gib = 131,
gpus = [(0, 97_000)],
native_ctx = 196608,
extra_args = ["-c", "128000"],
)
assert plan["c_arg"] == 128000
assert plan["use_fit"] is True
def test_ctx_size_extra_wins_over_first_class_field(self):
plan = _drive_native(
n_ctx = 4096,
model_gib = 8,
gpus = [(0, 24_000)],
extra_args = ["--ctx-size", "128000"],
)
assert plan["original_ctx"] == 128000
assert plan["c_arg"] == 128000
class TestFittableAutoPickRegressions:
def test_small_model_one_gpu(self):
plan = _drive_native(model_gib = 8, gpus = [(0, 24_000)], kv_per_token_bytes = 8192)
assert plan["use_fit"] is False
assert plan["gpu_indices"] == [0]
assert plan["c_arg"] > FIT_MIN_CTX
def test_medium_model_needs_multi_gpu(self):
plan = _drive_native(
model_gib = 60,
gpus = [(0, 40_000), (1, 40_000)],
kv_per_token_bytes = 8192,
)
assert plan["use_fit"] is False
assert plan["gpu_indices"] == [0, 1]
def test_no_kv_metadata_fittable_auto(self):
plan = _drive_native(model_gib = 8, gpus = [(0, 24_000)], can_estimate_kv = False)
assert plan["use_fit"] is False
assert plan["gpu_indices"] == [0]
class TestTightFitPinsToGPU:
"""Models that fit at 91-95% of free VRAM must use the GPU."""
def test_rtx_4090_qwen_24gb_class(self):
plan = _drive_native(model_gib = 20.8, gpus = [(0, 22_805)], kv_per_token_bytes = 25_000)
assert plan["use_fit"] is False
assert plan["gpu_indices"] == [0]
def test_explicit_ctx_at_94_pct_pins_to_gpu(self):
plan = _drive_native(
n_ctx = 4096,
model_gib = 20.8,
gpus = [(0, 22_805)],
kv_per_token_bytes = 25_000,
)
assert plan["use_fit"] is False
assert plan["gpu_indices"] == [0]
def test_genuine_overflow_still_uses_fit(self):
plan = _drive_native(
n_ctx = 4096,
model_gib = 23,
gpus = [(0, 22_000)],
kv_per_token_bytes = 25_000,
)
assert plan["use_fit"] is True
assert plan["gpu_indices"] is None
@pytest.mark.parametrize("platform_tag", ["linux", "windows", "mac", "rocm"])
def test_identical_decision_across_platforms(platform_tag):
"""Decision takes ``[(gpu_idx, free_mib), ...]`` regardless of source;
identical inputs must yield identical plans."""
plan_a = _drive(n_ctx = 0, model_gib = 8, gpus = [(0, 24_000)])
plan_b = _drive(n_ctx = 0, model_gib = 8, gpus = [(0, 24_000)])
assert plan_a == plan_b, platform_tag
class TestClassifyGpuOffload:
def _backend(self, stdout_lines):
inst = LlamaCppBackend.__new__(LlamaCppBackend)
inst._stdout_lines = list(stdout_lines)
return inst
def test_cuda_buffer_present_returns_true(self):
inst = self._backend(
[
"load_tensors: offloaded 33/33 layers to GPU",
"load_tensors: CUDA0 model buffer size = 21000.0 MiB",
"load_tensors: CPU_Mapped model buffer size = 0.6 MiB",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is True
def test_cpu_only_buffer_returns_false(self):
inst = self._backend(
[
"load_tensors: CPU_Mapped model buffer size = 21000.0 MiB",
"load_tensors: CPU model buffer size = 0.6 MiB",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
def test_no_buffer_lines_returns_none(self):
inst = self._backend(
[
"INFO [main] starting server",
"load_tensors: file format = GGUF V3",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is None
def test_no_gpus_detected_returns_none(self):
inst = self._backend(
[
"load_tensors: CPU_Mapped model buffer size = 21000.0 MiB",
]
)
assert inst._classify_gpu_offload(False, []) is None
def test_user_did_not_intend_gpu_returns_none(self):
inst = self._backend(
[
"load_tensors: CPU_Mapped model buffer size = 21000.0 MiB",
]
)
assert inst._classify_gpu_offload(False, [(0, 22805)]) is None
def test_rocm_buffer_marker_returns_true(self):
inst = self._backend(
[
"load_tensors: ROCm0 model buffer size = 21000.0 MiB",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is True
def test_metal_buffer_marker_returns_true(self):
inst = self._backend(
[
"load_tensors: Metal model buffer size = 8000.0 MiB",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is True
def test_offloaded_zero_count_returns_false(self):
inst = self._backend(
[
"load_tensors: offloaded 0/33 layers to GPU",
"load_tensors: CUDA0 model buffer size = 21000.0 MiB",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
def test_offloaded_draft_then_main_returns_true(self):
inst = self._backend(
[
"load_tensors: offloaded 0/2 layers to GPU",
"load_tensors: offloaded 33/33 layers to GPU",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is True
def test_main_on_cpu_with_draft_on_gpu_returns_false(self):
inst = self._backend(
[
"load_tensors: offloaded 0/33 layers to GPU",
"load_tensors: offloaded 1/1 layers to GPU",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
def test_main_on_gpu_with_draft_on_cpu_returns_true(self):
inst = self._backend(
[
"load_tensors: offloaded 33/33 layers to GPU",
"load_tensors: offloaded 0/1 layers to GPU",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is True
def test_cuda_host_buffer_excluded_returns_false(self):
# CUDA_Host is CPU-pinned memory, not a model offload.
inst = self._backend(
[
"load_tensors: CUDA_Host model buffer size = 500.0 MiB",
"load_tensors: CPU model buffer size = 21000.0 MiB",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
def test_device_info_gpu_row_alone_is_inconclusive(self):
# device_info lists available devices, not where the model loaded, so a
# GPU row alone is not proof of offload.
inst = self._backend(
[
"print_info: device_info:",
" - CUDA0 : 24564 MiB free",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is None
def test_cpu_buffers_with_gpu_device_row_returns_false(self):
inst = self._backend(
[
"load_tensors: CPU model buffer size = 21000.0 MiB",
"print_info: device_info:",
" - CUDA0 : 24564 MiB free",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
def test_device_info_cpu_only_returns_false(self):
inst = self._backend(
[
"print_info: device_info:",
" - CPU : 64000 MiB free",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
def test_system_info_cuda_before_device_info_does_not_count(self):
inst = self._backend(
[
"system_info: CUDA : ARCHS = 890 | n_threads = 8",
"print_info: device_info:",
" - CPU : 64000 MiB free",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
@pytest.mark.parametrize(
"marker",
["CUDA0", "ROCm0", "HIP0", "Metal", "Vulkan0", "OpenCL0", "SYCL0", "MUSA0", "CANN0"],
)
def test_all_gpu_buffer_markers_return_true(self, marker):
assert (
classify_gpu_offload_lines([f"load_tensors: {marker} model buffer size = 8000.0 MiB"])
is True
)
def test_module_level_no_signal_returns_none(self):
assert classify_gpu_offload_lines(["INFO starting server"]) is None
def test_select_gpus_ranks_by_usable_not_raw_free():
# Usable: 80 GB card 25.9 GB, 32 GB card 27.4 GB; raw-free ranking would split the model.
gpus = [(0, 30000), (1, 29000)]
totals = {0: 81920, 1: 32607}
model = int(27000 * 1024 * 1024)
idxs, use_fit = LlamaCppBackend._select_gpus(model, gpus, total_by_idx = totals)
assert idxs == [1] and use_fit is False
def test_select_gpus_reserves_per_device_overhead():
# a multi-GPU split pays ~1 GiB per extra GPU, pushing the 2-GPU need past the pool
gpus = [(0, 16000), (1, 16000)]
totals = {0: 16384, 1: 16384}
gib = 1024 * 1024 * 1024
model = int(30000 * 1024 * 1024)
idxs, use_fit = LlamaCppBackend._select_gpus(model, gpus, total_by_idx = totals)
assert idxs == [0, 1] and use_fit is False
idxs2, use_fit2 = LlamaCppBackend._select_gpus(
model, gpus, total_by_idx = totals, per_device_overhead_bytes = gib
)
assert idxs2 is None and use_fit2 is True
small = int(15000 * 1024 * 1024)
a, _ = LlamaCppBackend._select_gpus(small, gpus, total_by_idx = totals)
b, _ = LlamaCppBackend._select_gpus(
small, gpus, total_by_idx = totals, per_device_overhead_bytes = gib
)
assert a == [0] and b == [0]
def _force_apple(monkeypatch):
import platform as _platform
monkeypatch.setattr(_platform, "system", lambda: "Darwin")
monkeypatch.setattr(_platform, "machine", lambda: "arm64")
# The working-set reader falls back to torch.mps; keep the host's torch out of it.
monkeypatch.setitem(sys.modules, "torch", None)
def _install_fake_torch_mps(
monkeypatch,
working_set_bytes,
available = True,
):
fake = _types.ModuleType("torch")
fake.backends = _types.SimpleNamespace(
mps = _types.SimpleNamespace(is_available = lambda: available)
)
fake.mps = _types.SimpleNamespace(recommended_max_memory = lambda: working_set_bytes)
monkeypatch.setitem(sys.modules, "torch", fake)
def _install_fake_mlx(monkeypatch, working_set_bytes):
"""Minimal mlx.core stub exposing metal.is_available() and device_info()."""
mlx = _types.ModuleType("mlx")
mlx_core = _types.ModuleType("mlx.core")
mlx_core.metal = _types.SimpleNamespace(is_available = lambda: True)
mlx_core.device_info = lambda: {"max_recommended_working_set_size": working_set_bytes}
mlx.core = mlx_core
monkeypatch.setitem(sys.modules, "mlx", mlx)
monkeypatch.setitem(sys.modules, "mlx.core", mlx_core)
def _install_fake_psutil(
monkeypatch,
*,
total,
available = None,
):
"""psutil stub: the budget reads both total and available, so a test must pin both or available
comes from the host running the suite."""
fake = _types.ModuleType("psutil")
vm = _types.SimpleNamespace(total = total)
if available is not None:
vm.available = available
fake.virtual_memory = lambda: vm
monkeypatch.setitem(sys.modules, "psutil", fake)
@pytest.mark.usefixtures("real_apple_metal_budget")
class TestAppleUnifiedMemoryBudget:
def test_zero_off_apple_silicon(self, monkeypatch):
import platform as _platform
monkeypatch.setattr(_platform, "system", lambda: "Linux")
monkeypatch.setattr(_platform, "machine", lambda: "x86_64")
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == 0
def test_uses_metal_working_set(self, monkeypatch):
_force_apple(monkeypatch)
ws = 27 * GIB
_install_fake_mlx(monkeypatch, ws)
_install_fake_psutil(monkeypatch, total = 36 * GIB, available = 34 * GIB)
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == int(
ws * _APPLE_UNIFIED_MEMORY_FRACTION
)
def test_falls_back_to_total_ram_without_mlx(self, monkeypatch):
_force_apple(monkeypatch)
monkeypatch.setitem(sys.modules, "mlx", None)
_install_fake_psutil(monkeypatch, total = 36 * GIB, available = 36 * GIB)
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == int(
36 * GIB * _APPLE_UNIFIED_MEMORY_FRACTION
)
def test_without_mlx_torch_mps_supplies_the_working_set(self, monkeypatch):
"""macOS 13 has no MLX wheel; budgeting from total RAM exceeds what ggml-metal can use."""
_force_apple(monkeypatch)
monkeypatch.setitem(sys.modules, "mlx", None)
_install_fake_torch_mps(monkeypatch, 10 * GIB)
_install_fake_psutil(monkeypatch, total = 16 * GIB, available = 14 * GIB)
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == int(
10 * GIB * _APPLE_UNIFIED_MEMORY_FRACTION
)
def test_mlx_wins_over_torch_mps(self, monkeypatch):
_force_apple(monkeypatch)
_install_fake_mlx(monkeypatch, 10 * GIB)
_install_fake_torch_mps(monkeypatch, 12 * GIB)
_install_fake_psutil(monkeypatch, total = 16 * GIB, available = 14 * GIB)
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == int(
10 * GIB * _APPLE_UNIFIED_MEMORY_FRACTION
)
def test_torch_without_mps_keeps_total_ram(self, monkeypatch):
_force_apple(monkeypatch)
monkeypatch.setitem(sys.modules, "mlx", None)
_install_fake_torch_mps(monkeypatch, 10 * GIB, available = False)
_install_fake_psutil(monkeypatch, total = 16 * GIB, available = 16 * GIB)
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == int(
16 * GIB * _APPLE_UNIFIED_MEMORY_FRACTION
)
def test_a_busy_machine_budgets_from_what_is_free(self, monkeypatch):
"""Metal budget uses free memory: the static working set is the same loaded or idle."""
_force_apple(monkeypatch)
_install_fake_mlx(monkeypatch, 10 * GIB)
_install_fake_psutil(monkeypatch, total = 16 * GIB, available = 6 * GIB)
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == int(
6 * GIB * _APPLE_UNIFIED_MEMORY_FRACTION
)
def test_available_never_raises_the_budget(self, monkeypatch):
"""Only ever lowers it: the device ceiling still bounds a free machine."""
_force_apple(monkeypatch)
_install_fake_mlx(monkeypatch, 10 * GIB)
_install_fake_psutil(monkeypatch, total = 64 * GIB, available = 60 * GIB)
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == int(
10 * GIB * _APPLE_UNIFIED_MEMORY_FRACTION
)
def test_a_psutil_without_available_keeps_the_old_answer(self, monkeypatch):
"""Nothing to cap with, so behave exactly as before rather than zero."""
_force_apple(monkeypatch)
_install_fake_mlx(monkeypatch, 10 * GIB)
_install_fake_psutil(monkeypatch, total = 16 * GIB)
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == int(
10 * GIB * _APPLE_UNIFIED_MEMORY_FRACTION
)
def test_the_working_set_still_applies_without_psutil(self, monkeypatch):
_force_apple(monkeypatch)
_install_fake_mlx(monkeypatch, 10 * GIB)
monkeypatch.setitem(sys.modules, "psutil", None)
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == int(
10 * GIB * _APPLE_UNIFIED_MEMORY_FRACTION
)
def test_zero_when_no_budget_resolvable(self, monkeypatch):
_force_apple(monkeypatch)
monkeypatch.setitem(sys.modules, "mlx", None)
monkeypatch.setitem(sys.modules, "psutil", None)
assert LlamaCppBackend._apple_metal_memory_budget_bytes() == 0
# Unpatched: conftest pins the probe to 0.
_REAL_WIRED_CEILING = LlamaCppBackend.__dict__["_apple_metal_wired_ceiling_bytes"].__func__
def _install_wired_probes(monkeypatch, *, sysctl_mb, working_set, in_use):
"""``sysctl_mb`` or ``in_use`` of None is an unreadable probe."""
from core.inference import llama_cpp as _llama_cpp
from utils.hardware import hardware as _hardware
_force_apple(monkeypatch)
_install_fake_mlx(monkeypatch, working_set)
sysctl_out = "" if sysctl_mb is None else f"{sysctl_mb}\n"
monkeypatch.setattr(
_llama_cpp.subprocess,
"run",
lambda *a, **k: _types.SimpleNamespace(stdout = sysctl_out),
)
monkeypatch.setattr(
_hardware,
"_read_apple_gpu_stats",
lambda: {} if in_use is None else {"vram_used_bytes": in_use},
)
class TestAppleWiredCeiling:
def test_zero_off_apple_silicon(self, monkeypatch):
import platform as _platform
_install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = 2 * GIB)
monkeypatch.setattr(_platform, "system", lambda: "Linux")
monkeypatch.setattr(_platform, "machine", lambda: "x86_64")
assert _REAL_WIRED_CEILING() == 0
def test_the_working_set_is_the_limit_by_default(self, monkeypatch):
_install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = 2 * GIB)
assert _REAL_WIRED_CEILING() == int(46 * GIB * _APPLE_WIRED_CEILING_FRACTION)
def test_older_mlx_reads_the_legacy_alias(self, monkeypatch):
_install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = 2 * GIB)
mlx_core = sys.modules["mlx.core"]
mlx_core.metal.device_info = mlx_core.device_info
monkeypatch.delattr(mlx_core, "device_info")
assert _REAL_WIRED_CEILING() == int(46 * GIB * _APPLE_WIRED_CEILING_FRACTION)
def test_a_set_sysctl_limit_wins(self, monkeypatch):
_install_wired_probes(
monkeypatch, sysctl_mb = 56 * 1024, working_set = 48 * GIB, in_use = 2 * GIB
)
assert _REAL_WIRED_CEILING() == int(54 * GIB * _APPLE_WIRED_CEILING_FRACTION)
def test_unread_gpu_memory_is_unresolved_not_zero(self, monkeypatch):
_install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 48 * GIB, in_use = None)
assert _REAL_WIRED_CEILING() == 0
def test_unread_sysctl_is_unresolved_not_the_default(self, monkeypatch):
_install_wired_probes(monkeypatch, sysctl_mb = None, working_set = 48 * GIB, in_use = 2 * GIB)
assert _REAL_WIRED_CEILING() == 0
def test_no_readable_limit_is_unresolved(self, monkeypatch):
_install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 0, in_use = 2 * GIB)
assert _REAL_WIRED_CEILING() == 0
def test_without_mlx_the_default_limit_comes_from_torch_mps(self, monkeypatch):
_install_wired_probes(monkeypatch, sysctl_mb = 0, working_set = 0, in_use = 2 * GIB)
monkeypatch.setitem(sys.modules, "mlx", None)
_install_fake_torch_mps(monkeypatch, 48 * GIB)
assert _REAL_WIRED_CEILING() == int(46 * GIB * _APPLE_WIRED_CEILING_FRACTION)
@pytest.mark.parametrize(
"counter,expected",
[
("", 0),
('"In use system memory"=2147483648,', int(46 * GIB * _APPLE_WIRED_CEILING_FRACTION)),
],
)
def test_ioreg_resolves_only_with_the_in_use_counter(self, monkeypatch, counter, expected):
_force_apple(monkeypatch)
_install_fake_mlx(monkeypatch, 48 * GIB)
block = f'"PerformanceStatistics" = {{"Device Utilization %"=3,{counter}"Alloc system memory"=1}}'
def fake_run(cmd, **kwargs):
if cmd[0] == "ioreg":
return _types.SimpleNamespace(stdout = block.encode())
return _types.SimpleNamespace(stdout = "0\n")
monkeypatch.setattr("subprocess.run", fake_run)
assert _REAL_WIRED_CEILING() == expected
class TestAppleContextCap:
"""The real ``_fit_context_to_vram`` against the reporter's M3 Pro case."""
def test_caps_native_context_into_unified_budget(self):
inst = _make_backend(native_ctx = 262144)
inst._can_estimate_kv = lambda: True
inst._estimate_kv_cache_bytes = lambda n, *a, **k: 0 if n <= 0 else int(n * 64_000)
model_size_fit = int(15.7 * GIB)
budget_mib = int(27 * GIB * _APPLE_UNIFIED_MEMORY_FRACTION) // (1024 * 1024)
native_footprint_mib = (model_size_fit + inst._estimate_kv_cache_bytes(262144)) // (
1024 * 1024
)
assert native_footprint_mib > budget_mib
capped = inst._fit_context_to_vram(
262144, budget_mib, model_size_fit, None, budget_frac = 1.0
)
assert capped < 262144
capped_footprint_mib = (model_size_fit + inst._estimate_kv_cache_bytes(capped)) // (
1024 * 1024
)
assert capped_footprint_mib <= budget_mib
class TestAppleBranchEndToEnd:
"""Drive the Apple elif glue (cap / floor / explicit) via _drive, no GPU."""
def test_auto_context_capped_below_native(self):
plan = _drive_native(
model_gib = 15.7,
gpus = [],
native_ctx = 262144,
kv_per_token_bytes = 64_000,
apple_budget_mib = 23_000,
)
assert 0 < plan["c_arg"] < 262144
assert plan["use_fit"] is True
assert plan["gpu_indices"] is None
assert plan["max_available_ctx"] == plan["c_arg"]
def test_floors_to_fallback_when_weights_exceed_budget(self):
plan = _drive_native(
model_gib = 100,
gpus = [],
native_ctx = 262144,
apple_budget_mib = 20_000,
)
assert plan["c_arg"] == FIT_MIN_CTX
assert plan["use_fit"] is True
assert plan["gpu_indices"] is None
def test_explicit_context_honored_verbatim(self):
plan = _drive(
n_ctx = 200_000,
model_gib = 15.7,
gpus = [],
native_ctx = 262144,
kv_per_token_bytes = 64_000,
apple_budget_mib = 23_000,
)
assert plan["c_arg"] == 200_000
assert plan["use_fit"] is True
assert plan["max_available_ctx"] < 262144
class TestAppleMtpFlatReserve:
"""Apple cap reserves the flat MTP fraction up front (like _pin_fraction) so
an unsized MTP draft (Qwen3.6-MTP, #6529) can't over-commit."""
def test_flat_reserve_keeps_draft_within_budget(self):
kw = dict(
n_ctx = 0,
model_gib = 15.7,
gpus = [],
native_ctx = 262144,
kv_per_token_bytes = 64_000,
apple_budget_mib = 23_000,
)
no_reserve = _drive(**kw, flat_mtp_reserve = 0.0)
with_reserve = _drive(**kw, flat_mtp_reserve = 0.05)
def footprint_mib(ctx):
return (15.7 * GIB + ctx * 64_000) / (1024 * 1024)
assert footprint_mib(no_reserve["c_arg"]) + 0.05 * 23_000 > 23_000
assert with_reserve["c_arg"] < no_reserve["c_arg"]
assert footprint_mib(with_reserve["c_arg"]) + 0.05 * 23_000 <= 23_000
def test_no_reserve_is_a_noop_when_mtp_absent(self):
kw = dict(
n_ctx = 0,
model_gib = 15.7,
gpus = [],
native_ctx = 262144,
kv_per_token_bytes = 64_000,
apple_budget_mib = 23_000,
)
assert _drive(**kw, flat_mtp_reserve = 0.0) == _drive(**kw)
class TestAppleNoKvMetadataFloor:
"""Sparse KV metadata keeps Apple's safety floor instead of native context."""
def test_sparse_kv_floors_auto_context(self):
plan = _drive_native(
model_gib = 15.7,
gpus = [],
native_ctx = 262144,
can_estimate_kv = False,
apple_budget_mib = 23_000,
)
assert plan["c_arg"] == FIT_MIN_CTX
assert plan["use_fit"] is True
assert plan["gpu_indices"] is None
def test_sparse_kv_still_honors_explicit_context(self):
plan = _drive(
n_ctx = 100_000,
model_gib = 15.7,
gpus = [],
native_ctx = 262144,
can_estimate_kv = False,
apple_budget_mib = 23_000,
)
assert plan["c_arg"] == 100_000
class TestPartialOffloadIsReportable:
"""The counts behind the boolean."""
def test_a_split_load_keeps_both_numbers(self):
assert parse_gpu_offload_counts(["load_tensors: offloaded 38/60 layers to GPU"]) == (38, 60)
def test_the_boolean_cannot_tell_these_apart(self):
split = ["load_tensors: offloaded 12/60 layers to GPU"]
whole = ["load_tensors: offloaded 60/60 layers to GPU"]
assert classify_gpu_offload_lines(split) is classify_gpu_offload_lines(whole) is True
assert parse_gpu_offload_counts(split) != parse_gpu_offload_counts(whole)
def test_the_main_model_wins_over_a_draft(self):
lines = [
"load_tensors: offloaded 3/3 layers to GPU",
"load_tensors: offloaded 12/60 layers to GPU",
]
assert parse_gpu_offload_counts(lines) == (12, 60)
def test_a_drafter_of_equal_size_does_not_mask_the_main_model(self):
# Tie: first line wins, the main model is logged before its drafter.
lines = [
"load_tensors: offloaded 16/32 layers to GPU",
"load_tensors: offloaded 32/32 layers to GPU",
]
assert parse_gpu_offload_counts(lines) == (16, 32)
def test_no_counted_line_is_not_a_guess(self):
assert parse_gpu_offload_counts(["INFO starting server"]) is None
assert parse_gpu_offload_counts([]) is None
def test_a_zero_total_is_not_reportable(self):
assert parse_gpu_offload_counts(["offloaded 0/0 layers to GPU"]) is None
def test_an_extras_ngl_reads_as_the_users_own_placement(self):
from core.inference.llama_cpp import (
_GPU_OFFLOAD_OVERRIDE_FLAGS,
_extra_args_set_any_flag,
)
assert _extra_args_set_any_flag(["-ngl", "20"], _GPU_OFFLOAD_OVERRIDE_FLAGS)
assert _extra_args_set_any_flag(["--gpu-layers=20"], _GPU_OFFLOAD_OVERRIDE_FLAGS)
assert not _extra_args_set_any_flag(["--threads", "8"], _GPU_OFFLOAD_OVERRIDE_FLAGS)
assert not _extra_args_set_any_flag(None, _GPU_OFFLOAD_OVERRIDE_FLAGS)
def test_an_all_cpu_device_table_means_the_backend_did_not_load(self):
from core.inference.llama_cpp import llama_saw_gpu_device
cpu_only = [
"device_info: available devices",
" - CPU: 12 cores",
]
with_gpu = [
"device_info: available devices",
" - CUDA0: NVIDIA GeForce RTX 4090",
" - CPU: 12 cores",
]
assert llama_saw_gpu_device(cpu_only) is False
assert llama_saw_gpu_device(with_gpu) is True
assert llama_saw_gpu_device(["INFO starting server"]) is None
assert llama_saw_gpu_device([" - CUDA0: something"]) is None
def test_a_fit_on_in_extras_is_not_a_pinned_split(self):
from core.inference.llama_cpp import _GPU_LAYER_FLAGS, _extra_args_set_any_flag
assert not _extra_args_set_any_flag(["--fit", "on"], _GPU_LAYER_FLAGS)
assert not _extra_args_set_any_flag(["--fit", "off"], _GPU_LAYER_FLAGS)
assert _extra_args_set_any_flag(["-ngl", "20"], _GPU_LAYER_FLAGS)
def test_a_cpu_device_is_deliberate_placement_too(self):
from core.inference.llama_cpp import _device_selection_is_cpu
assert _device_selection_is_cpu(["--device", "cpu"])
assert _device_selection_is_cpu(["-dev", "none"])
assert _device_selection_is_cpu(None, {"LLAMA_ARG_DEVICE": "cpu"})
assert not _device_selection_is_cpu(["--device", "CUDA0"])
assert not _device_selection_is_cpu(None)