Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""MEASURED_* throughputs are stored as ms per token, since time composes and throughput does not."""
from __future__ import annotations
import pytest
from core.inference.offload_cost_model import (
REFERENCE_CONTIGUOUS_MS_PER_GIB,
Access,
HostProfile,
Placement,
TensorGroup,
generation_penalty_ms,
prefill_penalty_ms_per_token,
rank,
)
GIB = float(1024**3)
def ms(t_per_s: float) -> float:
return 1000.0 / t_per_s
# Anchors at 128K for UD-Q4_K_XL on the dense Qwen3.8-27B.
DENSE_BASE = ms(75.37)
DENSE_LM_HEAD = ms(42.60)
DENSE_FFN = ms(13.63)
DENSE_BOTH = ms(11.39)
DENSE_KV_HOST = ms(1.03)
MOE_BASE = ms(182.0)
MOE_EXPERTS = ms(71.63)
MOE_KV_HOST = ms(3.24)
DENSE_FFN_G = TensorGroup("ffn", int(10.092 * GIB), Access.CONTIGUOUS)
DENSE_LM_G = TensorGroup("lm_head", int(0.9713 * GIB), Access.SINGLE_MATVEC)
DENSE_KV_BYTES = 16 * 4 * 512 * 2 * 131072
MOE_EXPERT_G = TensorGroup(
"experts", int(18.320 * GIB), Access.SCATTERED, activation_fraction = 8 / 256
)
MOE_KV_BYTES = 10 * 2 * 512 * 2 * 131072
def rel_err(predicted: float, measured: float) -> float:
return abs(predicted - measured) / measured
# Under-predicts 128K anchors by ~7% uniformly; a uniform offset cannot change an ordering.
ANCHOR_TOL = 0.10
@pytest.mark.parametrize(
"label,placement,measured_delta",
[
("lm_head only", Placement([DENSE_LM_G]), DENSE_LM_HEAD - DENSE_BASE),
("ffn only", Placement([DENSE_FFN_G]), DENSE_FFN - DENSE_BASE),
("ffn + lm_head", Placement([DENSE_FFN_G, DENSE_LM_G]), DENSE_BOTH - DENSE_BASE),
(
"kv to host",
Placement([], kv_host_bytes = DENSE_KV_BYTES),
DENSE_KV_HOST - DENSE_BASE,
),
("moe experts", Placement([MOE_EXPERT_G]), MOE_EXPERTS - MOE_BASE),
(
"moe kv to host",
Placement([], kv_host_bytes = MOE_KV_BYTES),
MOE_KV_HOST - MOE_BASE,
),
],
)
def test_the_model_reproduces_every_measured_anchor(label, placement, measured_delta):
assert rel_err(generation_penalty_ms(placement), measured_delta) < ANCHOR_TOL, label
def test_the_kv_ratio_transfers_across_two_unrelated_models():
"""The dense-only KV rate predicts the MoE within the same tolerance, so cache spill is a rule."""
dense_rate = (DENSE_KV_HOST - DENSE_BASE) / (DENSE_KV_BYTES / GIB)
moe_rate = (MOE_KV_HOST - MOE_BASE) / (MOE_KV_BYTES / GIB)
assert rel_err(moe_rate, dense_rate) < 0.03
def test_the_cache_is_the_worst_byte_to_move_by_an_order_of_magnitude():
"""20x per byte against contiguous weights. This is the whole reason the
planner uses -ot (which leaves the cache resident) instead of -ngl."""
one_gib_weights = Placement([TensorGroup("w", int(GIB), Access.CONTIGUOUS)])
one_gib_cache = Placement([], kv_host_bytes = int(GIB))
assert generation_penalty_ms(one_gib_cache) > 15 * generation_penalty_ms(one_gib_weights)
def test_ffn_is_cheaper_per_byte_than_lm_head():
"""FFN goes first because it costs less per byte than lm_head, though lm_head is smaller."""
per_gib = {
access: generation_penalty_ms(Placement([TensorGroup("x", int(GIB), access)]))
for access in (Access.CONTIGUOUS, Access.SINGLE_MATVEC, Access.SCATTERED)
}
assert per_gib[Access.CONTIGUOUS] < per_gib[Access.SINGLE_MATVEC]
assert per_gib[Access.SINGLE_MATVEC] < per_gib[Access.SCATTERED]
def test_spilling_two_groups_costs_MORE_than_the_sum_not_less():
"""Penalties must rank in ms: as throughput percentages lm_head looks like a discount on top of FFN."""
alone = generation_penalty_ms(Placement([DENSE_LM_G]))
ffn_only = generation_penalty_ms(Placement([DENSE_FFN_G]))
both = generation_penalty_ms(Placement([DENSE_FFN_G, DENSE_LM_G]))
assert both > ffn_only + alone
marginal = both - ffn_only
assert marginal > alone
def test_the_measured_marginal_cost_of_lm_head_really_does_rise():
"""Same claim, against the raw numbers rather than the model."""
alone = DENSE_LM_HEAD - DENSE_BASE
marginal_on_top_of_ffn = DENSE_BOTH - DENSE_FFN
assert marginal_on_top_of_ffn > alone
assert 1.3 < marginal_on_top_of_ffn / alone < 1.5
@pytest.mark.parametrize(
"gib,measured_delta",
[
# Depth 0: compared among themselves, not to the 128K anchors.
(2.184, ms(43.67) - ms(87.30)),
(4.610, ms(26.62) - ms(87.30)),
(7.053, ms(19.71) - ms(87.30)),
(10.092, ms(14.94) - ms(87.30)),
],
)
def test_partial_spilling_is_linear(gib, measured_delta):
"""Four-point least-squares fit: 5.544 ms/GiB, -0.10 ms intercept, so partial spilling is linear."""
predicted = generation_penalty_ms(
Placement([TensorGroup("part", int(gib * GIB), Access.CONTIGUOUS)])
)
assert rel_err(predicted, measured_delta) < 0.10
def test_spilling_less_always_costs_less():
"""Monotonicity. Without it the planner could prefer a larger spill."""
costs = [
generation_penalty_ms(Placement([TensorGroup("p", int(gib * GIB), Access.CONTIGUOUS)]))
for gib in (1.0, 3.0, 5.0, 10.0)
]
assert costs == sorted(costs)
def test_moe_wins_at_generation_and_loses_at_prefill():
"""MoE spills cheaply in generation (8 of 256 experts read) but not in prefill, which reads all."""
moe = Placement([MOE_EXPERT_G])
dense = Placement([DENSE_FFN_G])
assert generation_penalty_ms(moe) < generation_penalty_ms(dense)
assert prefill_penalty_ms_per_token(moe) > prefill_penalty_ms_per_token(dense)
def test_the_measured_penalties_show_that_same_crossover():
"""Against the raw anchors: MoE is hurt less on generation, more on prefill."""
moe_gen = MOE_EXPERTS / MOE_BASE
dense_gen = DENSE_FFN / DENSE_BASE
assert moe_gen < dense_gen
moe_pp, dense_pp = 5522.0 / 1397.0, 2095.0 / 1141.0
assert moe_pp > dense_pp
def test_a_smaller_host_makes_every_spill_worse():
"""Spill cost is held slightly above the measured ratio, since under-warning promises a faster spill."""
big = generation_penalty_ms(Placement([DENSE_FFN_G]), HostProfile(threads = 192))
small = generation_penalty_ms(Placement([DENSE_FFN_G]), HostProfile(threads = 16))
assert small > 2 * big
measured_ratio = (ms(5.83) - ms(87.30)) / (ms(14.94) - ms(87.30))
assert small / big >= measured_ratio
assert small / big < measured_ratio * 1.1
def test_thread_scaling_matches_the_measured_sweep():
"""Predicted ratio between 16 and 64 threads against the measured 11.82/5.83."""
at16 = generation_penalty_ms(Placement([DENSE_FFN_G]), HostProfile(threads = 16))
at64 = generation_penalty_ms(Placement([DENSE_FFN_G]), HostProfile(threads = 64))
measured_ratio = 11.82 / 5.83
assert rel_err(at16 / at64, measured_ratio) < 0.20
def test_prefill_ignores_host_threads_while_generation_does_not():
"""Prefill is link-bound and ignores host threads; batch-1 generation runs on CPU and is core-bound."""
p = Placement([DENSE_FFN_G])
big, small = HostProfile(threads = 192), HostProfile(threads = 8)
assert prefill_penalty_ms_per_token(p, host = small) == prefill_penalty_ms_per_token(p, host = big)
assert generation_penalty_ms(p, small) > 2 * generation_penalty_ms(p, big)
def test_prefill_amortises_over_the_ubatch():
"""Each ubatch copies weights once for all its tokens, so prefill is cheap per byte moved."""
p = Placement([DENSE_FFN_G])
assert prefill_penalty_ms_per_token(p, n_ubatch = 512) == pytest.approx(
prefill_penalty_ms_per_token(p, n_ubatch = 256) / 2.0
)
assert prefill_penalty_ms_per_token(p) < generation_penalty_ms(p) / 100.0
def test_unified_memory_hosts_gain_nothing_from_spilling():
"""Apple Silicon, AMD APUs and Vulkan iGPUs report host RAM as VRAM: spilling frees nothing, so
the planner must not charge a penalty for it or claim any freed memory."""
unified = HostProfile(unified_memory = True)
assert generation_penalty_ms(Placement([DENSE_FFN_G, DENSE_LM_G]), unified) == 0.0
assert generation_penalty_ms(Placement([], kv_host_bytes = DENSE_KV_BYTES), unified) == 0.0
def test_ranking_puts_the_measured_best_placement_first():
"""Ranking recovers the measured best order: resident, FFN, FFN plus lm_head, cache last."""
resident = Placement([])
ffn = Placement([DENSE_FFN_G])
ffn_lm = Placement([DENSE_FFN_G, DENSE_LM_G])
kv = Placement([], kv_host_bytes = DENSE_KV_BYTES)
order = [p for p, _ in rank([kv, ffn_lm, ffn, resident])]
assert order == [resident, ffn, ffn_lm, kv]
def test_a_prefill_heavy_mix_can_reorder_dense_against_moe():
"""A prefill-weighted caller needs its own ranking, not the generation-only one."""
moe, dense = Placement([MOE_EXPERT_G]), Placement([DENSE_FFN_G])
gen_first = [p for p, _ in rank([dense, moe], n_generated = 1, n_prompt = 0)]
pp_first = [p for p, _ in rank([dense, moe], n_generated = 0, n_prompt = 4096)]
assert gen_first[0] is moe
assert pp_first[0] is dense
def test_the_cross_host_floor_matches_the_measured_cloud_hosts():
"""The host-thread floor is fitted to measured cloud VMs, not the 192-thread reference box alone."""
rate = lambda t: ( # noqa: E731 - one expression, reads better inline
HostProfile(threads = t).generation_slowdown * REFERENCE_CONTIGUOUS_MS_PER_GIB
)
for threads, measured in ((12, 24.21), (48, 6.82), (192, 5.498)):
ratio = rate(threads) / measured
assert 0.85 <= ratio <= 1.15, (threads, rate(threads), measured, ratio)
assert rate(2) > rate(8) > rate(12) > rate(48) > rate(192)
assert rate(2) > 10 * rate(192)
def test_a_host_cache_is_not_free_during_prefill():
"""Host-resident KV must cost prefill time too: the cache crosses the same link as the weights."""
resident = Placement()
kv_host = Placement(kv_host_bytes = int(4 * GIB))
assert prefill_penalty_ms_per_token(resident) == 0.0
assert prefill_penalty_ms_per_token(kv_host) > 0.0
ordered = rank([kv_host, resident], n_generated = 0, n_prompt = 4096)
assert ordered[0][0] is resident, "resident must win a pure-prefill ranking"
assert ordered[0][1] < ordered[1][1], "and it must not be a tie"
assert prefill_penalty_ms_per_token(kv_host, host = HostProfile(unified_memory = True)) == 0.0