Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Ranks offload placements; spilled weights are CPU-bound in generation but link-bound in prefill."""
from __future__ import annotations
from dataclasses import dataclass, field
from enum import Enum
GIB = float(1024**3)
# Fit on 192-core B200 host: 5.544 ms/GiB, ~0 intercept, so spilling is linear
REFERENCE_CONTIGUOUS_MS_PER_GIB = 5.544
# reference host thread count the rate above was measured at
REFERENCE_HOST_THREADS = 192
# ms/GiB = a / threads + b, fit over 4..192 threads on one host (Amdahl, not power law)
_THREAD_PARALLEL_MS_PER_GIB = 138.4
_THREAD_SERIAL_MS_PER_GIB = 5.57
# Cross-machine fit (12/48 vCPU) is steeper; take the costlier of the two fits, since optimism quotes spills running several times slower
_CROSS_HOST_PARALLEL_MS_PER_GIB = 278.2
_CROSS_HOST_SERIAL_MS_PER_GIB = 1.03
# Known gap: dense Q2_K on small hosts is under-priced (cost tracks dequant complexity)
class Access(str, Enum):
"""How a tensor group is read, which sets its cost per byte."""
#: Dense FFN: large contiguous matmuls, best case for the CPU backend.
CONTIGUOUS = "contiguous"
#: lm_head: one tall matvec, parallelises worse than FFN blocks.
SINGLE_MATVEC = "single_matvec"
# : MoE experts: 8-of-256 scattered small matmuls per layer per token.
SCATTERED = "scattered"
# : The attention cache. By far the worst thing to move off the device.
KV_CACHE = "kv_cache"
# Cost per activated byte relative to CONTIGUOUS, from measured ms/GiB ratios.
_RATE_RATIOS: dict[Access, float] = {
Access.CONTIGUOUS: 1.00,
Access.SINGLE_MATVEC: 1.77,
Access.SCATTERED: 2.49,
Access.KV_CACHE: 20.1,
}
# Coarse constant bracketed by measured dense 49.4 and MoE 66.9 GiB/s
PREFILL_STREAM_GIB_S = 55.0
# Multi-group spills cost more than the sum (contention), measured 6%
MULTI_GROUP_CONTENTION = 0.06
@dataclass(frozen = True)
class TensorGroup:
"""A set of weights the planner can place as a unit."""
name: str
bytes_total: int
access: Access
#: Fraction of ``bytes_total`` read per generated token (n_expert_used / n_expert for MoE).
activation_fraction: float = 1.0
@property
def activated_bytes(self) -> int:
return int(self.bytes_total * self.activation_fraction)
@dataclass(frozen = True)
class HostProfile:
"""What the host can bring to bear on CPU-resident weights.
``threads`` is the honest knob. Generation cost scales with it (measured
2.42 / 5.83 / 11.82 t/s at 4 / 16 / 64), with diminishing returns as the
memory subsystem saturates, which ``thread_scaling_exponent`` captures.
"""
threads: int = REFERENCE_HOST_THREADS
#: Unified-memory hosts, where spilling moves nothing.
unified_memory: bool = False
@property
def generation_slowdown(self) -> float:
"""How much slower than the reference host this one is per host byte."""
if self.threads <= 0:
return 1.0
def rate(threads: float) -> float:
"""Takes the costlier of two fits, so a small host is not quoted a server's throughput."""
return max(
_THREAD_PARALLEL_MS_PER_GIB / threads + _THREAD_SERIAL_MS_PER_GIB,
_CROSS_HOST_PARALLEL_MS_PER_GIB / threads + _CROSS_HOST_SERIAL_MS_PER_GIB,
)
return rate(float(self.threads)) / rate(float(REFERENCE_HOST_THREADS))
@dataclass
class Placement:
"""A candidate: which groups sit in host memory, and where the cache is."""
host_groups: list[TensorGroup] = field(default_factory = list)
#: KV bytes forced to host; only ``--no-kv-offload`` sets this.
kv_host_bytes: int = 0
def generation_penalty_ms(placement: Placement, host: HostProfile | None = None) -> float:
"""Extra ms per generated token versus everything resident; zero on unified-memory hosts."""
host = host or HostProfile()
if host.unified_memory:
return 0.0
groups = list(placement.host_groups)
per_group = [
(g.activated_bytes / GIB) * REFERENCE_CONTIGUOUS_MS_PER_GIB * _RATE_RATIOS[g.access]
for g in groups
]
if placement.kv_host_bytes > 0:
per_group.append(
(placement.kv_host_bytes / GIB)
* REFERENCE_CONTIGUOUS_MS_PER_GIB
* _RATE_RATIOS[Access.KV_CACHE]
)
total = sum(per_group)
if len([c for c in per_group if c > 0.0]) > 1:
total *= 1.0 + MULTI_GROUP_CONTENTION
return total * host.generation_slowdown
def prefill_penalty_ms_per_token(
placement: Placement,
n_ubatch: int = 512,
host: HostProfile | None = None,
) -> float:
"""Uses full bytes, not activated ones, since a 512-token ubatch selects nearly every expert anyway."""
host = host or HostProfile()
if host.unified_memory:
return 0.0
# Uncalibrated lower bound, so rank() never calls a cache spill free
host_bytes = sum(g.bytes_total for g in placement.host_groups) + placement.kv_host_bytes
if host_bytes <= 0 or n_ubatch <= 0:
return 0.0
per_ubatch_ms = (host_bytes / GIB) / PREFILL_STREAM_GIB_S * 1000.0
return per_ubatch_ms / float(n_ubatch)
def rank(
candidates: list[Placement],
host: HostProfile | None = None,
n_generated: int = 1,
n_prompt: int = 0,
n_ubatch: int = 512,
) -> list[tuple[Placement, float]]:
"""Sorts cheapest first; both terms are whole-request ms, so prefill and decode add up."""
scored = [
(
c,
n_generated * generation_penalty_ms(c, host)
+ n_prompt * prefill_penalty_ms_per_token(c, n_ubatch, host),
)
for c in candidates
]
return sorted(scored, key = lambda pair: pair[1])