Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Canonical memory estimate projected byte for byte onto POST /api/inference/estimate-memory and
GET /api/models/kv-cache-estimate; no client change; weights_bytes differs by route; pure."""
from __future__ import annotations
from typing import Any, Optional
from models.inference import MemoryEstimate
__all__ = [
"EMPTY_BREAKDOWN",
"build_memory_estimate",
"project_estimate_memory_response",
"project_kv_cache_estimate",
]
class _EmptyBreakdown:
"""Stands in for a planner run that did not happen.
Every attribute is absent, so :func:`build_memory_estimate`'s ``getattr``
defaults apply and the planner-derived fields come back at their "not
computed" values. ``gpu_bytes`` in particular resolves to ``None`` rather
than ``0``, which is the distinction that matters: never ran, as opposed to
ran and found nothing on the card.
A class rather than ``None`` so callers have one object to pass and the
projection stays total, instead of every call site growing its own branch.
"""
__slots__ = ()
def __repr__(self) -> str: # pragma: no cover - debugging aid
return "<EMPTY_BREAKDOWN>"
EMPTY_BREAKDOWN = _EmptyBreakdown()
class _Unset:
"""Distinguishes "not overridden" from an override whose value is None.
``gpu_bytes`` needs all three states -- a number, a real ``None`` meaning the
planner never ran, and "take whatever the breakdown had" -- and ``None``
cannot carry the third.
"""
__slots__ = ()
def __bool__(self) -> bool: # pragma: no cover - defensive
return False
_UNSET = _Unset()
def build_memory_estimate(
breakdown: Any,
*,
quant_file_bytes: int,
native_context: Optional[int] = None,
gpu_floor_bytes: Optional[int] = None,
floor_can_offload: bool = False,
context_is_pinned: bool = True,
inherited_device_pin: bool = False,
spec_unpriced: bool = False,
context_fitted: Optional[int] = None,
moe_offload_unmodelled: bool = False,
gpu_bytes: Any = _UNSET,
compute_bytes: Any = _UNSET,
total_bytes: Any = _UNSET,
n_ctx: Any = _UNSET,
) -> MemoryEstimate:
"""quant_file_bytes comes from the caller, as 0 when unknown rather than the resident total."""
resident = int(getattr(breakdown, "weights_bytes", 0) or 0)
quant = int(quant_file_bytes or 0)
# Deliberately not clamped against resident: the figures come from different sources
return MemoryEstimate(
available = True,
reason = None,
quant_file_bytes = quant,
resident_files_bytes = resident,
kv_bytes = int(getattr(breakdown, "kv_bytes", 0) or 0),
kv_checkpoint_bytes = int(getattr(breakdown, "kv_checkpoint_bytes", 0) or 0),
compute_bytes = (
int(getattr(breakdown, "compute_bytes", 0) or 0)
if isinstance(compute_bytes, _Unset)
else int(compute_bytes or 0)
),
drafter_runtime_bytes = int(getattr(breakdown, "drafter_runtime_bytes", 0) or 0),
drafter_runtime_gpu_bytes = int(getattr(breakdown, "drafter_runtime_gpu_bytes", 0) or 0),
projector_runtime_bytes = int(getattr(breakdown, "projector_runtime_bytes", 0) or 0),
drafter_kv_unsized = bool(getattr(breakdown, "drafter_kv_unsized", False)),
adapters_unsized = bool(getattr(breakdown, "adapters_unsized", False)),
total_bytes = (
int(getattr(breakdown, "total_bytes", 0) or 0)
if isinstance(total_bytes, _Unset)
else int(total_bytes or 0)
),
# Not `or 0`: zero means an all-CPU launch and must stay distinct from None
gpu_bytes = (
(None if getattr(breakdown, "gpu_bytes", None) is None else int(breakdown.gpu_bytes))
if isinstance(gpu_bytes, _Unset)
else (None if gpu_bytes is None else int(gpu_bytes))
),
gpu_floor_bytes = None if gpu_floor_bytes is None else int(gpu_floor_bytes),
floor_can_offload = bool(floor_can_offload),
kv_estimable = bool(getattr(breakdown, "kv_estimable", True)),
kv_on_gpu = bool(getattr(breakdown, "kv_on_gpu", True)),
n_ctx = (
int(getattr(breakdown, "n_ctx", 0) or 0)
if isinstance(n_ctx, _Unset)
else int(n_ctx or 0)
),
native_context = native_context,
context_fitted = context_fitted,
cache_type_kv = getattr(breakdown, "cache_type_kv", None),
n_parallel = int(getattr(breakdown, "n_parallel", 1) or 1),
layer_count = getattr(breakdown, "layer_count", None),
gpu_layers = getattr(breakdown, "gpu_layers", None),
moe_offload_unmodelled = bool(moe_offload_unmodelled),
lookup_table_bytes = int(getattr(breakdown, "lookup_table_bytes", 0) or 0),
lookup_table_on_disk = bool(getattr(breakdown, "lookup_table_on_disk", False)),
mlx_offload = (
None
if getattr(breakdown, "offloadable", None) is None
else {
"ple": breakdown.ple_offload,
"experts": breakdown.expert_offload,
"offloadable": list(breakdown.offloadable),
}
),
context_is_pinned = bool(context_is_pinned),
inherited_device_pin = bool(inherited_device_pin),
spec_unpriced = bool(spec_unpriced),
)
def project_estimate_memory_response(estimate: MemoryEstimate) -> dict:
"""Here weights_bytes is the resident-files total, which this route has always meant by it."""
return {
"available": estimate.available,
"reason": estimate.reason,
# The aggregate meaning. See the module docstring.
"weights_bytes": estimate.resident_files_bytes,
"kv_bytes": estimate.kv_bytes,
"kv_checkpoint_bytes": estimate.kv_checkpoint_bytes,
"compute_bytes": estimate.compute_bytes,
"drafter_runtime_bytes": estimate.drafter_runtime_bytes,
"drafter_runtime_gpu_bytes": estimate.drafter_runtime_gpu_bytes,
"projector_runtime_bytes": estimate.projector_runtime_bytes,
"drafter_kv_unsized": estimate.drafter_kv_unsized,
"adapters_unsized": estimate.adapters_unsized,
"total_bytes": estimate.total_bytes,
"gpu_bytes": estimate.gpu_bytes or 0,
"kv_estimable": estimate.kv_estimable,
"kv_on_gpu": estimate.kv_on_gpu,
"n_ctx": estimate.n_ctx,
"context_fitted": estimate.context_fitted,
"context_is_pinned": estimate.context_is_pinned,
"gpu_floor_bytes": estimate.gpu_floor_bytes,
"floor_can_offload": estimate.floor_can_offload,
"cache_type_kv": estimate.cache_type_kv,
"n_parallel": estimate.n_parallel,
"layer_count": estimate.layer_count,
"gpu_layers": estimate.gpu_layers,
"moe_offload_unmodelled": estimate.moe_offload_unmodelled,
"lookup_table_bytes": estimate.lookup_table_bytes,
"lookup_table_on_disk": estimate.lookup_table_on_disk,
"mlx_offload": estimate.mlx_offload,
}
def project_kv_cache_estimate(
estimate: MemoryEstimate,
*,
kv_bytes: Optional[int] = None,
spec_bytes: Optional[int] = None,
spec_fixed_bytes: Optional[int] = None,
projector_bytes: Optional[int] = None,
kv_checkpoint_bytes: Optional[int] = None,
) -> dict:
"""weights_bytes is the quant file alone, and None (not 0) means no such term for this route."""
return {
# The quant-file meaning. See the module docstring.
"weights_bytes": estimate.quant_file_bytes or None,
# Off the GPU and inside weights_bytes, so the bar can take it off the VRAM segment.
"lookup_table_bytes": estimate.lookup_table_bytes or None,
"kv_bytes": kv_bytes or None,
"native_context": estimate.native_context,
"spec_bytes": spec_bytes,
"n_ctx": estimate.n_ctx,
"projector_bytes": projector_bytes,
"kv_checkpoint_bytes": kv_checkpoint_bytes,
"spec_fixed_bytes": spec_fixed_bytes,
# Deliberately NOT `or None`. Zero means an all-CPU launch.
"gpu_bytes": estimate.gpu_bytes,
"compute_bytes": estimate.compute_bytes or None,
"total_bytes": estimate.total_bytes or None,
"gpu_floor_bytes": estimate.gpu_floor_bytes,
"context_is_pinned": estimate.context_is_pinned,
"inherited_device_pin": estimate.inherited_device_pin,
"spec_unpriced": estimate.spec_unpriced,
}