Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Spill weights with -ot, never the KV cache: spilling a layer moves its cache to host RAM too."""
from __future__ import annotations
import os
import struct
from dataclasses import dataclass, field
from enum import Enum
from typing import Iterable, Mapping, Optional, Sequence
from core.inference.offload_cost_model import (
Access,
HostProfile,
Placement,
TensorGroup,
generation_penalty_ms,
)
from core.inference.offload_layout import (
LM_HEAD_PATTERN,
BlockLayout,
ModelLayout,
spill_pattern_for,
)
GIB = 1024**3
MIB = 1024**2
class ContextPolicy(Enum):
"""Whether the planner may shrink a context the user asked for.
llama.cpp's fitter shrinks context before spilling anything, and on
throughput grounds that is right: a resident smaller context beats a spilled
larger one. But context is a user-visible feature, not a free variable, so
quietly trading it away is not a safe default.
"""
NEVER_REDUCE = "never"
# Shrink if that avoids spilling entirely.
PREFER_RESIDENT = "prefer_resident"
# Shrink only when no rung of the ladder fits.
FIT_ONLY = "fit_only"
class SpillOrder(Enum):
"""Which blocks to spill when only some are needed.
UNMEASURED: every -ot measurement so far spilled all blocks or none, so the
ordering is justified by byte-minimality alone, not by benchmark. Contiguous
runs may schedule better (adjacent host blocks can merge into one graph
split), which would favour FRONT/BACK over LARGEST. Hence configurable.
"""
# Best-fit-decreasing: fewest blocks and least overshoot (overshoot costs bandwidth).
LARGEST_FIRST = "largest_first"
FRONT_FIRST = "front_first"
BACK_FIRST = "back_first"
@dataclass(frozen = True)
class PlanOptions:
# Compute buffer + CUDA context + scratch per device. 1 GiB OOMed on prefill (measured 1.07 GiB).
overhead_bytes_per_device: int = (3 * GIB) // 2
# GPU-resident bytes outside the layout (mmproj, MTP draft), charged once to the pooled budget.
extra_resident_bytes: int = 0
# Layer-split cost per device after the first, applied as max(0, n_gpus - 1); kept out of the per-device overhead, which would withhold a GiB on one card
pipeline_overhead_bytes: int = 0
# Host RAM this planner refuses to spend, so a spill does not push the box into swap.
host_ram_headroom_bytes: int = 2 * GIB
context_policy: ContextPolicy = ContextPolicy.NEVER_REDUCE
min_ctx: int = 4096
spill_order: SpillOrder = SpillOrder.LARGEST_FIRST
allow_lm_head_spill: bool = True
# Spilled decode runs on CPU (ggml offloads ops only at batch >= 32), so cost scales with cores.
host: HostProfile = field(default_factory = HostProfile)
# q8_0 KV is ~35% slower; without GGML_CUDA_FA_ALL_QUANTS mismatched K/V pairs fall to CPU.
allow_kv_quant: bool = False
kv_quant_type: str = "q8_0"
# -nkvo (or LLAMA_ARG_KV_OFFLOAD false) puts the whole KV cache on host (llama-kv-cache.cpp), so it is not charged to VRAM
kv_on_host: bool = False
lazy_read_bytes: int = 0
@dataclass(frozen = True)
class Plan:
"""False means emit nothing new: the planner abstained or the load needs no help; llama.cpp defaults then apply, always safe"""
changed: bool = False
n_ctx: int = 0
ot_patterns: tuple[str, ...] = field(default_factory = tuple)
load_mode_none: bool = False
cache_type_k: Optional[str] = None
cache_type_v: Optional[str] = None
spilled_blocks: tuple[int, ...] = field(default_factory = tuple)
spilled_lm_head: bool = False
# mmap stays: it makes an over-commit pageable instead of OOM-killed.
insufficient: bool = False
vram_bytes: int = 0
host_bytes: int = 0
# Predicted extra ms per token versus fully resident on the planned host; 0.0 when nothing spills.
predicted_gen_penalty_ms: float = 0.0
reason: str = ""
@property
def spills_anything(self) -> bool:
return bool(self.spilled_blocks) or self.spilled_lm_head
def _usable_vram(vram_bytes_per_device: Sequence[int], opts: PlanOptions) -> int:
"""Total VRAM: each device pays overhead; a split pays per device after the first; the pool pays
once."""
pooled = sum(max(0, v - opts.overhead_bytes_per_device) for v in vram_bytes_per_device)
split = max(0, len(vram_bytes_per_device) - 1) * max(0, opts.pipeline_overhead_bytes)
return pooled - split - max(0, opts.extra_resident_bytes)
def _select_blocks(
blocks: Iterable[BlockLayout], deficit: int, order: SpillOrder
) -> tuple[list[BlockLayout], int]:
"""Blocks to spill to free at least ``deficit``, and what they actually free."""
remaining = [b for b in blocks if b.spillable_bytes > 0]
if order is SpillOrder.FRONT_FIRST:
remaining.sort(key = lambda b: b.index)
elif order is SpillOrder.BACK_FIRST:
remaining.sort(key = lambda b: -b.index)
else:
remaining.sort(key = lambda b: -b.spillable_bytes)
chosen: list[BlockLayout] = []
freed = 0
while freed < deficit and remaining:
if order is SpillOrder.LARGEST_FIRST:
residual = deficit - freed
# Prefer the SMALLEST block that closes the gap: the last pick must not overshoot by a whole large block.
covering = [b for b in remaining if b.spillable_bytes >= residual]
pick = min(covering, key = lambda b: b.spillable_bytes) if covering else remaining[0]
else:
pick = remaining[0]
remaining.remove(pick)
chosen.append(pick)
freed += pick.spillable_bytes
return chosen, freed
def _spill_penalty_ms(
layout: ModelLayout, chosen: Sequence[BlockLayout], spill_lm_head: bool, host: HostProfile
) -> float:
"""Spilled weights run on the CPU, so cost tracks host cores; MoE pays only for routed experts."""
groups: list[TensorGroup] = []
spilled = sum(b.spillable_bytes for b in chosen)
if spilled:
if layout.is_moe and layout.n_expert and layout.n_expert_used:
groups.append(
TensorGroup(
"experts",
spilled,
Access.SCATTERED,
activation_fraction = layout.n_expert_used / layout.n_expert,
)
)
else:
groups.append(TensorGroup("ffn", spilled, Access.CONTIGUOUS))
if spill_lm_head and layout.lm_head_bytes:
groups.append(TensorGroup("lm_head", layout.lm_head_bytes, Access.SINGLE_MATVEC))
if not groups:
return 0.0
return generation_penalty_ms(Placement(host_groups = groups), host)
def _kv_elem_bytes(quantised: bool) -> int:
return 1 if quantised else 2
def cache_bytes(
layout: ModelLayout,
n_ctx: int,
*,
kv_quantised: bool = False,
kv_bytes_floor: int = 0,
) -> int:
"""Never below the caller's measured floor, since the layout's f16 product can undercount."""
return max(layout.kv_bytes(n_ctx, _kv_elem_bytes(kv_quantised)), max(0, kv_bytes_floor))
def resident_floor_bytes(
layout: ModelLayout,
n_ctx: int,
*,
kv_quantised: bool = False,
kv_bytes_floor: int = 0,
kv_on_host: bool = False,
) -> int:
"""Minimum VRAM with every spillable tensor on host; below it only a smaller quant or context helps."""
if kv_on_host:
# Both caches follow the same scalar, so neither is VRAM here.
return layout.block_resident_bytes + layout.lm_head_bytes + layout.other_resident_bytes
return (
layout.block_resident_bytes
+ layout.lm_head_bytes
+ layout.other_resident_bytes
+ layout.recurrent_bytes
+ cache_bytes(layout, n_ctx, kv_quantised = kv_quantised, kv_bytes_floor = kv_bytes_floor)
)
def all_resident_bytes(
layout: ModelLayout,
n_ctx: int,
*,
kv_quantised: bool = False,
kv_bytes_floor: int = 0,
kv_on_host: bool = False,
) -> int:
"""VRAM needed if nothing spills; token_embd excluded as llama.cpp pins it to the CPU."""
return (
resident_floor_bytes(
layout,
n_ctx,
kv_quantised = kv_quantised,
kv_bytes_floor = kv_bytes_floor,
kv_on_host = kv_on_host,
)
+ layout.spillable_bytes
)
def max_context_for(
layout: ModelLayout,
vram_bytes_per_device: Sequence[int],
*,
spill_all_ffn: bool = False,
spill_lm_head: bool = False,
kv_quantised: bool = False,
opts: Optional[PlanOptions] = None,
) -> int:
"""Largest context whose cache fits, rounded down to 256 as CUDA wants."""
opts = opts or PlanOptions()
if not layout.complete or layout.kv_bytes_per_token_f16 <= 0:
return 0
fixed = (
layout.block_resident_bytes
+ layout.other_resident_bytes
+ layout.recurrent_bytes
+ (0 if spill_lm_head else layout.lm_head_bytes)
+ (0 if spill_all_ffn else layout.spillable_bytes)
)
free = _usable_vram(vram_bytes_per_device, opts) - fixed
if free <= 0:
return 0
per_token = layout.kv_bytes_per_token_f16 * _kv_elem_bytes(kv_quantised) // 2
if per_token <= 0:
return 0
ctx = (free // per_token) // 256 * 256
if layout.n_ctx_train:
ctx = min(ctx, layout.n_ctx_train)
return max(0, ctx)
def plan_placement(
layout: ModelLayout,
vram_bytes_per_device: Sequence[int],
host_ram_bytes: Optional[int],
requested_ctx: int,
*,
opts: Optional[PlanOptions] = None,
kv_bytes_floor: int = 0,
split_weights_per_device: Sequence[float] = (),
kv_layer_weights: Sequence[int] = (),
) -> Plan:
"""Rungs ordered in ms per token, not percent (percent losses use different baselines); split_weights_per_device is raw free VRAM, not the budget"""
opts = opts or PlanOptions()
if not layout.complete or not vram_bytes_per_device:
return Plan(reason = "layout or device inventory incomplete, leaving llama.cpp defaults")
if opts.host.unified_memory:
# Unified memory: spilling frees nothing, and Metal keeps mmap zero-copy.
return Plan(reason = "unified memory host, spilling frees no device memory")
budget = _usable_vram(vram_bytes_per_device, opts)
if budget <= 0:
return Plan(reason = "no creditable VRAM after per-device overhead and reserved allocations")
n_ctx = requested_ctx if requested_ctx > 0 else layout.n_ctx_train
if layout.n_ctx_train:
n_ctx = min(n_ctx, layout.n_ctx_train)
if n_ctx <= 0:
return Plan(reason = "no usable context length")
# PREFER_RESIDENT runs before the ladder: a smaller resident context beats a larger spilled one.
if (
opts.context_policy is ContextPolicy.PREFER_RESIDENT
and all_resident_bytes(
layout, n_ctx, kv_bytes_floor = kv_bytes_floor, kv_on_host = opts.kv_on_host
)
> budget
):
shrunk = max_context_for(layout, vram_bytes_per_device, opts = opts)
if shrunk >= opts.min_ctx:
return _finish(
layout,
opts,
min(shrunk, n_ctx),
[],
False,
host_ram_bytes,
reason = (
f"shrank context {n_ctx} -> {min(shrunk, n_ctx)} to keep every tensor "
"resident, which outruns a larger spilled context"
),
)
for quantised in _kv_modes(opts):
plan = _plan_at(
layout,
opts,
n_ctx,
budget,
host_ram_bytes,
quantised,
kv_bytes_floor,
vram_bytes_per_device,
split_weights_per_device or vram_bytes_per_device,
kv_layer_weights,
)
if plan is not None:
return plan
# Nothing fit at the requested context. Only now may FIT_ONLY shrink it.
if opts.context_policy in (ContextPolicy.FIT_ONLY, ContextPolicy.PREFER_RESIDENT):
for quantised in _kv_modes(opts):
shrunk = max_context_for(
layout,
vram_bytes_per_device,
spill_all_ffn = True,
spill_lm_head = opts.allow_lm_head_spill,
kv_quantised = quantised,
opts = opts,
)
shrunk = min(shrunk, n_ctx)
if shrunk >= opts.min_ctx:
plan = _plan_at(
layout,
opts,
shrunk,
budget,
host_ram_bytes,
quantised,
kv_bytes_floor,
vram_bytes_per_device,
split_weights_per_device or vram_bytes_per_device,
kv_layer_weights,
)
if plan is not None:
return plan
floor = resident_floor_bytes(
layout, n_ctx, kv_bytes_floor = kv_bytes_floor, kv_on_host = opts.kv_on_host
)
return Plan(
changed = False,
n_ctx = n_ctx,
insufficient = True,
vram_bytes = floor,
reason = (
f"even with every spillable tensor on the host the load needs "
f"{floor / GIB:.2f} GiB of VRAM against {budget / GIB:.2f} GiB usable; "
"keeping mmap so llama.cpp can page rather than be OOM-killed. "
"A smaller quant or a shorter context is the fix, not more offload"
),
)
def _kv_modes(opts: PlanOptions) -> tuple[bool, ...]:
"""f16 first, then q8_0 only if the caller opted in."""
return (False, True) if opts.allow_kv_quant else (False,)
def _device_slots(n_slots: int, split_weights: Sequence[float]) -> list[list[int]]:
"""Mirrors llama.cpp's default tensor split: free VRAM prefix-summed and normalised per device."""
def f32(value: float) -> float:
return struct.unpack("=f", struct.pack("=f", value))[0]
weights = [max(0, v) for v in split_weights]
total = sum(weights)
if total <= 0:
return [list(range(n_slots))] + [[] for _ in weights[1:]]
cumulative: list[float] = []
running = f32(0.0)
for w in weights:
running = f32(running + f32(w))
cumulative.append(running)
cumulative = [f32(value / running) for value in cumulative]
slots: list[list[int]] = [[] for _ in weights]
for row in range(n_slots):
fraction = f32(f32(row) / f32(n_slots))
# std::upper_bound: first cumulative strictly greater than fraction.
device = next((i for i, c in enumerate(cumulative) if c > fraction), len(weights) - 1)
slots[device].append(row)
return slots
def _per_device_usage(
layout: ModelLayout,
opts: PlanOptions,
n_ctx: int,
spilled_indices: set[int],
spill_lm_head: bool,
vram_bytes_per_device: Sequence[int],
*,
quantised: bool,
kv_bytes_floor: int,
split_weights_per_device: Sequence[float] = (),
kv_layer_weights: Sequence[int] = (),
) -> tuple[Optional[str], list[int], list[list[int]]]:
if len(vram_bytes_per_device) <= 1:
return None, [], []
# Recurrent, partial-attention or SWA layouts matter only for an even split; no vector means abstain.
uneven_cache = (
layout.recurrent_bytes > 0 or layout.n_attention_layers != layout.n_layers or layout.has_swa
)
weights = [max(0, int(w)) for w in kv_layer_weights]
if len(weights) != layout.n_layers or not any(weights):
weights = []
if uneven_cache and not weights:
if layout.recurrent_bytes > 0:
return "the recurrent state's per-layer split is not visible in the layout", [], []
if layout.n_attention_layers != layout.n_layers:
return (
f"only {layout.n_attention_layers} of {layout.n_layers} layers hold a cache "
"and the layout does not say which",
[],
[],
)
return (
"the cache is per-layer uneven (sliding-window attention) and no per-layer "
"vector was supplied to say which layers are full-context",
[],
[],
)
if layout.has_excluded_blocks:
return "the GGUF carries trailing blocks that shift llama.cpp's row count", [], []
n_slots = layout.n_layers + 1
if n_slots <= 1:
return None, [], []
cache = (
0
if opts.kv_on_host
else cache_bytes(layout, n_ctx, kv_quantised = quantised, kv_bytes_floor = kv_bytes_floor)
)
# Scaled without under-booking the caller's total. Uniform when unsupplied.
total_weight = sum(weights)
if weights and total_weight > 0:
kv_by_layer = [(cache * w + total_weight - 1) // total_weight for w in weights]
else:
per = (cache + layout.n_layers - 1) // layout.n_layers if layout.n_layers else 0
kv_by_layer = [per] * layout.n_layers
by_index = {b.index: b for b in layout.blocks}
output_row_bytes = layout.other_resident_bytes + (0 if spill_lm_head else layout.lm_head_bytes)
slots = _device_slots(n_slots, split_weights_per_device or vram_bytes_per_device)
usage: list[int] = []
for device, rows in enumerate(slots):
used = 0
for row in rows:
if row == n_slots - 1:
used += output_row_bytes
continue
block = by_index.get(row)
if block is None:
continue
used += block.resident_bytes
if row < len(kv_by_layer):
used += kv_by_layer[row]
if row not in spilled_indices:
used += block.spillable_bytes
# Unlisted tensors sit on the main device, devices[0] after -sm none pruning.
if device == 0:
used += max(0, opts.extra_resident_bytes)
usage.append(used)
return None, usage, slots
def _per_device_shortfall(
layout: ModelLayout,
opts: PlanOptions,
n_ctx: int,
spilled_indices: set[int],
spill_lm_head: bool,
vram_bytes_per_device: Sequence[int],
*,
quantised: bool,
kv_bytes_floor: int,
split_weights_per_device: Sequence[float] = (),
kv_layer_weights: Sequence[int] = (),
) -> Optional[str]:
"""A pooled budget is not a per-device fit: rows split by free VRAM, not budget bytes; a per-device shortfall is a hard throw, unguarded under --fit off"""
error, usage, slots = _per_device_usage(
layout,
opts,
n_ctx,
spilled_indices,
spill_lm_head,
vram_bytes_per_device,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
split_weights_per_device = split_weights_per_device,
kv_layer_weights = kv_layer_weights,
)
if error is not None:
return error
for device, (used, rows) in enumerate(zip(usage, slots)):
fixed_reserve = max(0, opts.overhead_bytes_per_device)
if device > 0:
fixed_reserve += max(0, opts.pipeline_overhead_bytes)
raw_vram = max(0, vram_bytes_per_device[device])
headroom = max(0, raw_vram - fixed_reserve)
if used + fixed_reserve > raw_vram:
return (
f"device {device} would still hold {used / GIB:.2f} GiB of its "
f"{len(rows)}-row share against {headroom / GIB:.2f} GiB usable"
)
return None
def _select_blocks_per_device(
layout: ModelLayout,
opts: PlanOptions,
n_ctx: int,
vram_bytes_per_device: Sequence[int],
*,
quantised: bool,
kv_bytes_floor: int,
spill_lm_head: bool = False,
split_weights_per_device: Sequence[float] = (),
kv_layer_weights: Sequence[int] = (),
) -> Optional[list[BlockLayout]]:
error, usage, slots = _per_device_usage(
layout,
opts,
n_ctx,
set(),
spill_lm_head,
vram_bytes_per_device,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
split_weights_per_device = split_weights_per_device,
kv_layer_weights = kv_layer_weights,
)
if error is not None:
return None
by_index = {block.index: block for block in layout.blocks}
chosen: list[BlockLayout] = []
for device, (used, rows) in enumerate(zip(usage, slots)):
fixed_reserve = max(0, opts.overhead_bytes_per_device)
if device > 0:
fixed_reserve += max(0, opts.pipeline_overhead_bytes)
deficit = used + fixed_reserve - max(0, vram_bytes_per_device[device])
if deficit <= 0:
continue
candidates = [
by_index[row] for row in rows if row in by_index and by_index[row].spillable_bytes > 0
]
local, freed = _select_blocks(candidates, deficit, opts.spill_order)
if freed < deficit:
return None
chosen.extend(local)
if (
_per_device_shortfall(
layout,
opts,
n_ctx,
{block.index for block in chosen},
spill_lm_head,
vram_bytes_per_device,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
split_weights_per_device = split_weights_per_device,
kv_layer_weights = kv_layer_weights,
)
is not None
):
return None
return chosen
def _plan_at(
layout: ModelLayout,
opts: PlanOptions,
n_ctx: int,
budget: int,
host_ram_bytes: Optional[int],
quantised: bool,
kv_bytes_floor: int = 0,
vram_bytes_per_device: Sequence[int] = (),
split_weights_per_device: Sequence[float] = (),
kv_layer_weights: Sequence[int] = (),
) -> Optional[Plan]:
"""One pass of the ladder at a fixed context and cache dtype."""
needed = all_resident_bytes(
layout,
n_ctx,
kv_quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
kv_on_host = opts.kv_on_host,
)
if needed <= budget:
return _finish(
layout,
opts,
n_ctx,
[],
False,
host_ram_bytes,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
reason = (
f"the whole load fits in VRAM ({needed / GIB:.2f} of "
f"{budget / GIB:.2f} GiB usable), so nothing is spilled"
),
)
deficit = needed - budget
chosen, freed = _select_blocks(layout.blocks, deficit, opts.spill_order)
if freed >= deficit:
if len(vram_bytes_per_device) > 1:
per_device = _select_blocks_per_device(
layout,
opts,
n_ctx,
vram_bytes_per_device,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
split_weights_per_device = split_weights_per_device,
kv_layer_weights = kv_layer_weights,
)
if per_device is not None:
per_device_freed = sum(block.spillable_bytes for block in per_device)
if needed - per_device_freed <= budget:
chosen = per_device
freed = per_device_freed
uneven = _per_device_shortfall(
layout,
opts,
n_ctx,
{b.index for b in chosen},
False,
vram_bytes_per_device,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
split_weights_per_device = split_weights_per_device,
kv_layer_weights = kv_layer_weights,
)
if uneven is not None:
if opts.allow_lm_head_spill and layout.lm_head_bytes:
with_head = _select_blocks_per_device(
layout,
opts,
n_ctx,
vram_bytes_per_device,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
spill_lm_head = True,
split_weights_per_device = split_weights_per_device,
kv_layer_weights = kv_layer_weights,
)
if with_head is not None:
with_head_freed = sum(block.spillable_bytes for block in with_head)
with_head_freed += layout.lm_head_bytes
if needed - with_head_freed <= budget:
return _finish(
layout,
opts,
n_ctx,
with_head,
True,
host_ram_bytes,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
reason = (
"spilled the output head after its device could not cover "
"the local shortfall with FFN blocks alone"
),
)
return Plan(
n_ctx = n_ctx,
reason = (
f"the selected spill still does not fit device by device: {uneven}; "
"leaving llama.cpp's own fitter to place it"
),
)
return _finish(
layout,
opts,
n_ctx,
chosen,
False,
host_ram_bytes,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
reason = (
f"spilled the FFN of {len(chosen)} of {len(layout.blocks)} blocks "
f"({freed / GIB:.2f} GiB) to cover a {deficit / GIB:.2f} GiB deficit, "
"keeping the KV cache resident"
),
)
# lm_head spills last: cheaper once FFN offload is host-bandwidth-bound.
if opts.allow_lm_head_spill and layout.lm_head_bytes:
if freed + layout.lm_head_bytes >= deficit:
uneven = _per_device_shortfall(
layout,
opts,
n_ctx,
{b.index for b in chosen},
True,
vram_bytes_per_device,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
split_weights_per_device = split_weights_per_device,
kv_layer_weights = kv_layer_weights,
)
if uneven is not None:
return Plan(
n_ctx = n_ctx,
reason = (
"spilling every block and lm_head still does not fit device by "
f"device: {uneven}; leaving llama.cpp's own fitter to place it"
),
)
return _finish(
layout,
opts,
n_ctx,
chosen,
True,
host_ram_bytes,
quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
reason = (
f"spilled every block's FFN ({freed / GIB:.2f} GiB) plus lm_head "
f"({layout.lm_head_bytes / GIB:.2f} GiB) to cover a "
f"{deficit / GIB:.2f} GiB deficit"
),
)
return None
def _finish(
layout: ModelLayout,
opts: PlanOptions,
n_ctx: int,
chosen: list[BlockLayout],
spill_lm_head: bool,
host_ram_bytes: Optional[int],
*,
quantised: bool = False,
kv_bytes_floor: int = 0,
reason: str = "",
) -> Plan:
"""Assemble patterns, decide the load mode, and account for both sides."""
patterns: list[str] = []
indices = sorted(b.index for b in chosen)
if indices:
# Global pattern only if no dropped blocks: \d+ would also match trailing nextn/MTP blocks.
spillable = [b.index for b in layout.blocks if b.spillable_bytes > 0]
all_of_them = set(indices) == set(spillable) and not layout.has_excluded_blocks
patterns.append(spill_pattern_for(layout, None if all_of_them else indices))
if spill_lm_head:
patterns.append(LM_HEAD_PATTERN)
spilled_bytes = sum(b.spillable_bytes for b in chosen) + (
layout.lm_head_bytes if spill_lm_head else 0
)
lazy = min(max(0, int(opts.lazy_read_bytes or 0)), layout.per_layer_embd_bytes)
host_bytes = layout.token_embd_bytes + layout.per_layer_embd_bytes - lazy + spilled_bytes
if opts.kv_on_host:
# -nkvo cache lives in host RAM; the mmap decision must count it.
host_bytes += (
cache_bytes(layout, n_ctx, kv_quantised = quantised, kv_bytes_floor = kv_bytes_floor)
+ layout.recurrent_bytes
)
vram_bytes = (
all_resident_bytes(
layout,
n_ctx,
kv_quantised = quantised,
kv_bytes_floor = kv_bytes_floor,
kv_on_host = opts.kv_on_host,
)
- spilled_bytes
)
# mmap costs 2-4.6x on host weight reads; disable only when host RAM fits the host side, else mmap keeps an over-commit pageable
if host_ram_bytes is None:
load_mode_none = False
else:
load_mode_none = host_bytes <= max(0, host_ram_bytes - opts.host_ram_headroom_bytes)
cache_type = opts.kv_quant_type if quantised else None
changed = bool(patterns) or load_mode_none or cache_type is not None
return Plan(
changed = changed,
n_ctx = n_ctx,
ot_patterns = tuple(patterns),
load_mode_none = load_mode_none,
# Matched K/V pairs only: unmatched ones are not compiled without GGML_CUDA_FA_ALL_QUANTS and fall back to CPU
cache_type_k = cache_type,
cache_type_v = cache_type,
spilled_blocks = tuple(indices),
spilled_lm_head = spill_lm_head,
vram_bytes = vram_bytes,
host_bytes = host_bytes,
predicted_gen_penalty_ms = _spill_penalty_ms(layout, chosen, spill_lm_head, opts.host),
reason = reason,
)
def plan_to_args(plan: Plan) -> list[str]:
"""The launch flags for ``plan``. Empty when it changes nothing."""
args: list[str] = []
for pattern in plan.ot_patterns:
args.extend(["-ot", f"{pattern}=CPU"])
if plan.load_mode_none:
args.extend(["--load-mode", "none"])
if plan.cache_type_k and plan.cache_type_v:
args.extend(["--cache-type-k", plan.cache_type_k])
args.extend(["--cache-type-v", plan.cache_type_v])
return args
_SMART_OFFLOAD_ON = ("1", "true", "yes", "on", "enabled")
def smart_offload_enabled(env: Optional[Mapping[str, str]] = None) -> bool:
"""Off unless enabled: it prices only generation, not prefill, and was slower on most planned cells."""
raw = (os.environ if env is None else env).get("UNSLOTH_SMART_OFFLOAD")
if raw is None:
return False
return str(raw).strip().lower() in _SMART_OFFLOAD_ON