Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

30.4 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Spill weights with -ot, never the KV cache: spilling a layer moves its cache to host RAM too."""

from __future__ import annotations

import os
import struct
from dataclasses import dataclass, field
from enum import Enum
from typing import Iterable, Mapping, Optional, Sequence

from core.inference.offload_cost_model import (
    Access,
    HostProfile,
    Placement,
    TensorGroup,
    generation_penalty_ms,
)
from core.inference.offload_layout import (
    LM_HEAD_PATTERN,
    BlockLayout,
    ModelLayout,
    spill_pattern_for,
)

GIB = 1024**3
MIB = 1024**2


class ContextPolicy(Enum):
    """Whether the planner may shrink a context the user asked for.

    llama.cpp's fitter shrinks context before spilling anything, and on
    throughput grounds that is right: a resident smaller context beats a spilled
    larger one. But context is a user-visible feature, not a free variable, so
    quietly trading it away is not a safe default.
    """

    NEVER_REDUCE = "never"
    # Shrink if that avoids spilling entirely.
    PREFER_RESIDENT = "prefer_resident"
    # Shrink only when no rung of the ladder fits.
    FIT_ONLY = "fit_only"


class SpillOrder(Enum):
    """Which blocks to spill when only some are needed.

    UNMEASURED: every -ot measurement so far spilled all blocks or none, so the
    ordering is justified by byte-minimality alone, not by benchmark. Contiguous
    runs may schedule better (adjacent host blocks can merge into one graph
    split), which would favour FRONT/BACK over LARGEST. Hence configurable.
    """

    # Best-fit-decreasing: fewest blocks and least overshoot (overshoot costs bandwidth).
    LARGEST_FIRST = "largest_first"
    FRONT_FIRST = "front_first"
    BACK_FIRST = "back_first"


@dataclass(frozen = True)
class PlanOptions:
    # Compute buffer + CUDA context + scratch per device. 1 GiB OOMed on prefill (measured 1.07 GiB).
    overhead_bytes_per_device: int = (3 * GIB) // 2
    # GPU-resident bytes outside the layout (mmproj, MTP draft), charged once to the pooled budget.
    extra_resident_bytes: int = 0
    # Layer-split cost per device after the first, applied as max(0, n_gpus - 1); kept out of the per-device overhead, which would withhold a GiB on one card
    pipeline_overhead_bytes: int = 0
    # Host RAM this planner refuses to spend, so a spill does not push the box into swap.
    host_ram_headroom_bytes: int = 2 * GIB
    context_policy: ContextPolicy = ContextPolicy.NEVER_REDUCE
    min_ctx: int = 4096
    spill_order: SpillOrder = SpillOrder.LARGEST_FIRST
    allow_lm_head_spill: bool = True
    # Spilled decode runs on CPU (ggml offloads ops only at batch >= 32), so cost scales with cores.
    host: HostProfile = field(default_factory = HostProfile)
    # q8_0 KV is ~35% slower; without GGML_CUDA_FA_ALL_QUANTS mismatched K/V pairs fall to CPU.
    allow_kv_quant: bool = False
    kv_quant_type: str = "q8_0"
    # -nkvo (or LLAMA_ARG_KV_OFFLOAD false) puts the whole KV cache on host (llama-kv-cache.cpp), so it is not charged to VRAM
    kv_on_host: bool = False
    lazy_read_bytes: int = 0


@dataclass(frozen = True)
class Plan:
    """False means emit nothing new: the planner abstained or the load needs no help; llama.cpp defaults then apply, always safe"""

    changed: bool = False
    n_ctx: int = 0
    ot_patterns: tuple[str, ...] = field(default_factory = tuple)
    load_mode_none: bool = False
    cache_type_k: Optional[str] = None
    cache_type_v: Optional[str] = None
    spilled_blocks: tuple[int, ...] = field(default_factory = tuple)
    spilled_lm_head: bool = False
    # mmap stays: it makes an over-commit pageable instead of OOM-killed.
    insufficient: bool = False
    vram_bytes: int = 0
    host_bytes: int = 0
    # Predicted extra ms per token versus fully resident on the planned host; 0.0 when nothing spills.
    predicted_gen_penalty_ms: float = 0.0
    reason: str = ""

    @property
    def spills_anything(self) -> bool:
        return bool(self.spilled_blocks) or self.spilled_lm_head


def _usable_vram(vram_bytes_per_device: Sequence[int], opts: PlanOptions) -> int:
    """Total VRAM: each device pays overhead; a split pays per device after the first; the pool pays
    once."""
    pooled = sum(max(0, v - opts.overhead_bytes_per_device) for v in vram_bytes_per_device)
    split = max(0, len(vram_bytes_per_device) - 1) * max(0, opts.pipeline_overhead_bytes)
    return pooled - split - max(0, opts.extra_resident_bytes)


def _select_blocks(
    blocks: Iterable[BlockLayout], deficit: int, order: SpillOrder
) -> tuple[list[BlockLayout], int]:
    """Blocks to spill to free at least ``deficit``, and what they actually free."""
    remaining = [b for b in blocks if b.spillable_bytes > 0]
    if order is SpillOrder.FRONT_FIRST:
        remaining.sort(key = lambda b: b.index)
    elif order is SpillOrder.BACK_FIRST:
        remaining.sort(key = lambda b: -b.index)
    else:
        remaining.sort(key = lambda b: -b.spillable_bytes)

    chosen: list[BlockLayout] = []
    freed = 0
    while freed < deficit and remaining:
        if order is SpillOrder.LARGEST_FIRST:
            residual = deficit - freed
            # Prefer the SMALLEST block that closes the gap: the last pick must not overshoot by a whole large block.
            covering = [b for b in remaining if b.spillable_bytes >= residual]
            pick = min(covering, key = lambda b: b.spillable_bytes) if covering else remaining[0]
        else:
            pick = remaining[0]
        remaining.remove(pick)
        chosen.append(pick)
        freed += pick.spillable_bytes
    return chosen, freed


def _spill_penalty_ms(
    layout: ModelLayout, chosen: Sequence[BlockLayout], spill_lm_head: bool, host: HostProfile
) -> float:
    """Spilled weights run on the CPU, so cost tracks host cores; MoE pays only for routed experts."""
    groups: list[TensorGroup] = []
    spilled = sum(b.spillable_bytes for b in chosen)
    if spilled:
        if layout.is_moe and layout.n_expert and layout.n_expert_used:
            groups.append(
                TensorGroup(
                    "experts",
                    spilled,
                    Access.SCATTERED,
                    activation_fraction = layout.n_expert_used / layout.n_expert,
                )
            )
        else:
            groups.append(TensorGroup("ffn", spilled, Access.CONTIGUOUS))
    if spill_lm_head and layout.lm_head_bytes:
        groups.append(TensorGroup("lm_head", layout.lm_head_bytes, Access.SINGLE_MATVEC))
    if not groups:
        return 0.0
    return generation_penalty_ms(Placement(host_groups = groups), host)


def _kv_elem_bytes(quantised: bool) -> int:
    return 1 if quantised else 2


def cache_bytes(
    layout: ModelLayout,
    n_ctx: int,
    *,
    kv_quantised: bool = False,
    kv_bytes_floor: int = 0,
) -> int:
    """Never below the caller's measured floor, since the layout's f16 product can undercount."""
    return max(layout.kv_bytes(n_ctx, _kv_elem_bytes(kv_quantised)), max(0, kv_bytes_floor))


def resident_floor_bytes(
    layout: ModelLayout,
    n_ctx: int,
    *,
    kv_quantised: bool = False,
    kv_bytes_floor: int = 0,
    kv_on_host: bool = False,
) -> int:
    """Minimum VRAM with every spillable tensor on host; below it only a smaller quant or context helps."""
    if kv_on_host:
        # Both caches follow the same scalar, so neither is VRAM here.
        return layout.block_resident_bytes + layout.lm_head_bytes + layout.other_resident_bytes
    return (
        layout.block_resident_bytes
        + layout.lm_head_bytes
        + layout.other_resident_bytes
        + layout.recurrent_bytes
        + cache_bytes(layout, n_ctx, kv_quantised = kv_quantised, kv_bytes_floor = kv_bytes_floor)
    )


def all_resident_bytes(
    layout: ModelLayout,
    n_ctx: int,
    *,
    kv_quantised: bool = False,
    kv_bytes_floor: int = 0,
    kv_on_host: bool = False,
) -> int:
    """VRAM needed if nothing spills; token_embd excluded as llama.cpp pins it to the CPU."""
    return (
        resident_floor_bytes(
            layout,
            n_ctx,
            kv_quantised = kv_quantised,
            kv_bytes_floor = kv_bytes_floor,
            kv_on_host = kv_on_host,
        )
        + layout.spillable_bytes
    )


def max_context_for(
    layout: ModelLayout,
    vram_bytes_per_device: Sequence[int],
    *,
    spill_all_ffn: bool = False,
    spill_lm_head: bool = False,
    kv_quantised: bool = False,
    opts: Optional[PlanOptions] = None,
) -> int:
    """Largest context whose cache fits, rounded down to 256 as CUDA wants."""
    opts = opts or PlanOptions()
    if not layout.complete or layout.kv_bytes_per_token_f16 <= 0:
        return 0
    fixed = (
        layout.block_resident_bytes
        + layout.other_resident_bytes
        + layout.recurrent_bytes
        + (0 if spill_lm_head else layout.lm_head_bytes)
        + (0 if spill_all_ffn else layout.spillable_bytes)
    )
    free = _usable_vram(vram_bytes_per_device, opts) - fixed
    if free <= 0:
        return 0
    per_token = layout.kv_bytes_per_token_f16 * _kv_elem_bytes(kv_quantised) // 2
    if per_token <= 0:
        return 0
    ctx = (free // per_token) // 256 * 256
    if layout.n_ctx_train:
        ctx = min(ctx, layout.n_ctx_train)
    return max(0, ctx)


def plan_placement(
    layout: ModelLayout,
    vram_bytes_per_device: Sequence[int],
    host_ram_bytes: Optional[int],
    requested_ctx: int,
    *,
    opts: Optional[PlanOptions] = None,
    kv_bytes_floor: int = 0,
    split_weights_per_device: Sequence[float] = (),
    kv_layer_weights: Sequence[int] = (),
) -> Plan:
    """Rungs ordered in ms per token, not percent (percent losses use different baselines); split_weights_per_device is raw free VRAM, not the budget"""
    opts = opts or PlanOptions()

    if not layout.complete or not vram_bytes_per_device:
        return Plan(reason = "layout or device inventory incomplete, leaving llama.cpp defaults")
    if opts.host.unified_memory:
        # Unified memory: spilling frees nothing, and Metal keeps mmap zero-copy.
        return Plan(reason = "unified memory host, spilling frees no device memory")
    budget = _usable_vram(vram_bytes_per_device, opts)
    if budget <= 0:
        return Plan(reason = "no creditable VRAM after per-device overhead and reserved allocations")

    n_ctx = requested_ctx if requested_ctx > 0 else layout.n_ctx_train
    if layout.n_ctx_train:
        n_ctx = min(n_ctx, layout.n_ctx_train)
    if n_ctx <= 0:
        return Plan(reason = "no usable context length")

    # PREFER_RESIDENT runs before the ladder: a smaller resident context beats a larger spilled one.
    if (
        opts.context_policy is ContextPolicy.PREFER_RESIDENT
        and all_resident_bytes(
            layout, n_ctx, kv_bytes_floor = kv_bytes_floor, kv_on_host = opts.kv_on_host
        )
        > budget
    ):
        shrunk = max_context_for(layout, vram_bytes_per_device, opts = opts)
        if shrunk >= opts.min_ctx:
            return _finish(
                layout,
                opts,
                min(shrunk, n_ctx),
                [],
                False,
                host_ram_bytes,
                reason = (
                    f"shrank context {n_ctx} -> {min(shrunk, n_ctx)} to keep every tensor "
                    "resident, which outruns a larger spilled context"
                ),
            )

    for quantised in _kv_modes(opts):
        plan = _plan_at(
            layout,
            opts,
            n_ctx,
            budget,
            host_ram_bytes,
            quantised,
            kv_bytes_floor,
            vram_bytes_per_device,
            split_weights_per_device or vram_bytes_per_device,
            kv_layer_weights,
        )
        if plan is not None:
            return plan

    # Nothing fit at the requested context. Only now may FIT_ONLY shrink it.
    if opts.context_policy in (ContextPolicy.FIT_ONLY, ContextPolicy.PREFER_RESIDENT):
        for quantised in _kv_modes(opts):
            shrunk = max_context_for(
                layout,
                vram_bytes_per_device,
                spill_all_ffn = True,
                spill_lm_head = opts.allow_lm_head_spill,
                kv_quantised = quantised,
                opts = opts,
            )
            shrunk = min(shrunk, n_ctx)
            if shrunk >= opts.min_ctx:
                plan = _plan_at(
                    layout,
                    opts,
                    shrunk,
                    budget,
                    host_ram_bytes,
                    quantised,
                    kv_bytes_floor,
                    vram_bytes_per_device,
                    split_weights_per_device or vram_bytes_per_device,
                    kv_layer_weights,
                )
                if plan is not None:
                    return plan

    floor = resident_floor_bytes(
        layout, n_ctx, kv_bytes_floor = kv_bytes_floor, kv_on_host = opts.kv_on_host
    )
    return Plan(
        changed = False,
        n_ctx = n_ctx,
        insufficient = True,
        vram_bytes = floor,
        reason = (
            f"even with every spillable tensor on the host the load needs "
            f"{floor / GIB:.2f} GiB of VRAM against {budget / GIB:.2f} GiB usable; "
            "keeping mmap so llama.cpp can page rather than be OOM-killed. "
            "A smaller quant or a shorter context is the fix, not more offload"
        ),
    )


def _kv_modes(opts: PlanOptions) -> tuple[bool, ...]:
    """f16 first, then q8_0 only if the caller opted in."""
    return (False, True) if opts.allow_kv_quant else (False,)


def _device_slots(n_slots: int, split_weights: Sequence[float]) -> list[list[int]]:
    """Mirrors llama.cpp's default tensor split: free VRAM prefix-summed and normalised per device."""

    def f32(value: float) -> float:
        return struct.unpack("=f", struct.pack("=f", value))[0]

    weights = [max(0, v) for v in split_weights]
    total = sum(weights)
    if total <= 0:
        return [list(range(n_slots))] + [[] for _ in weights[1:]]
    cumulative: list[float] = []
    running = f32(0.0)
    for w in weights:
        running = f32(running + f32(w))
        cumulative.append(running)
    cumulative = [f32(value / running) for value in cumulative]
    slots: list[list[int]] = [[] for _ in weights]
    for row in range(n_slots):
        fraction = f32(f32(row) / f32(n_slots))
        # std::upper_bound: first cumulative strictly greater than fraction.
        device = next((i for i, c in enumerate(cumulative) if c > fraction), len(weights) - 1)
        slots[device].append(row)
    return slots


def _per_device_usage(
    layout: ModelLayout,
    opts: PlanOptions,
    n_ctx: int,
    spilled_indices: set[int],
    spill_lm_head: bool,
    vram_bytes_per_device: Sequence[int],
    *,
    quantised: bool,
    kv_bytes_floor: int,
    split_weights_per_device: Sequence[float] = (),
    kv_layer_weights: Sequence[int] = (),
) -> tuple[Optional[str], list[int], list[list[int]]]:
    if len(vram_bytes_per_device) <= 1:
        return None, [], []
    # Recurrent, partial-attention or SWA layouts matter only for an even split; no vector means abstain.
    uneven_cache = (
        layout.recurrent_bytes > 0 or layout.n_attention_layers != layout.n_layers or layout.has_swa
    )
    weights = [max(0, int(w)) for w in kv_layer_weights]
    if len(weights) != layout.n_layers or not any(weights):
        weights = []
    if uneven_cache and not weights:
        if layout.recurrent_bytes > 0:
            return "the recurrent state's per-layer split is not visible in the layout", [], []
        if layout.n_attention_layers != layout.n_layers:
            return (
                f"only {layout.n_attention_layers} of {layout.n_layers} layers hold a cache "
                "and the layout does not say which",
                [],
                [],
            )
        return (
            "the cache is per-layer uneven (sliding-window attention) and no per-layer "
            "vector was supplied to say which layers are full-context",
            [],
            [],
        )
    if layout.has_excluded_blocks:
        return "the GGUF carries trailing blocks that shift llama.cpp's row count", [], []

    n_slots = layout.n_layers + 1
    if n_slots <= 1:
        return None, [], []
    cache = (
        0
        if opts.kv_on_host
        else cache_bytes(layout, n_ctx, kv_quantised = quantised, kv_bytes_floor = kv_bytes_floor)
    )
    # Scaled without under-booking the caller's total. Uniform when unsupplied.
    total_weight = sum(weights)
    if weights and total_weight > 0:
        kv_by_layer = [(cache * w + total_weight - 1) // total_weight for w in weights]
    else:
        per = (cache + layout.n_layers - 1) // layout.n_layers if layout.n_layers else 0
        kv_by_layer = [per] * layout.n_layers
    by_index = {b.index: b for b in layout.blocks}
    output_row_bytes = layout.other_resident_bytes + (0 if spill_lm_head else layout.lm_head_bytes)

    slots = _device_slots(n_slots, split_weights_per_device or vram_bytes_per_device)
    usage: list[int] = []
    for device, rows in enumerate(slots):
        used = 0
        for row in rows:
            if row == n_slots - 1:
                used += output_row_bytes
                continue
            block = by_index.get(row)
            if block is None:
                continue
            used += block.resident_bytes
            if row < len(kv_by_layer):
                used += kv_by_layer[row]
            if row not in spilled_indices:
                used += block.spillable_bytes
        # Unlisted tensors sit on the main device, devices[0] after -sm none pruning.
        if device == 0:
            used += max(0, opts.extra_resident_bytes)
        usage.append(used)
    return None, usage, slots


def _per_device_shortfall(
    layout: ModelLayout,
    opts: PlanOptions,
    n_ctx: int,
    spilled_indices: set[int],
    spill_lm_head: bool,
    vram_bytes_per_device: Sequence[int],
    *,
    quantised: bool,
    kv_bytes_floor: int,
    split_weights_per_device: Sequence[float] = (),
    kv_layer_weights: Sequence[int] = (),
) -> Optional[str]:
    """A pooled budget is not a per-device fit: rows split by free VRAM, not budget bytes; a per-device shortfall is a hard throw, unguarded under --fit off"""
    error, usage, slots = _per_device_usage(
        layout,
        opts,
        n_ctx,
        spilled_indices,
        spill_lm_head,
        vram_bytes_per_device,
        quantised = quantised,
        kv_bytes_floor = kv_bytes_floor,
        split_weights_per_device = split_weights_per_device,
        kv_layer_weights = kv_layer_weights,
    )
    if error is not None:
        return error
    for device, (used, rows) in enumerate(zip(usage, slots)):
        fixed_reserve = max(0, opts.overhead_bytes_per_device)
        if device > 0:
            fixed_reserve += max(0, opts.pipeline_overhead_bytes)
        raw_vram = max(0, vram_bytes_per_device[device])
        headroom = max(0, raw_vram - fixed_reserve)
        if used + fixed_reserve > raw_vram:
            return (
                f"device {device} would still hold {used / GIB:.2f} GiB of its "
                f"{len(rows)}-row share against {headroom / GIB:.2f} GiB usable"
            )
    return None


def _select_blocks_per_device(
    layout: ModelLayout,
    opts: PlanOptions,
    n_ctx: int,
    vram_bytes_per_device: Sequence[int],
    *,
    quantised: bool,
    kv_bytes_floor: int,
    spill_lm_head: bool = False,
    split_weights_per_device: Sequence[float] = (),
    kv_layer_weights: Sequence[int] = (),
) -> Optional[list[BlockLayout]]:
    error, usage, slots = _per_device_usage(
        layout,
        opts,
        n_ctx,
        set(),
        spill_lm_head,
        vram_bytes_per_device,
        quantised = quantised,
        kv_bytes_floor = kv_bytes_floor,
        split_weights_per_device = split_weights_per_device,
        kv_layer_weights = kv_layer_weights,
    )
    if error is not None:
        return None
    by_index = {block.index: block for block in layout.blocks}
    chosen: list[BlockLayout] = []
    for device, (used, rows) in enumerate(zip(usage, slots)):
        fixed_reserve = max(0, opts.overhead_bytes_per_device)
        if device > 0:
            fixed_reserve += max(0, opts.pipeline_overhead_bytes)
        deficit = used + fixed_reserve - max(0, vram_bytes_per_device[device])
        if deficit <= 0:
            continue
        candidates = [
            by_index[row] for row in rows if row in by_index and by_index[row].spillable_bytes > 0
        ]
        local, freed = _select_blocks(candidates, deficit, opts.spill_order)
        if freed < deficit:
            return None
        chosen.extend(local)
    if (
        _per_device_shortfall(
            layout,
            opts,
            n_ctx,
            {block.index for block in chosen},
            spill_lm_head,
            vram_bytes_per_device,
            quantised = quantised,
            kv_bytes_floor = kv_bytes_floor,
            split_weights_per_device = split_weights_per_device,
            kv_layer_weights = kv_layer_weights,
        )
        is not None
    ):
        return None
    return chosen


def _plan_at(
    layout: ModelLayout,
    opts: PlanOptions,
    n_ctx: int,
    budget: int,
    host_ram_bytes: Optional[int],
    quantised: bool,
    kv_bytes_floor: int = 0,
    vram_bytes_per_device: Sequence[int] = (),
    split_weights_per_device: Sequence[float] = (),
    kv_layer_weights: Sequence[int] = (),
) -> Optional[Plan]:
    """One pass of the ladder at a fixed context and cache dtype."""
    needed = all_resident_bytes(
        layout,
        n_ctx,
        kv_quantised = quantised,
        kv_bytes_floor = kv_bytes_floor,
        kv_on_host = opts.kv_on_host,
    )
    if needed <= budget:
        return _finish(
            layout,
            opts,
            n_ctx,
            [],
            False,
            host_ram_bytes,
            quantised = quantised,
            kv_bytes_floor = kv_bytes_floor,
            reason = (
                f"the whole load fits in VRAM ({needed / GIB:.2f} of "
                f"{budget / GIB:.2f} GiB usable), so nothing is spilled"
            ),
        )

    deficit = needed - budget
    chosen, freed = _select_blocks(layout.blocks, deficit, opts.spill_order)
    if freed >= deficit:
        if len(vram_bytes_per_device) > 1:
            per_device = _select_blocks_per_device(
                layout,
                opts,
                n_ctx,
                vram_bytes_per_device,
                quantised = quantised,
                kv_bytes_floor = kv_bytes_floor,
                split_weights_per_device = split_weights_per_device,
                kv_layer_weights = kv_layer_weights,
            )
            if per_device is not None:
                per_device_freed = sum(block.spillable_bytes for block in per_device)
                if needed - per_device_freed <= budget:
                    chosen = per_device
                    freed = per_device_freed
        uneven = _per_device_shortfall(
            layout,
            opts,
            n_ctx,
            {b.index for b in chosen},
            False,
            vram_bytes_per_device,
            quantised = quantised,
            kv_bytes_floor = kv_bytes_floor,
            split_weights_per_device = split_weights_per_device,
            kv_layer_weights = kv_layer_weights,
        )
        if uneven is not None:
            if opts.allow_lm_head_spill and layout.lm_head_bytes:
                with_head = _select_blocks_per_device(
                    layout,
                    opts,
                    n_ctx,
                    vram_bytes_per_device,
                    quantised = quantised,
                    kv_bytes_floor = kv_bytes_floor,
                    spill_lm_head = True,
                    split_weights_per_device = split_weights_per_device,
                    kv_layer_weights = kv_layer_weights,
                )
                if with_head is not None:
                    with_head_freed = sum(block.spillable_bytes for block in with_head)
                    with_head_freed += layout.lm_head_bytes
                    if needed - with_head_freed <= budget:
                        return _finish(
                            layout,
                            opts,
                            n_ctx,
                            with_head,
                            True,
                            host_ram_bytes,
                            quantised = quantised,
                            kv_bytes_floor = kv_bytes_floor,
                            reason = (
                                "spilled the output head after its device could not cover "
                                "the local shortfall with FFN blocks alone"
                            ),
                        )
            return Plan(
                n_ctx = n_ctx,
                reason = (
                    f"the selected spill still does not fit device by device: {uneven}; "
                    "leaving llama.cpp's own fitter to place it"
                ),
            )
        return _finish(
            layout,
            opts,
            n_ctx,
            chosen,
            False,
            host_ram_bytes,
            quantised = quantised,
            kv_bytes_floor = kv_bytes_floor,
            reason = (
                f"spilled the FFN of {len(chosen)} of {len(layout.blocks)} blocks "
                f"({freed / GIB:.2f} GiB) to cover a {deficit / GIB:.2f} GiB deficit, "
                "keeping the KV cache resident"
            ),
        )

    # lm_head spills last: cheaper once FFN offload is host-bandwidth-bound.
    if opts.allow_lm_head_spill and layout.lm_head_bytes:
        if freed + layout.lm_head_bytes >= deficit:
            uneven = _per_device_shortfall(
                layout,
                opts,
                n_ctx,
                {b.index for b in chosen},
                True,
                vram_bytes_per_device,
                quantised = quantised,
                kv_bytes_floor = kv_bytes_floor,
                split_weights_per_device = split_weights_per_device,
                kv_layer_weights = kv_layer_weights,
            )
            if uneven is not None:
                return Plan(
                    n_ctx = n_ctx,
                    reason = (
                        "spilling every block and lm_head still does not fit device by "
                        f"device: {uneven}; leaving llama.cpp's own fitter to place it"
                    ),
                )
            return _finish(
                layout,
                opts,
                n_ctx,
                chosen,
                True,
                host_ram_bytes,
                quantised = quantised,
                kv_bytes_floor = kv_bytes_floor,
                reason = (
                    f"spilled every block's FFN ({freed / GIB:.2f} GiB) plus lm_head "
                    f"({layout.lm_head_bytes / GIB:.2f} GiB) to cover a "
                    f"{deficit / GIB:.2f} GiB deficit"
                ),
            )
    return None


def _finish(
    layout: ModelLayout,
    opts: PlanOptions,
    n_ctx: int,
    chosen: list[BlockLayout],
    spill_lm_head: bool,
    host_ram_bytes: Optional[int],
    *,
    quantised: bool = False,
    kv_bytes_floor: int = 0,
    reason: str = "",
) -> Plan:
    """Assemble patterns, decide the load mode, and account for both sides."""
    patterns: list[str] = []
    indices = sorted(b.index for b in chosen)
    if indices:
        # Global pattern only if no dropped blocks: \d+ would also match trailing nextn/MTP blocks.
        spillable = [b.index for b in layout.blocks if b.spillable_bytes > 0]
        all_of_them = set(indices) == set(spillable) and not layout.has_excluded_blocks
        patterns.append(spill_pattern_for(layout, None if all_of_them else indices))
    if spill_lm_head:
        patterns.append(LM_HEAD_PATTERN)

    spilled_bytes = sum(b.spillable_bytes for b in chosen) + (
        layout.lm_head_bytes if spill_lm_head else 0
    )
    lazy = min(max(0, int(opts.lazy_read_bytes or 0)), layout.per_layer_embd_bytes)
    host_bytes = layout.token_embd_bytes + layout.per_layer_embd_bytes - lazy + spilled_bytes
    if opts.kv_on_host:
        # -nkvo cache lives in host RAM; the mmap decision must count it.
        host_bytes += (
            cache_bytes(layout, n_ctx, kv_quantised = quantised, kv_bytes_floor = kv_bytes_floor)
            + layout.recurrent_bytes
        )
    vram_bytes = (
        all_resident_bytes(
            layout,
            n_ctx,
            kv_quantised = quantised,
            kv_bytes_floor = kv_bytes_floor,
            kv_on_host = opts.kv_on_host,
        )
        - spilled_bytes
    )

    # mmap costs 2-4.6x on host weight reads; disable only when host RAM fits the host side, else mmap keeps an over-commit pageable
    if host_ram_bytes is None:
        load_mode_none = False
    else:
        load_mode_none = host_bytes <= max(0, host_ram_bytes - opts.host_ram_headroom_bytes)

    cache_type = opts.kv_quant_type if quantised else None
    changed = bool(patterns) or load_mode_none or cache_type is not None
    return Plan(
        changed = changed,
        n_ctx = n_ctx,
        ot_patterns = tuple(patterns),
        load_mode_none = load_mode_none,
        # Matched K/V pairs only: unmatched ones are not compiled without GGML_CUDA_FA_ALL_QUANTS and fall back to CPU
        cache_type_k = cache_type,
        cache_type_v = cache_type,
        spilled_blocks = tuple(indices),
        spilled_lm_head = spill_lm_head,
        vram_bytes = vram_bytes,
        host_bytes = host_bytes,
        predicted_gen_penalty_ms = _spill_penalty_ms(layout, chosen, spill_lm_head, opts.host),
        reason = reason,
    )


def plan_to_args(plan: Plan) -> list[str]:
    """The launch flags for ``plan``. Empty when it changes nothing."""
    args: list[str] = []
    for pattern in plan.ot_patterns:
        args.extend(["-ot", f"{pattern}=CPU"])
    if plan.load_mode_none:
        args.extend(["--load-mode", "none"])
    if plan.cache_type_k and plan.cache_type_v:
        args.extend(["--cache-type-k", plan.cache_type_k])
        args.extend(["--cache-type-v", plan.cache_type_v])
    return args


_SMART_OFFLOAD_ON = ("1", "true", "yes", "on", "enabled")


def smart_offload_enabled(env: Optional[Mapping[str, str]] = None) -> bool:
    """Off unless enabled: it prices only generation, not prefill, and was slower on most planned cells."""
    raw = (os.environ if env is None else env).get("UNSLOTH_SMART_OFFLOAD")
    if raw is None:
        return False
    return str(raw).strip().lower() in _SMART_OFFLOAD_ON