Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""GGUF tensor layout by placement bucket; file IO and GGUF keys live here, not in offload_planner."""
from __future__ import annotations
import logging
import re
from dataclasses import dataclass, field
from typing import Optional
import os
from core.inference.lookup_tables import is_per_layer_embedding
logger = logging.getLogger(__name__)
# Hybrid Mamba-2 archs storing head_count_kv (and on Bolt expert_used_count) per layer.
_NEMOTRON_H_ARCHS = frozenset({"nemotron_h", "nemotron_h_moe"})
_BLOCK_RE = re.compile(r"^blk\.(\d+)\.(.+)$")
# Sparse experts incl. fused/chunked forms; NOT ffn_routed_*: kimi-k3 reads it every token
_MOE_EXPERT_RE = re.compile(r"^ffn_(up|gate|down|gate_up)_(exps|chexps)\.weight$")
# Dense FFN. Fully activated: every byte crosses the bus every token.
_DENSE_FFN_RE = re.compile(r"^ffn_(up|gate|down)\.weight$")
@dataclass(frozen = True)
class BlockLayout:
"""One transformer block, split into what may and may not be spilled."""
index: int
# ffn_*_exps (MoE) or plain ffn_* (dense). Safe to push to host RAM.
spillable_bytes: int
# attention, norms, routers, shared experts, ssm: on the critical path every token, or the KV cache hangs off them.
resident_bytes: int
@dataclass(frozen = True)
class ModelLayout:
"""Everything the planner needs, and nothing about files."""
arch: str = ""
n_layers: int = 0
n_attention_layers: int = 0
blocks: tuple[BlockLayout, ...] = field(default_factory = tuple)
# Rides the layer list at index n_layer_all, so it is GPU-resident for any -ngl >= 1 and can only be moved with an
# explicit override.
lm_head_bytes: int = 0
# llama.cpp pins dev_input to CPU: never VRAM, but charged to host RAM
token_embd_bytes: int = 0
per_layer_embd_bytes: int = 0
# Every tensor's bytes, excluded blocks included; the file size less its metadata.
tensor_bytes: int = 0
# output_norm and friends: GPU-resident, too small to be worth spilling.
other_resident_bytes: int = 0
# Attention cache for ONE token at f16, across the attention layers only.
kv_bytes_per_token_f16: int = 0
# Mamba conv/SSM state; context independent, and follows the layer, which -ot never moves
recurrent_bytes: int = 0
n_ctx_train: int = 0
is_moe: bool = False
# Offloaded MoE experts move only expert_used/expert_count per token; a dense FFN moves all of it.
n_expert: int = 0
n_expert_used: int = 0
# Trailing nextn/MTP blocks dropped; an unbounded ^blk\.\d+\. pattern would match them
has_excluded_blocks: bool = False
# Dropped block bytes: --spec-type draft-mtp loads them on the target, GPU first
excluded_block_bytes: int = 0
# SWA: per-layer cache sizes differ, so a multi-device split abstains
has_swa: bool = False
# False when a needed quantity could not be read. The planner abstains.
complete: bool = False
@property
def spillable_bytes(self) -> int:
return sum(b.spillable_bytes for b in self.blocks)
@property
def block_resident_bytes(self) -> int:
return sum(b.resident_bytes for b in self.blocks)
def kv_bytes(
self,
n_ctx: int,
bytes_per_elem: int = 2,
) -> int:
"""Attention cache at ``n_ctx``. bytes_per_elem 2 = f16, 1 = q8_0-ish."""
if self.kv_bytes_per_token_f16 <= 0 or n_ctx <= 0:
return 0
return self.kv_bytes_per_token_f16 * n_ctx * bytes_per_elem // 2
def _field(
reader,
key: str,
default = None,
):
f = reader.fields.get(key)
if f is None:
return default
try:
return f.contents()
except Exception: # a malformed field must not take the whole load down
return default
_SPLIT_SHARD_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})\.gguf$", re.IGNORECASE)
def split_shard_paths(path: str) -> Optional[list[str]]:
"""Every shard of the split, in order; None unless the name is llama.cpp's split pattern."""
directory, name = os.path.split(path)
match = _SPLIT_SHARD_RE.match(name)
if not match:
return None
prefix, _index, total = match.groups()
return [
os.path.join(directory, f"{prefix}-{i:05d}-of-{int(total):05d}.gguf")
for i in range(1, int(total) + 1)
]
def layout_from_gguf(path: str, *, all_shards: bool = False) -> ModelLayout:
"""Returns an incomplete layout rather than raising, so the planner abstains; all_shards reads every sibling shard, and all must be present"""
try:
import struct
import numpy as np
from gguf import GGUFEndian, GGUFReader, GGUFValueType
class _LayoutReader(GGUFReader):
# GGUFReader makes a numpy view per array element, seconds for a vocabulary; no string array is read here.
def _get_field_parts(self, orig_offs, raw_type):
if raw_type != GGUFValueType.ARRAY:
return super()._get_field_parts(orig_offs, raw_type)
raw_itype = self._get(orig_offs, np.uint32)
alen = self._get(orig_offs + 4, np.uint64)
itype = GGUFValueType(raw_itype[0])
types = [GGUFValueType.ARRAY, itype]
nptype = self.gguf_scalar_to_np.get(itype)
if nptype is not None:
items = self._get(orig_offs + 12, nptype, alen[0])
return 12 + int(items.nbytes), [raw_itype, alen, items], [2], types
if itype != GGUFValueType.STRING:
return super()._get_field_parts(orig_offs, raw_type)
length = struct.Struct("<Q" if self.endianess == GGUFEndian.LITTLE else ">Q")
offs = orig_offs + 12
for _ in range(int(alen[0])):
offs += 8 + length.unpack_from(self.data, offs)[0]
return offs - orig_offs, [raw_itype, alen], [], types
readers = [_LayoutReader(path)]
if all_shards and int(_field(readers[0], "split.count") or 0) > 1:
shards = split_shard_paths(path)
if not shards or not all(os.path.isfile(p) for p in shards):
logger.debug("offload layout: split %s is missing a shard", path)
return ModelLayout()
readers = [_LayoutReader(p) for p in shards]
except Exception as exc:
logger.debug("offload layout: cannot read %s (%s)", path, exc)
return ModelLayout()
try:
return _layout_from_readers(readers)
except Exception as exc:
logger.debug("offload layout: cannot interpret %s (%s)", path, exc)
return ModelLayout()
def _layout_from_reader(reader) -> ModelLayout:
return _layout_from_readers([reader])
def _layout_from_readers(readers) -> ModelLayout:
"""One reader per shard, the first carrying the metadata."""
reader = readers[0]
# Split GGUF: GGUFReader maps one shard, so a partial read undercounts and can OOM under --fit off; abstain unless every shard is given
if (int(_field(reader, "split.count") or 0) or 1) != len(readers):
return ModelLayout()
arch = str(_field(reader, "general.architecture") or "")
if not arch:
return ModelLayout()
blocks_total = _field(reader, f"{arch}.block_count")
if not blocks_total:
return ModelLayout()
blocks_total = int(blocks_total)
# MTP blocks sit outside llama.cpp's target context, so the attention count excludes them.
nextn = int(_field(reader, f"{arch}.nextn_predict_layers") or 0)
n_layers = max(0, blocks_total - nextn)
# Hybrid: only 1 in full_attention_interval layers carries a KV cache, the rest are recurrent. Absent (or 0) means
# every layer is attention.
fai = int(_field(reader, f"{arch}.full_attention_interval") or 0)
n_attention = -(-n_layers // fai) if fai > 0 else n_layers
n_recurrent = max(0, n_layers - n_attention)
n_kv_head = _field(reader, f"{arch}.attention.head_count_kv")
n_head = _field(reader, f"{arch}.attention.head_count")
n_embd = _field(reader, f"{arch}.embedding_length")
key_len = _field(reader, f"{arch}.attention.key_length")
val_len = _field(reader, f"{arch}.attention.value_length")
if key_len is None and n_embd and n_head:
key_len = int(n_embd) // int(n_head)
if val_len is None:
val_len = key_len
if not n_kv_head or not key_len or not val_len:
return ModelLayout()
nemotron_h = arch in _NEMOTRON_H_ARCHS
if nemotron_h and isinstance(n_kv_head, (list, tuple)):
# Per-layer head_count_kv: 0 marks a layer without an attention cache. The recurrent (Mamba) layers are the ones
# with neither attention nor an FFN; the rest are dense or MoE FFN blocks.
heads = [int(h) for h in list(n_kv_head)[:n_layers]]
ffn = _field(reader, f"{arch}.feed_forward_length")
ffn = (
[int(f) for f in list(ffn)[:n_layers]]
if isinstance(ffn, (list, tuple))
else [0] * len(heads)
)
n_attention = sum(1 for h in heads if h > 0)
n_recurrent = sum(1 for h, f in zip(heads, ffn) if h == 0 and f == 0)
kv_per_token = sum(heads) * (int(key_len) + int(val_len)) * 2
else:
kv_per_token = int(n_attention) * int(n_kv_head) * (int(key_len) + int(val_len)) * 2
# Full context on every layer is safe for TOTAL but cannot show which layers hold the big caches.
has_swa = bool(_field(reader, f"{arch}.attention.sliding_window") or 0)
# Mamba conv and SSM state, one f32 copy per sequence; zero without recurrent layers.
d_inner = int(_field(reader, f"{arch}.ssm.inner_size") or 0)
d_state = int(_field(reader, f"{arch}.ssm.state_size") or 0)
n_group = int(_field(reader, f"{arch}.ssm.group_count") or 0)
d_conv = int(_field(reader, f"{arch}.ssm.conv_kernel") or 0)
recurrent = 0
if n_recurrent and d_inner and d_state and d_conv:
n_embd_r = max(0, d_conv - 1) * (d_inner + 2 * n_group * d_state)
n_embd_s = d_state * d_inner
recurrent = n_recurrent * (n_embd_r + n_embd_s) * 4
n_expert = int(_field(reader, f"{arch}.expert_count") or 0)
n_expert_used = _field(reader, f"{arch}.expert_used_count") or 0
if nemotron_h and isinstance(n_expert_used, (list, tuple)):
n_expert_used = max(
(int(n) for n in n_expert_used), default = 0
) # per layer on Nemotron 3.5 Bolt
n_expert_used = int(n_expert_used)
is_moe = bool(n_expert)
spill: dict[int, int] = {}
resident: dict[int, int] = {}
lm_head = 0
token_embd = 0
per_layer_embd = 0
other_resident = 0
for tensor in (t for r in readers for t in r.tensors):
name = str(tensor.name)
nbytes = int(tensor.n_bytes)
match = _BLOCK_RE.match(name)
if match:
index = int(match.group(1))
tail = match.group(2)
# Shared experts and routers run on every token, so they are not spillable.
spillable = _MOE_EXPERT_RE.match(tail) or (not is_moe and _DENSE_FFN_RE.match(tail))
if spillable:
spill[index] = spill.get(index, 0) + nbytes
else:
resident[index] = resident.get(index, 0) + nbytes
continue
# Before the token_embd substring below, which would otherwise swallow the per-layer table.
if is_per_layer_embedding(name):
per_layer_embd += nbytes
# token_embd_norm is a repeating-layer tensor, not a host-pinned input embedding.
elif "token_embd" in name and not name.startswith("token_embd_norm"):
token_embd += nbytes
elif name == "output.weight":
lm_head += nbytes
else:
other_resident += nbytes
if not spill and not resident:
return ModelLayout()
# Tied embeddings: llama.cpp allocates a second full matrix on the output device; count it resident, else the lm_head rung credits a spill that moves nothing
if not lm_head and token_embd:
other_resident += token_embd
# Trailing nextn/MTP blocks are not loaded without a draft: spilling them frees nothing
all_block_indices = set(spill) | set(resident)
block_indices = sorted(i for i in all_block_indices if i < n_layers)
has_excluded = any(i >= n_layers for i in all_block_indices)
excluded_bytes = sum(
spill.get(i, 0) + resident.get(i, 0) for i in all_block_indices if i >= n_layers
)
blocks = tuple(
BlockLayout(
index = i,
spillable_bytes = spill.get(i, 0),
resident_bytes = resident.get(i, 0),
)
for i in block_indices
)
return ModelLayout(
arch = arch,
n_layers = n_layers,
n_attention_layers = int(n_attention),
has_swa = has_swa,
blocks = blocks,
lm_head_bytes = lm_head,
token_embd_bytes = token_embd,
per_layer_embd_bytes = per_layer_embd,
tensor_bytes = sum(int(t.n_bytes) for r in readers for t in r.tensors),
other_resident_bytes = other_resident,
kv_bytes_per_token_f16 = kv_per_token,
recurrent_bytes = recurrent,
n_ctx_train = int(_field(reader, f"{arch}.context_length") or 0),
is_moe = is_moe,
n_expert = n_expert,
n_expert_used = n_expert_used,
has_excluded_blocks = has_excluded,
excluded_block_bytes = excluded_bytes,
complete = True,
)
def spill_pattern_for(layout: ModelLayout, indices: Optional[list[int]] = None) -> str:
"""Anchored: llama.cpp uses regex_search, so an unanchored pattern also moves attention tensors."""
# Must match _MOE_EXPERT_RE, or the plan credits bytes the pattern never moves
body = "ffn_(up|gate|down|gate_up)_(exps|chexps)" if layout.is_moe else "ffn_(up|gate|down)"
if indices is None:
block = r"\d+"
else:
block = "|".join(str(i) for i in sorted(indices))
block = f"({block})"
return rf"^blk\.{block}\.{body}\.weight$"
LM_HEAD_PATTERN = r"^output\.weight$"