Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
"""Opt-in REAP expert pruning before an export (Core's ``FastLanguageModel.reap_prune``)."""
from __future__ import annotations
import functools
import json
import os
import weakref
from pathlib import Path
from typing import Any, List, Optional, Tuple
from loggers import get_logger
logger = get_logger(__name__)
# Core's top-k keys plus the raw config.json aliases transformers maps through attribute_map.
_NUM_EXPERTS_KEYS = ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts")
_TOP_K_KEYS = ("num_experts_per_tok", "top_k", "moe_top_k", "top_k_experts", "moe_topk", "moe_k")
# Decoder layers that hand state to the next layer (prev_* arguments), which Core refuses since it
# calibrates one layer at a time: GLM-5 / HY-V4 sparse-attention top-k indices, ZAYA router states.
_STATEFUL_MODEL_TYPES = frozenset({"glm_moe_dsa", "glm5_next", "glm5_next_text", "hy_v4", "zaya"})
# Hash-routed MoE (tid2eid): Core refuses it since dropped experts cannot be remapped.
_HASH_ROUTED_MODEL_TYPES = frozenset({"deepseek_v4"})
_N_GROUP_KEYS = ("n_group", "num_expert_group")
_TOPK_GROUP_KEYS = ("topk_group", "top_k_group")
REPORT_NAME = "reap_report.json"
# The export worker's pid, so a parent only removes staging copies it can prove a dead worker made.
_OWNER_FILE = ".unsloth_reap_owner"
def _first_int(
config: dict,
keys,
per_layer = False,
) -> Optional[int]:
for key in keys:
value = config.get(key)
if isinstance(value, int) and not isinstance(value, bool):
return value
if (
per_layer
and isinstance(value, list)
and value
and all(isinstance(v, int) for v in value)
):
return max(value)
return None
def _model_parallel(distributed) -> bool:
"""A saved distributed_config Core refuses: expert / tensor / pipeline / FSDP / sequence parallel."""
if not isinstance(distributed, dict):
return False
if distributed.get("enable_expert_parallel") or distributed.get("enable_sequence_parallel"):
return True
return any(
isinstance(distributed.get(k), int) and distributed.get(k) > 1
for k in ("tp_size", "pp_size", "fsdp_size")
)
# Quantization a 16-bit load unpacks (gpt-oss mxfp4; bitsandbytes only for Hub ids Unsloth maps to
# 16-bit); anything else stays packed and Core refuses it only after a full load.
_UNPACKED_QUANT_METHODS = {"mxfp4"}
# Older ModelOpt exports declare quantization here (attach_hf_quant_config reads it too).
_HF_QUANT_CONFIG = "hf_quant_config.json"
@functools.lru_cache(maxsize = 1)
def _int_to_float_mapper() -> dict:
"""Unsloth's quantized-to-16-bit Hub id table, read from mapper.py (plain data) without importing
unsloth, which would initialize the GPU stack in the server process."""
import importlib.util
spec = importlib.util.find_spec("unsloth")
roots = list(spec.submodule_search_locations or []) if spec else []
for root in roots:
path = Path(root) / "models" / "mapper.py"
if path.is_file():
module_spec = importlib.util.spec_from_file_location("_unsloth_mapper_data", path)
module = importlib.util.module_from_spec(module_spec)
module_spec.loader.exec_module(module)
return dict(module.INT_TO_FLOAT_MAPPER)
return {}
def _maps_to_16_bit(model: str) -> bool:
"""A quantized Hub id Unsloth loads as its 16-bit original when asked for 16-bit."""
try:
return str(model).lower() in _int_to_float_mapper()
except Exception as exc:
logger.debug("Could not read Unsloth's model mapper: %s", exc)
return False
class MoeProbeError(RuntimeError):
"""The config could not be read (network, Hub, I/O): unknown, not a dense answer, so an export
with REAP on must wait or have REAP turned off rather than go out unpruned."""
def _cached_hub_file(repo: str, filename: str, offline: bool) -> Optional[bool]:
"""True if cached, False if recorded absent (offline: snapshot cached without it), else None."""
try:
from huggingface_hub import try_to_load_from_cache
from utils.hf_cache_settings import active_hf_hub_cache
cache_dir = active_hf_hub_cache()
found = try_to_load_from_cache(repo, filename, cache_dir = cache_dir)
if isinstance(found, str):
return True
if found is not None: # _CACHED_NO_EXIST
return False
if offline and any(
isinstance(try_to_load_from_cache(repo, name, cache_dir = cache_dir), str)
for name in ("config.json", "adapter_config.json")
):
return False
except Exception as exc:
logger.debug("Could not read the Hub cache for '%s': %s", repo, exc)
return None
def hub_file_exists(
repo: str,
filename: str,
hf_token = None,
) -> bool:
"""False only for a definite absence; anything unknown (timeout, auth, gated) raises MoeProbeError."""
error = None
try:
from huggingface_hub import HfApi, constants
offline = bool(constants.HF_HUB_OFFLINE)
if not offline:
return bool(HfApi(token = hf_token).file_exists(repo, filename))
except Exception as exc:
offline, error = False, exc
cached = _cached_hub_file(repo, filename, offline)
if cached is not None:
return cached
raise MoeProbeError(f"Could not check '{repo}' for {filename}: {error or 'offline'}") from error
def _export_runs_transformers_4(model: str, hf_token = None) -> bool:
"""Whether Studio loads it under transformers 4.x (4.57 tier); if unresolved the option stays,
and Core refuses clearly, never exporting unpruned."""
try:
from utils.transformers_version import (
TRANSFORMERS_DEFAULT_VERSION,
get_transformers_activation_tier,
)
tier = get_transformers_activation_tier(model, hf_token)
return tier == "457" or (
tier == "default" and str(TRANSFORMERS_DEFAULT_VERSION).startswith("4.")
)
except Exception as exc:
logger.debug("Could not resolve the transformers tier of '%s': %s", model, exc)
return False
def hub_adapter_has_mtp_head(repo: str, hf_token = None) -> bool:
"""Unsloth's loader re-attaches a Hub LoRA repo's trained MTP head, which REAP refuses."""
return hub_file_exists(repo, "mtp_head.safetensors", hf_token)
def _quantization_allows_reap(holders, bnb_maps_to_16_bit: bool) -> bool:
for holder in holders:
quant = holder.get("quantization_config")
if not quant:
continue
if not isinstance(quant, dict):
return False
method = str(quant.get("quant_method") or "").lower()
if not method and (quant.get("load_in_4bit") or quant.get("load_in_8bit")):
method = "bitsandbytes" # configs saved before quant_method existed
if method in _UNPACKED_QUANT_METHODS or (method == "bitsandbytes" and bnb_maps_to_16_bit):
continue
return False
return True
def moe_info_from_config(
config: Optional[dict], bnb_maps_to_16_bit: bool = False
) -> Optional[dict]:
"""``{num_experts, top_k}`` (plus ``n_group`` / ``topk_group`` for group-limited routers) for a
routed-MoE config dict, else None."""
if not isinstance(config, dict):
return None
holders = [h for h in (config.get("text_config"), config) if isinstance(h, dict)]
# Like Core's _first_attr: each field from the first level that has it, text_config first.
def first(keys, per_layer = False):
return next((v for h in holders if (v := _first_int(h, keys, per_layer)) is not None), None)
# Core refuses routers with identity (zero) experts and bidirectional / diffusion models.
if any(h.get("zero_expert_num") for h in holders) or any(
h.get("use_bidirectional_attention") == "all"
or str(h.get("model_type", "")).startswith("diffusion")
for h in holders
):
return None
if any(_model_parallel(h.get("distributed_config")) for h in holders):
return None
if not _quantization_allows_reap(holders, bnb_maps_to_16_bit):
return None
if any(
str(h.get("model_type", "")) in _STATEFUL_MODEL_TYPES | _HASH_ROUTED_MODEL_TYPES
for h in holders
):
return None
experts = first(_NUM_EXPERTS_KEYS)
top_k = first(_TOP_K_KEYS, per_layer = True)
if not (experts and top_k and experts > top_k):
return None
info = {"num_experts": experts, "top_k": top_k}
n_group, topk_group = first(_N_GROUP_KEYS), first(_TOPK_GROUP_KEYS)
if n_group and n_group > 1:
if not topk_group:
# Core refuses grouped routing it cannot prune per group.
return None
info.update(n_group = n_group, topk_group = topk_group)
return info
def moe_export_info(
model: str,
hf_token = None,
local_files_only = True,
_depth = 0,
adapter_base: Optional[str] = None,
) -> Optional[dict]:
"""MoE info for a local checkpoint / model folder or a Hub id (a LoRA folder resolves to its
base, or to ``adapter_base`` when the caller already authorized it). None for a dense model, a
repo without config.json or a malformed one; ``MoeProbeError`` when the config could not be read."""
try:
from utils.models.diffusion_lm import _local_adapter_base
from utils.models.model_config import _read_raw_config_json
from utils.paths import is_local_path, normalize_path
if is_local_path(model):
local = Path(normalize_path(model)).expanduser()
has_config = (local / "config.json").is_file()
# The loader re-attaches a LoRA folder's trained MTP head, which REAP refuses.
if (local / "adapter_config.json").is_file() and (
local / "mtp_head.safetensors"
).is_file():
return None
# The export loader follows adapter_config.json even beside a config.json.
if not has_config or (local / "adapter_config.json").is_file():
base = adapter_base or _local_adapter_base(str(local))
if base and _depth < 2 and base != model:
# A Hub base of a local adapter may not be cached yet: read it online.
return moe_export_info(base, hf_token, is_local_path(base), _depth + 1)
if not has_config:
return None
if not (local / "adapter_config.json").is_file():
from core.export.drafter_export import training_manifest
# The export attaches a full checkpoint's trained MTP head, which REAP refuses.
manifest = training_manifest(str(local))
if manifest and manifest.get("strategy") == "mtp":
return None
local = is_local_path(model)
if local and (Path(normalize_path(model)).expanduser() / _HF_QUANT_CONFIG).is_file():
return None
config = _read_raw_config_json(model, hf_token, local_files_only, None)
info = moe_info_from_config(config, bnb_maps_to_16_bit = not local and _maps_to_16_bit(model))
if info is not None and _export_runs_transformers_4(model, hf_token):
return None
if (
info is not None
and not local
and not (isinstance(config, dict) and config.get("quantization_config"))
and hub_file_exists(model, _HF_QUANT_CONFIG, hf_token)
):
return None
return info
except MoeProbeError:
raise
except ValueError as exc:
logger.debug("MoE detection skipped for '%s': %s", model, exc)
return None
except Exception as exc:
raise MoeProbeError(f"Could not read the config of '{model}': {exc}") from exc
def _options_key(options: dict) -> Tuple[float, int, Optional[str]]:
return (
float(options.get("sparsity", 0.25)),
int(options.get("num_calibration_samples", 512)),
(options.get("dataset") or "").strip() or None,
)
def _loaded_in_4bit(model: Any) -> bool:
if getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False):
return True
config = getattr(model, "config", None)
quant = getattr(config, "quantization_config", None)
if quant is not None:
get = quant.get if isinstance(quant, dict) else lambda k, d = None: getattr(quant, k, d)
if get("load_in_4bit", False) or get("load_in_8bit", False):
return True
return False
def _calibration_dataset(dataset: Optional[str], hf_token):
"""A custom dataset opened with the request's token (export workers keep tokens out of their
environment, so Core's own load would be anonymous); the default stays Core's."""
if not dataset:
return None
from datasets import load_dataset
kwargs = {} if hf_token is None else {"token": hf_token}
# Like Core: no split forced, take `train` else the only split.
splits = load_dataset(dataset, streaming = True, **kwargs)
if not isinstance(splits, dict):
return splits
if "train" in splits:
return splits["train"]
if len(splits) == 1:
return next(iter(splits.values()))
raise ValueError(
f"the calibration dataset '{dataset}' has no train split but several others "
f"({', '.join(sorted(splits))}); use a dataset with a train split or a single split"
)
def current_reap_key(backend: Any):
"""The REAP settings the loaded model was pruned with, else None. A key whose model is no
longer the loaded one (an unload that failed part way before a new load) is not this model's."""
key = getattr(backend, "_reap_key", None)
ref = getattr(backend, "_reap_model", None)
model = getattr(backend, "current_model", None)
if key is None or ref is None or model is None or ref() is not model:
return None
return key
def apply_reap(
backend: Any,
options: Optional[dict],
staging_parent,
hf_token = None,
) -> Tuple[bool, str]:
"""Prune once per load (multi-format reuses it); hf_token opens gated calibration data."""
if not options:
return True, ""
key = _options_key(options)
applied = current_reap_key(backend)
if applied is None and getattr(backend, "_reap_key", None) is not None:
# Left by an unload that failed part way: it describes an older model, not this load.
discard_reap_staging(backend)
if applied == key:
return True, ""
if applied is not None:
return False, (
"This model was already pruned with different REAP settings. Start the export again "
"to load a fresh copy."
)
model, tokenizer = backend.current_model, backend.current_tokenizer
if model is None:
return False, "No model loaded. Please select a checkpoint first."
if _loaded_in_4bit(model):
return False, (
"REAP expert pruning needs the model loaded in 16-bit, but it was loaded in 4-bit. "
"Turn off 4-bit loading and export again."
)
if getattr(model, "__dict__", {}).get("_unsloth_mtp_head") is not None:
# Core refuses this: the attached MTP head would keep the old expert count.
return False, (
"REAP cannot prune a checkpoint with an attached MTP head. Export it without expert "
"pruning."
)
from core.export.export import _staging_dir
from unsloth import FastLanguageModel
try:
if getattr(backend, "is_peft", False):
# REAP scores the fine-tuned experts, so the adapters are merged first (16-bit, in memory).
logger.info("REAP: merging the LoRA adapters into the 16-bit base before pruning...")
model = model.merge_and_unload()
backend.current_model = model
backend.is_peft = False
# Kept until the next load: the base export must still refuse this LoRA checkpoint.
backend._reap_merged_peft = True
sweep_reap_staging()
staging = _staging_dir(staging_parent)
_register_staging(staging.name)
backend._reap_staging = staging
sparsity, samples, dataset = key
# save_directory repoints every later save / merge / GGUF at the pruned copy, named after the model.
source = str(getattr(model.config, "_name_or_path", "") or "")
name = Path(source.rstrip("/\\") or "model").name
FastLanguageModel.reap_prune(
model,
tokenizer,
sparsity = sparsity,
dataset = _calibration_dataset(dataset, hf_token),
num_calibration_samples = samples,
save_directory = str(Path(staging.name) / f"{name or 'model'}-REAP"),
)
except Exception as exc:
# A failure part way leaves the loaded model merged and partly pruned; never export it.
logger.warning("REAP failed, unloading the partly pruned model: %s", exc)
unloaded = False
try:
unloaded = backend.cleanup_memory() is not False
except Exception as cleanup_exc:
logger.warning("Could not unload after a failed REAP run: %s", cleanup_exc)
if not unloaded:
# cleanup_memory may report failure; this worker must never export the model again.
backend.current_model = None
backend.current_tokenizer = None
backend.current_checkpoint = None
discard_reap_staging(backend)
raise RuntimeError(
f"REAP expert pruning failed: {exc}. The model was unloaded since it may be partly "
"pruned; load it again to export."
) from exc
model._unsloth_reap_source = source or None
backend._reap_key = key
# The settings belong to this model object only (see current_reap_key).
backend._reap_model = weakref.ref(model)
report = getattr(model, "_unsloth_reap_report", None) or {}
kept = len(next(iter(report.values()), []))
return True, f"REAP kept {kept} experts per MoE layer in {len(report)} layers."
def gguf_report_name(gguf: str) -> str:
"""The report beside one GGUF file (``model.Q4_K_M.gguf`` -> ``model.Q4_K_M.reap_report.json``):
a reused GGUF folder keeps files from other loads, so each file carries its own."""
name = os.path.basename(str(gguf))
stem = name[: -len(".gguf")] if name.lower().endswith(".gguf") else name
return f"{stem}.reap_report.json"
def sync_reap_report(
backend: Any,
output_path: Optional[str],
name: str = REPORT_NAME,
files: Optional[List[str]] = None,
) -> None:
"""After an export: the report when the exported model is pruned, else remove one a previous
REAP export left in a reused folder, so the folder never describes other weights."""
if current_reap_key(backend) is not None:
write_reap_report(backend, output_path, name, files)
return
if output_path and Path(output_path).is_dir():
stale = Path(output_path) / name
try:
stale.unlink(missing_ok = True)
except OSError as exc:
logger.warning(f"Could not remove the stale {name}: {exc}")
def write_reap_report(
backend: Any,
output_path: Optional[str],
name: str = REPORT_NAME,
files: Optional[List[str]] = None,
) -> None:
"""Kept expert ids per layer, beside the export; files names the artifacts it describes."""
model = getattr(backend, "current_model", None)
report = getattr(model, "_unsloth_reap_report", None)
key = current_reap_key(backend)
if not report or key is None or not output_path or not Path(output_path).is_dir():
return
try:
payload = {
"method": "REAP (arXiv:2510.13999)",
"sparsity": key[0],
"num_calibration_samples": key[1],
"dataset": key[2] or "mlabonne/open-perfectblend",
"kept_experts": report,
}
if files:
payload["files"] = files
(Path(output_path) / name).write_text(json.dumps(payload, indent = 1), encoding = "utf-8")
except Exception as exc:
logger.warning(f"Could not write {name}: {exc}")
def discard_reap_staging(backend: Any) -> None:
staging = getattr(backend, "_reap_staging", None)
backend._reap_staging = None
backend._reap_key = None
backend._reap_model = None
backend._reap_merged_peft = False
if staging is not None:
try:
staging.cleanup()
except Exception as exc:
logger.warning(f"Could not remove the REAP staging copy: {exc}")
sweep_reap_staging()
def reap_staging_parent(save_directory: Optional[str]) -> Path:
"""Stages on the save folder's filesystem, else the exports root; _staging_dir picks the roomier one."""
from utils.paths.storage_roots import resolve_export_dir, resolve_export_write_dir
try:
target = resolve_export_write_dir(save_directory)
except Exception:
return resolve_export_dir()
# The save folder itself may not exist yet; its nearest existing ancestor is on the same drive.
for candidate in (target, *target.parents):
if candidate.is_dir():
return candidate
return resolve_export_dir()
def _registry() -> Path:
from utils.paths.storage_roots import cache_root
return cache_root() / "reap_staging.json"
def _read_registry() -> list:
try:
entries = json.loads(_registry().read_text(encoding = "utf-8"))
return [e for e in entries if isinstance(e, dict) and "path" in e and "pid" in e]
except Exception:
return []
def _write_registry(entries: list) -> None:
try:
path = _registry()
path.parent.mkdir(parents = True, exist_ok = True)
tmp = path.with_suffix(".tmp")
tmp.write_text(json.dumps(entries), encoding = "utf-8")
tmp.replace(path)
except Exception as exc:
logger.debug("Could not update the REAP staging registry: %s", exc)
def _register_staging(path: str) -> None:
import os
pid = os.getpid()
try:
(Path(path) / _OWNER_FILE).write_text(str(pid), encoding = "utf-8")
except OSError as exc:
logger.debug("Could not mark the REAP staging copy: %s", exc)
_write_registry(_read_registry() + [{"path": str(path), "pid": pid}])
def sweep_reap_staging(dead_pid: Optional[int] = None) -> None:
"""Remove staging copies whose worker is gone: a cancel kills the worker, so neither
``cleanup_memory`` nor the TemporaryDirectory finalizer runs. ``dead_pid`` is a worker the
caller just killed; otherwise an owner counts as gone when its process no longer exists.
Only folders carrying their owner marker are touched. Never raises."""
import shutil
try:
import psutil
alive = psutil.pid_exists
except Exception:
alive = None
entries = _read_registry()
if not entries:
return
keep = []
for entry in entries:
path, pid = Path(entry["path"]), entry["pid"]
if not path.exists():
continue
gone = pid == dead_pid or (dead_pid is None and alive is not None and not alive(pid))
try:
owned = (path / _OWNER_FILE).read_text(encoding = "utf-8").strip() == str(pid)
except OSError:
owned = False
if gone and owned:
shutil.rmtree(path, ignore_errors = True)
if not path.exists():
logger.info("Removed a REAP staging copy left by a stopped export: %s", path)
continue
keep.append(entry)
if keep != entries:
_write_registry(keep)