Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add REAP expert pruning for MoE models (#13834)

Daniel Han093390e4997f
diff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.py
index c61157b567f..9081ab2db53 100644
--- a/tests/test_merged_hub_destination.py
+++ b/tests/test_merged_hub_destination.py
@@ -346,6 +346,19 @@ def test_unforced_4bit_rejection_does_not_create_hub_resources(saving, options):
assert records["repos"] == records["branches"] == records["merges"] == records["uploads"] == []
[email protected]("options", [{"revision": "candidate"}, {"create_pr": True}, {}])
+def test_in_memory_reap_rejection_does_not_create_hub_resources(saving, options):
+ # A raw-PEFT wrap of a model REAP pruned without save_directory: the base on disk is unpruned.
+ env, records, _ = saving
+ model = PeftModel()
+ model._unsloth_reap_report = {"model.layers.0.mlp": [0, 2]}
+ with pytest.raises(RuntimeError, match = "pruned in memory only"):
+ env["unsloth_generic_push_to_hub_merged"](
+ model, "owner/new-model", save_method = "merged_16bit", **options
+ )
+ assert records["repos"] == records["branches"] == records["merges"] == records["uploads"] == []
+
+
def test_forced_4bit_mode_still_reaches_staged_save(saving):
env, records, _ = saving
env["unsloth_generic_push_to_hub_merged"](
@@ -716,3 +729,69 @@ def test_merged_dtype_refuses_an_older_zoo(saving):
)
# Refused before the base model is downloaded into the Hub cache.
assert records["merges"] == [] and prewarmed == []
+
+
+def test_adapter_push_against_a_local_reap_base_uploads_nothing(saving, tmp_path):
+ # reap_prune(..., save_directory) then LoRA: the adapter's base is a folder only this machine has.
+ env, records, _ = saving
+ model = PeftModel()
+ model._unsloth_reap_report = {"model.layers.0.mlp": [0, 2]}
+ model._unsloth_reap_saved = str(tmp_path)
+ with pytest.raises(RuntimeError, match = "local folder"):
+ env["unsloth_generic_push_to_hub_merged"](model, "owner/adapter", save_method = "lora")
+ assert records["repos"] == records["branches"] == records["merges"] == records["uploads"] == []
+
+
+def test_adapter_push_against_a_moved_local_reap_base_uploads_nothing(saving, tmp_path):
+ # The pruned base folder was moved or deleted after LoRA setup: the adapter still names it.
+ env, records, _ = saving
+ model = PeftModel()
+ model._unsloth_reap_report = {"model.layers.0.mlp": [0, 2]}
+ model._unsloth_reap_saved = str(tmp_path / "moved_away")
+ with pytest.raises(RuntimeError, match = "local folder"):
+ env["unsloth_generic_push_to_hub_merged"](model, "owner/adapter", save_method = "lora")
+ assert records["repos"] == records["branches"] == records["merges"] == records["uploads"] == []
+
+
+def test_adapter_push_after_reloading_a_local_reap_checkpoint_uploads_nothing(saving, tmp_path):
+ # reap_prune(..., save_directory) then a fresh process loads that folder: no in-process marker,
+ # but the provenance file reap_prune wrote there marks the adapter's base as a local REAP model.
+ env, records, _ = saving
+ (tmp_path / "unsloth_reap.json").write_text('{"kept_experts": {}}')
+ model = PeftModel()
+ model.peft_config = {"default": SimpleNamespace(base_model_name_or_path = str(tmp_path))}
+ with pytest.raises(RuntimeError, match = "REAP pruned model saved to a local folder"):
+ env["unsloth_generic_push_to_hub_merged"](model, "owner/adapter", save_method = "lora")
+ assert (
+ records["repos"]
+ == records["branches"]
+ == records["uploads"]
+ == records["adapter_saves"]
+ == []
+ )
+
+
+def test_adapter_push_on_an_ordinary_local_base_is_not_refused(saving, tmp_path):
+ env, records, _ = saving
+ model = PeftModel()
+ model.peft_config = {"default": SimpleNamespace(base_model_name_or_path = str(tmp_path))}
+ env["unsloth_generic_push_to_hub_merged"](model, "owner/adapter", save_method = "lora")
+ assert len(records["adapter_saves"]) == 1
+
+
[email protected]("kind", ["compressed", "torchao"])
+def test_in_memory_reap_is_refused_before_quantized_export_dispatch(saving, kind):
+ # Those export helpers install their runtime and create the Hub repo before any nested save runs.
+ env, records, _ = saving
+ called = []
+ if kind == "compressed":
+ env["_normalize_compressed_method"] = lambda method: ("FP8_DYNAMIC", False, "FP8")
+ env["_unsloth_save_compressed_tensors"] = lambda **kwargs: called.append(kwargs)
+ else:
+ env["_normalize_torchao_method"] = lambda method: ("int8", "INT8")
+ env["_unsloth_save_torchao"] = lambda **kwargs: called.append(kwargs)
+ model = PeftModel()
+ model._unsloth_reap_report = {"model.layers.0.mlp": [0, 2]}
+ with pytest.raises(RuntimeError, match = "pruned in memory only"):
+ env["unsloth_generic_push_to_hub_merged"](model, "owner/new-model", save_method = "fp8")
+ assert called == [] and records["repos"] == []
diff --git a/tests/test_reap_prune.py b/tests/test_reap_prune.py
new file mode 100644
index 00000000000..b994dc56dda
--- /dev/null
+++ b/tests/test_reap_prune.py
@@ -0,0 +1,1053 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""REAP expert pruning (unsloth/models/reap.py) against a brute-force oracle, on CPU tiny MoEs."""
+
+import copy
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+transformers = pytest.importorskip("transformers")
+if int(transformers.__version__.split(".")[0]) < 5:
+ pytest.skip(
+ reason = "REAP prunes transformers 5 fused MoE experts (3D expert weights); 4.x keeps a ModuleList",
+ allow_module_level = True,
+ )
+
+_MODULE = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "reap.py"
+_spec = importlib.util.spec_from_file_location("_unsloth_reap", _MODULE)
+reap = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(reap)
+
+
[email protected](autouse = True)
+def _one_thread():
+ # Tiny CPU ops thrash on a loaded many-core host; one thread is faster and deterministic.
+ threads = torch.get_num_threads()
+ torch.set_num_threads(1)
+ yield
+ torch.set_num_threads(threads)
+
+
[email protected](autouse = True)
+def _float64_moe_backend(monkeypatch):
+ # These models are float64; Unsloth's grouped_mm MoE backend (picked when a GPU is visible) has no
+ # float64 kernel, so pin the looped backend. The backend choice is lru_cached per module copy.
+ monkeypatch.setenv("UNSLOTH_MOE_BACKEND", "native_torch")
+ caches = [
+ getattr(vars(module).get("select_moe_backend"), "cache_clear", None)
+ for module in list(sys.modules.values())
+ if module is not None and hasattr(module, "__dict__")
+ ]
+ caches = [clear for clear in caches if callable(clear)]
+ for clear in caches:
+ clear()
+ yield
+ for clear in caches:
+ clear()
+
+
+def _qwen3_moe(
+ num_experts = 16,
+ top_k = 4,
+ seed = 0,
+):
+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM
+
+ config = Qwen3MoeConfig(
+ vocab_size = 96,
+ hidden_size = 32,
+ intermediate_size = 48,
+ moe_intermediate_size = 16,
+ num_hidden_layers = 3,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ num_experts = num_experts,
+ num_experts_per_tok = top_k,
+ norm_topk_prob = True,
+ mlp_only_layers = [0], # a dense layer in front, like GLM / DeepSeek
+ max_position_embeddings = 64,
+ )
+ torch.manual_seed(seed)
+ model = Qwen3MoeForCausalLM._from_config(
+ config, dtype = torch.float64, experts_implementation = "eager"
+ )
+ return model.eval()
+
+
+def _deepseek_v3(
+ n_group = 4,
+ topk_group = 2,
+ seed = 0,
+):
+ from transformers import DeepseekV3Config, DeepseekV3ForCausalLM
+
+ config = DeepseekV3Config(
+ vocab_size = 96,
+ hidden_size = 32,
+ intermediate_size = 48,
+ moe_intermediate_size = 16,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 4,
+ n_routed_experts = 16,
+ n_shared_experts = 1,
+ num_experts_per_tok = 4,
+ n_group = n_group,
+ topk_group = topk_group,
+ first_k_dense_replace = 0,
+ q_lora_rank = 16,
+ kv_lora_rank = 16,
+ qk_rope_head_dim = 8,
+ qk_nope_head_dim = 8,
+ v_head_dim = 8,
+ max_position_embeddings = 64,
+ )
+ torch.manual_seed(seed)
+ model = DeepseekV3ForCausalLM._from_config(
+ config, dtype = torch.float64, experts_implementation = "eager"
+ )
+ with torch.no_grad():
+ for module in model.modules():
+ bias = getattr(module, "e_score_correction_bias", None)
+ if bias is not None:
+ bias.copy_(torch.randn_like(bias) * 0.1)
+ return model.eval()
+
+
+def _samples(n = 7, seed = 1):
+ g = torch.Generator().manual_seed(seed)
+ return [
+ torch.randint(1, 96, (int(torch.randint(3, 19, (1,), generator = g)),), generator = g).tolist()
+ for _ in range(n)
+ ]
+
+
+def _oracle(model, samples, n_drop):
+ """Sequential REAP written from the paper: per sample (no padding), per (token, slot) eager
+ expert matmuls in float64, each layer pruned before the next one is scored."""
+ model = copy.deepcopy(model)
+ kept, scores = {}, {}
+ for index, layer in enumerate(model.model.layers):
+ block = layer.mlp
+ if not hasattr(block, "experts"):
+ continue
+ experts = block.experts
+ num_experts = experts.gate_up_proj.shape[0]
+ total = torch.zeros(num_experts, dtype = torch.float64)
+ count = torch.zeros(num_experts, dtype = torch.float64)
+ seen = {}
+ hook = experts.register_forward_hook(
+ lambda m, args, out: seen.update(x = args[0], idx = args[1], w = args[2])
+ )
+ with torch.no_grad():
+ for sample in samples:
+ model.model(input_ids = torch.tensor([sample]), use_cache = False)
+ for t in range(seen["x"].shape[0]):
+ for s in range(seen["idx"].shape[1]):
+ e = int(seen["idx"][t, s])
+ gate, up = torch.nn.functional.linear(
+ seen["x"][t], experts.gate_up_proj[e]
+ ).chunk(2)
+ f = torch.nn.functional.linear(
+ experts.act_fn(gate) * up, experts.down_proj[e]
+ )
+ total[e] += seen["w"][t, s].double() * f.double().norm()
+ count[e] += 1
+ hook.remove()
+ saliency = total / count.clamp(min = 1)
+ drop = set(torch.topk(saliency, n_drop, largest = False).indices.tolist())
+ keep = [i for i in range(num_experts) if i not in drop]
+ name = f"model.layers.{index}.mlp"
+ kept[name], scores[name] = keep, saliency
+ idx = torch.tensor(keep)
+ experts.gate_up_proj = torch.nn.Parameter(experts.gate_up_proj.data[idx])
+ experts.down_proj = torch.nn.Parameter(experts.down_proj.data[idx])
+ experts.num_experts = len(keep)
+ block.gate.weight = torch.nn.Parameter(block.gate.weight.data[idx])
+ block.gate.num_experts = len(keep)
+ return kept, scores
+
+
+def test_matches_sequential_oracle_with_padded_batches():
+ model = _qwen3_moe()
+ samples = _samples()
+ kept, scores = _oracle(model, samples, n_drop = 4)
+ grad_modes = []
+ hook = model.model.layers[1].register_forward_hook(
+ lambda *_: grad_modes.append(torch.is_grad_enabled())
+ )
+ # A small token budget forces several right-padded batches.
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, max_batch_tokens = 40, device = "cpu")
+ hook.remove()
+ # No autograd graph may keep calibration activations alive, and weights stay trainable.
+ assert grad_modes and not any(grad_modes)
+ assert model.model.layers[1].mlp.experts.gate_up_proj.requires_grad
+ assert model._unsloth_reap_report == kept
+ for name, oracle in scores.items():
+ torch.testing.assert_close(
+ torch.tensor(model._unsloth_reap_saliency[name], dtype = torch.float64),
+ oracle,
+ rtol = 1e-9,
+ atol = 1e-12,
+ )
+ assert model.config.num_experts == 12
+ for layer in model.model.layers[1:]:
+ assert layer.mlp.experts.gate_up_proj.shape[0] == 12
+ assert layer.mlp.gate.weight.shape[0] == 12
+ assert layer.mlp.experts.num_experts == 12
+
+
+def test_batching_does_not_change_the_result():
+ samples = _samples(n = 9, seed = 3)
+ a, b = _qwen3_moe(seed = 5), _qwen3_moe(seed = 5)
+ reap.reap_prune(a, None, sparsity = 0.5, dataset = samples, max_batch_tokens = 1, device = "cpu")
+ reap.reap_prune(b, None, sparsity = 0.5, dataset = samples, max_batch_tokens = 10_000, device = "cpu")
+ assert a._unsloth_reap_report == b._unsloth_reap_report
+ for name in a._unsloth_reap_saliency:
+ torch.testing.assert_close(
+ torch.tensor(a._unsloth_reap_saliency[name]),
+ torch.tensor(b._unsloth_reap_saliency[name]),
+ rtol = 1e-9,
+ atol = 1e-12,
+ )
+
+
+def test_pruned_model_saves_reloads_and_writes_report(tmp_path):
+ from transformers import AutoModelForCausalLM
+
+ model = _qwen3_moe()
+ report = tmp_path / "sub" / "reap.json"
+ reap.reap_prune(
+ model, None, sparsity = 0.25, dataset = _samples(), device = "cpu", report_path = str(report)
+ )
+ assert json.loads(report.read_text(encoding = "utf-8")) == model._unsloth_reap_report
+ model.save_pretrained(tmp_path / "pruned")
+ reloaded = AutoModelForCausalLM.from_pretrained(
+ tmp_path / "pruned", dtype = torch.float64, experts_implementation = "eager"
+ ).eval()
+ assert reloaded.config.num_experts == 12
+ ids = torch.tensor([_samples(n = 1, seed = 9)[0]])
+ with torch.no_grad():
+ torch.testing.assert_close(reloaded(ids).logits, model(ids).logits)
+
+
+def test_group_limited_router_drops_per_group():
+ model = _deepseek_v3()
+ biases = [layer.mlp.gate.e_score_correction_bias.clone() for layer in model.model.layers]
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")
+ for index, (layer, bias) in enumerate(zip(model.model.layers, biases)):
+ kept = model._unsloth_reap_report[f"model.layers.{index}.mlp"]
+ # 16 experts in 4 groups of 4, one dropped per group.
+ assert [sum(1 for e in kept if e // 4 == g) for g in range(4)] == [3, 3, 3, 3]
+ torch.testing.assert_close(layer.mlp.gate.e_score_correction_bias, bias[kept])
+ assert layer.mlp.experts.gate_up_proj.shape[0] == 12
+ assert model.config.n_routed_experts == 12
+ with torch.no_grad():
+ model(torch.tensor([_samples(n = 1)[0]]))
+
+
+def test_rejects_bad_requests():
+ model = _qwen3_moe()
+ samples = _samples()
+ with pytest.raises(ValueError, match = "drops 0"):
+ reap.reap_prune(model, None, sparsity = 0.01, dataset = samples, device = "cpu")
+ with pytest.raises(ValueError, match = "too aggressive"):
+ reap.reap_prune(model, None, sparsity = 0.9, dataset = samples, device = "cpu")
+ with pytest.raises(ValueError, match = r"in \(0, 1\)"):
+ reap.reap_prune(model, None, sparsity = 1.0, dataset = samples, device = "cpu")
+ model.vllm_engine = object()
+ with pytest.raises(RuntimeError, match = "vLLM engine"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ del model.vllm_engine
+ object.__setattr__(model, "_unsloth_mtp_head", torch.nn.Linear(1, 1))
+ with pytest.raises(RuntimeError, match = "MTP head"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ del model.__dict__["_unsloth_mtp_head"]
+ for layout, match in (
+ ({"enable_expert_parallel": True}, "expert parallel"),
+ ({"tp_size": 2}, "tensor parallel"),
+ ({"pp_size": 2}, "pipeline parallel"),
+ ({"fsdp_size": 2}, "FSDP"),
+ ({"enable_sequence_parallel": True}, "sequence parallel"),
+ ):
+ model.config.distributed_config = layout
+ with pytest.raises(RuntimeError, match = match):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ # Size 1 is not model parallel.
+ model.config.distributed_config = {"tp_size": 1, "pp_size": 1}
+ assert reap._model_parallel_layout(model) is None
+ del model.config.distributed_config
+ model.config.use_bidirectional_attention = "all"
+ with pytest.raises(ValueError, match = "causal"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ model.config.use_bidirectional_attention = None
+ model.model.layers[1].self_attn.is_causal = False
+ with pytest.raises(ValueError, match = "causal"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ model.model.layers[1].self_attn.is_causal = True
+ layer0 = model.model.layers[0]
+ original = layer0.forward
+ layer0.forward = lambda hidden_states, prev_topk_indices = None, **kwargs: original(
+ hidden_states, **kwargs
+ ) # GLM-5 / HY-V4 hand top-k indices from layer to layer
+ with pytest.raises(ValueError, match = "extra state to the next layer"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ del layer0.forward
+ model.peft_config = {}
+ with pytest.raises(RuntimeError, match = "get_peft_model"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ # Nothing was touched by the refused calls.
+ del model.peft_config
+ assert model.model.layers[1].mlp.experts.gate_up_proj.shape[0] == 16
+
+
+def test_full_layer_rerun_matches_residual_shortcut(monkeypatch):
+ samples = _samples(n = 5, seed = 4)
+ a, b = _qwen3_moe(seed = 2), _qwen3_moe(seed = 2)
+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ monkeypatch.setattr(reap, "_residual_split", lambda layer, block: None)
+ reap.reap_prune(b, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ assert a._unsloth_reap_report == b._unsloth_reap_report
+ ids = torch.tensor([samples[0]])
+ with torch.no_grad():
+ torch.testing.assert_close(a(ids).logits, b(ids).logits)
+
+
+def test_save_directory_becomes_the_merge_base(tmp_path):
+ model = _qwen3_moe()
+ reap.reap_prune(
+ model,
+ None,
+ sparsity = 0.25,
+ dataset = _samples(),
+ device = "cpu",
+ save_directory = str(tmp_path / "pruned"),
+ )
+ # Later merged / GGUF saves of a LoRA on top read the base checkpoint from config._name_or_path.
+ assert model.config._name_or_path == str((tmp_path / "pruned").resolve())
+ assert model._unsloth_reap_saved == model.config._name_or_path == model.name_or_path
+ saved = json.loads((tmp_path / "pruned" / "config.json").read_text(encoding = "utf-8"))
+ assert saved.get("num_experts", saved.get("num_local_experts")) == 12
+
+
+ "build",
+ [
+ # Router weight in a child module (`gate.wg`); top_k per layer, as in Hunyuan-A13B.
+ lambda T, base: T.HunYuanMoEV1Config(
+ **{**base, "num_experts_per_tok": [2] * base["num_hidden_layers"]}, num_experts = 8
+ ),
+ # [1, E] score bias in `gate.moe_statics`.
+ lambda T, base: T.Ernie4_5_MoeConfig(
+ **base, moe_num_experts = 8, moe_layer_start_index = 0, moe_k = 2
+ ),
+ ],
+ ids = ["hunyuan_v1_moe", "ernie4_5_moe"],
+)
+def test_per_expert_router_tensors_in_child_modules_are_sliced(build, tmp_path):
+ from transformers import AutoModelForCausalLM
+
+ base = dict(
+ vocab_size = 96,
+ hidden_size = 32,
+ intermediate_size = 48,
+ moe_intermediate_size = 16,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 4,
+ head_dim = 8,
+ max_position_embeddings = 64,
+ num_experts_per_tok = 2,
+ )
+ torch.manual_seed(0)
+ model = AutoModelForCausalLM.from_config(
+ build(transformers, base), dtype = torch.float64, experts_implementation = "eager"
+ ).eval()
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")
+ for name, tensor in list(model.named_parameters()) + list(model.named_buffers()):
+ if ".mlp." in name and "shared" not in name:
+ assert 8 not in tuple(tensor.shape), (name, tuple(tensor.shape))
+ model.save_pretrained(tmp_path)
+ reloaded = AutoModelForCausalLM.from_pretrained(
+ tmp_path, dtype = torch.float64, experts_implementation = "eager"
+ ).eval()
+ ids = torch.tensor([_samples(n = 1, seed = 9)[0]])
+ with torch.no_grad():
+ torch.testing.assert_close(reloaded(ids).logits, model(ids).logits)
+
+
+def test_unsupported_weights_refuse_before_any_layer_is_pruned():
+ model = _qwen3_moe()
+ experts = model.model.layers[2].mlp.experts
+ experts.down_proj = torch.nn.Parameter(
+ torch.zeros(experts.down_proj.shape, dtype = torch.int8), requires_grad = False
+ )
+ with pytest.raises(RuntimeError, match = "16-bit or 32-bit"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")
+ assert model.model.layers[1].mlp.experts.gate_up_proj.shape[0] == 16
+
+
+def test_offloaded_activations_give_the_same_result():
+ samples = _samples(n = 6, seed = 7)
+ a, b = _qwen3_moe(seed = 3), _qwen3_moe(seed = 3)
+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, max_batch_tokens = 30, device = "cpu")
+ reap.reap_prune(
+ b,
+ None,
+ sparsity = 0.25,
+ dataset = samples,
+ max_batch_tokens = 30,
+ device = "cpu",
+ offload_hidden_states = True,
+ )
+ assert a._unsloth_reap_report == b._unsloth_reap_report
+ assert a._unsloth_reap_saliency == b._unsloth_reap_saliency
+
+
+def test_pretokenized_padding_is_dropped_via_attention_mask():
+ samples = _samples(n = 5, seed = 11)
+ width = max(len(x) for x in samples)
+ padded = [
+ {
+ "input_ids": [0] * (width - len(x)) + x,
+ "attention_mask": [0] * (width - len(x)) + [1] * len(x),
+ }
+ for x in samples
+ ]
+ a, b = _qwen3_moe(seed = 4), _qwen3_moe(seed = 4)
+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ reap.reap_prune(b, None, sparsity = 0.25, dataset = padded, device = "cpu")
+ assert a._unsloth_reap_saliency == b._unsloth_reap_saliency
+
+
+def test_latent_moe_scores_expert_outputs_in_hidden_space():
+ # Nemotron-H with moe_latent_size: experts run in a latent space and fc2_latent_proj maps
+ # their summed output back, so ||f_j|| is taken after that projection.
+ NemotronHConfig = getattr(transformers, "NemotronHConfig", None)
+ if NemotronHConfig is None:
+ pytest.skip("this transformers has no Nemotron-H")
+ from transformers import NemotronHForCausalLM
+
+ config = NemotronHConfig(
+ vocab_size = 96,
+ hidden_size = 32,
+ layers_block_type = ["moe", "attention", "moe"],
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ intermediate_size = 48,
+ n_routed_experts = 8,
+ n_shared_experts = 1,
+ moe_intermediate_size = 16,
+ moe_shared_expert_intermediate_size = 16,
+ moe_latent_size = 12,
+ num_experts_per_tok = 2,
+ n_group = 1,
+ topk_group = 1,
+ mamba_num_heads = 4,
+ mamba_head_dim = 8,
+ n_groups = 1,
+ )
+ torch.manual_seed(0)
+ model = NemotronHForCausalLM._from_config(
+ config, dtype = torch.float64, experts_implementation = "eager"
+ ).eval()
+ samples = _samples()
+ block = model.model.layers[0].mixer
+ # The experts call's own inputs (latent states, routing): the router's return tuple varies by version.
+ calls = []
+ hook = block.experts.register_forward_pre_hook(lambda module, args: calls.append(args[:3]))
+ with torch.no_grad():
+ for x in samples:
+ model(torch.tensor([x]))
+ hook.remove()
+ total = torch.zeros(8, dtype = torch.float64)
+ count = torch.zeros(8, dtype = torch.float64)
+ with torch.no_grad():
+ for latent, index, weights in calls:
+ for t in range(latent.shape[0]):
+ for s in range(index.shape[1]):
+ e = int(index[t, s])
+ f = block.experts(
+ latent[t : t + 1], index[t : t + 1, s : s + 1], torch.ones(1, 1)
+ )
+ total[e] += weights[t, s] * (f @ block.fc2_latent_proj.weight.T).norm()
+ count[e] += 1
+ expected = total / count.clamp(min = 1)
+
+ reap.reap_prune(model, None, sparsity = 0.5, dataset = samples, device = "cpu")
+ torch.testing.assert_close(
+ torch.tensor(model._unsloth_reap_saliency["model.layers.0.mixer"], dtype = torch.float64),
+ expected,
+ rtol = 1e-9,
+ atol = 1e-12,
+ )
+ assert model.model.layers[0].mixer.experts.up_proj.shape[0] == 4
+
+
+def test_refuses_layers_with_several_moe_blocks_and_keeps_training_modes():
+ model = _qwen3_moe()
+ samples = _samples()
+ layer = model.model.layers[1]
+ layer.vision_mlp = copy.deepcopy(layer.mlp) # e.g. Ernie 4.5 VL's text_moe + vision_moe
+ with pytest.raises(ValueError, match = "one MoE block per decoder layer"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ assert layer.mlp.experts.gate_up_proj.shape[0] == 16
+ del layer.vision_mlp
+
+ model.train()
+ model.model.layers[0].eval() # e.g. a frozen tower kept in eval mode
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ assert model.training and model.model.layers[1].training
+ assert not model.model.layers[0].training
+
+
+def test_chunked_scoring_matches_one_pass(monkeypatch):
+ samples = _samples(n = 6, seed = 5)
+ a, b = _qwen3_moe(seed = 6), _qwen3_moe(seed = 6)
+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ monkeypatch.setattr(reap, "_TAP_CHUNK_BYTES", 1) # one token per experts call
+ reap.reap_prune(b, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ assert a._unsloth_reap_report == b._unsloth_reap_report
+ for name in a._unsloth_reap_saliency:
+ torch.testing.assert_close(
+ torch.tensor(a._unsloth_reap_saliency[name], dtype = torch.float64),
+ torch.tensor(b._unsloth_reap_saliency[name], dtype = torch.float64),
+ rtol = 1e-9,
+ atol = 1e-12,
+ )
+
+
+def _saliency_rank(rank, init_file, shards, out):
+ import torch.distributed as dist
+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = 2)
+ try:
+ saliency = reap._Saliency(4)
+ ids, weights, norms = shards[rank]
+ if ids is not None:
+ saliency.add(ids, weights, norms)
+ saliency.all_reduce()
+ out.put((rank, saliency.total.tolist(), saliency.count.tolist()))
+ finally:
+ dist.destroy_process_group()
+
+
+def test_saliency_is_summed_across_ranks(tmp_path):
+ # Two gloo ranks with different calibration shards (rank 1 routed nothing) must both end with
+ # the one-process sums, so every rank of a DDP job prunes the same experts.
+ import multiprocessing
+
+ ids = torch.tensor([0, 1, 1, 3])
+ weights = torch.tensor([0.5, 1.0, 0.25, 2.0])
+ norms = torch.tensor([2.0, 1.0, 4.0, 0.5])
+ one = reap._Saliency(4)
+ one.add(ids, weights, norms)
+ shards = [(ids, weights, norms), (None, None, None)]
+ ctx = multiprocessing.get_context("spawn")
+ out = ctx.Queue()
+ procs = [
+ ctx.Process(target = _saliency_rank, args = (r, str(tmp_path / "pg"), shards, out))
+ for r in range(2)
+ ]
+ for p in procs:
+ p.start()
+ results = sorted(out.get(timeout = 120) for _ in procs)
+ for p in procs:
+ p.join(timeout = 60)
+ assert p.exitcode == 0
+ for _, total, count in results:
+ assert total == one.total.tolist() and count == one.count.tolist()
+ # Single process (no process group): unchanged.
+ before = one.total.clone()
+ one.all_reduce()
+ assert torch.equal(one.total, before)
+
+
+def test_saliency_reduces_on_the_accelerator_for_xccl_and_hccl(monkeypatch):
+ # Accelerator-only backends cannot reduce a CPU tensor, and their devices have no float64.
+ dist = torch.distributed
+ seen = []
+ monkeypatch.setattr(dist, "is_available", lambda: True)
+ monkeypatch.setattr(dist, "is_initialized", lambda: True)
+ monkeypatch.setattr(dist, "get_world_size", lambda *a, **k: 2)
+ monkeypatch.setattr(dist, "get_backend", lambda *a, **k: "xccl")
+ monkeypatch.setattr(
+ dist, "all_reduce", lambda t, *a, **k: seen.append((t.device.type, t.dtype))
+ )
+ monkeypatch.setattr(
+ torch.accelerator, "current_accelerator", lambda *a, **k: torch.device("meta")
+ )
+ monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 0, raising = False)
+ saliency = reap._Saliency(4)
+ saliency.all_reduce()
+ assert seen == [("meta", torch.float32)]
+
+
+def test_dataset_dict_uses_a_split_not_its_names():
+ # load_dataset(...) without split= returns a DatasetDict; its iteration yields split names.
+ datasets = pytest.importorskip("datasets")
+ samples = _samples(n = 6, seed = 9)
+ split = datasets.Dataset.from_list([{"input_ids": ids} for ids in samples])
+ model = _qwen3_moe()
+ a = copy.deepcopy(model)
+ b = copy.deepcopy(model)
+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ reap.reap_prune(b, None, sparsity = 0.25, dataset = datasets.DatasetDict(train = split), device = "cpu")
+ assert a._unsloth_reap_report == b._unsloth_reap_report
+ two = datasets.DatasetDict(validation = split, test = split)
+ with pytest.raises(ValueError, match = "splits"):
+ reap.reap_prune(copy.deepcopy(model), None, sparsity = 0.25, dataset = two, device = "cpu")
+
+
+def _prune_rank(
+ rank,
+ init_file,
+ save_dir,
+ out,
+ fail_save = False,
+ bad_shard = False,
+ report = False,
+):
+ import json
+ import os
+ import torch.distributed as dist
+
+ os.environ["UNSLOTH_MOE_BACKEND"] = "native_torch"
+ torch.set_num_threads(1)
+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = 2)
+ try:
+ model = _qwen3_moe(seed = 0)
+ original = model.save_pretrained
+
+ def save(*args, **kwargs):
+ if fail_save:
+ raise OSError("No space left on device")
+ with open(os.path.join(os.path.dirname(save_dir), f"saved_by_{rank}"), "w") as f:
+ f.write("1")
+ return original(*args, **kwargs)
+
+ model.save_pretrained = save
+ report_path = None
+ if report:
+ report_path = os.path.join(os.path.dirname(save_dir), "report.json")
+ original_dump = json.dump
+
+ def dump(obj, f, *args, **kwargs):
+ if getattr(f, "name", "") == report_path:
+ if report == "fail":
+ raise OSError("Read-only file system")
+ with open(
+ os.path.join(os.path.dirname(save_dir), f"report_by_{rank}"), "w"
+ ) as m:
+ m.write("1")
+ return original_dump(obj, f, *args, **kwargs)
+
+ json.dump = dump
+ if bad_shard == "move" and rank == 1:
+
+ def fail_move(*args, **kwargs):
+ raise RuntimeError("CUDA out of memory while streaming the layer in")
+
+ reap._stream_in = fail_move
+ if bad_shard == "slice" and rank == 1:
+
+ def fail_slice(*args, **kwargs):
+ raise RuntimeError("out of memory while slicing experts")
+
+ reap._prune_block = fail_slice
+ if bad_shard == "post" and rank == 1:
+ # Rank 1 fails in the post-prune pass (an OOM on a larger shard, say).
+ original_prune = reap._prune_block
+
+ def prune_then_fail(block, *args, **kwargs):
+ original_prune(block, *args, **kwargs)
+
+ def boom(*_):
+ raise RuntimeError("out of memory in the post-prune pass")
+
+ block.register_forward_hook(boom)
+
+ reap._prune_block = prune_then_fail
+ # Each rank calibrates on its own shard, as a DDP job would.
+ try:
+ reap.reap_prune(
+ model,
+ None,
+ sparsity = 0.25,
+ dataset = (
+ ([[10**6] * 6] if bad_shard == "out_of_range" else [])
+ if bad_shard in (True, "out_of_range") and rank == 1
+ else _samples(n = 5, seed = 20 + rank)
+ ),
+ device = "cpu",
+ save_directory = save_dir,
+ report_path = report_path,
+ )
+ except Exception as e:
+ out.put((rank, "error", str(e)))
+ return
+ out.put((rank, model._unsloth_reap_report, model.config._name_or_path))
+ finally:
+ dist.destroy_process_group()
+
+
+def test_ddp_ranks_keep_the_same_experts_and_one_writes(tmp_path):
+ import multiprocessing
+
+ save_dir = str(tmp_path / "pruned")
+ ctx = multiprocessing.get_context("spawn")
+ out = ctx.Queue()
+ procs = [
+ ctx.Process(target = _prune_rank, args = (r, str(tmp_path / "pg"), save_dir, out))
+ for r in range(2)
+ ]
+ for p in procs:
+ p.start()
+ results = sorted(out.get(timeout = 300) for _ in procs)
+ for p in procs:
+ p.join(timeout = 60)
+ assert p.exitcode == 0
+ (_, report0, path0), (_, report1, path1) = results
+ assert report0 == report1 and path0 == path1 == save_dir
+ assert (tmp_path / "saved_by_0").exists() and not (tmp_path / "saved_by_1").exists()
+ # The one shared-data answer: both shards together, in one process.
+ one = _qwen3_moe(seed = 0)
+ reap.reap_prune(
+ one,
+ None,
+ sparsity = 0.25,
+ dataset = _samples(n = 5, seed = 20) + _samples(n = 5, seed = 21),
+ device = "cpu",
+ )
+ assert one._unsloth_reap_report == report0
+
+
+def test_a_failed_rank_zero_save_stops_every_rank(tmp_path):
+ # Rank 0's write fails (disk full): every rank must raise, none may wait forever.
+ import multiprocessing
+
+ ctx = multiprocessing.get_context("spawn")
+ out = ctx.Queue()
+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, True)
+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]
+ for p in procs:
+ p.start()
+ results = sorted(out.get(timeout = 300) for _ in procs)
+ for p in procs:
+ p.join(timeout = 60)
+ assert p.exitcode == 0
+ assert results[0][1] == "error" and "No space left" in results[0][2]
+ assert results[1][1] == "error" and "another rank failed to save" in results[1][2]
+
+
+def _run_ranks(tmp_path, **kwargs):
+ import multiprocessing
+
+ ctx = multiprocessing.get_context("spawn")
+ out = ctx.Queue()
+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out)
+ procs = [ctx.Process(target = _prune_rank, args = (r, *args), kwargs = kwargs) for r in range(2)]
+ for p in procs:
+ p.start()
+ results = sorted(out.get(timeout = 300) for _ in procs)
+ for p in procs:
+ p.join(timeout = 60)
+ assert p.exitcode == 0
+ return results
+
+
+def test_ddp_report_is_written_by_rank_zero_only(tmp_path):
+ (_, report0, _), (_, report1, _) = _run_ranks(tmp_path, report = True)
+ assert report0 == report1
+ assert (tmp_path / "report_by_0").exists() and not (tmp_path / "report_by_1").exists()
+ assert json.loads((tmp_path / "report.json").read_text(encoding = "utf-8")) == report0
+
+
+def test_a_failed_rank_zero_report_write_stops_every_rank(tmp_path):
+ results = _run_ranks(tmp_path, report = "fail")
+ assert results[0][1] == "error" and "Read-only file system" in results[0][2]
+ assert (
+ results[1][1] == "error" and "another rank failed to write the REAP report" in results[1][2]
+ )
+
+
+def test_one_bad_calibration_shard_stops_every_rank(tmp_path):
+ # Rank 1's shard is empty: it must not leave rank 0 waiting in the first per-layer collective.
+ import multiprocessing
+
+ ctx = multiprocessing.get_context("spawn")
+ out = ctx.Queue()
+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, False, True)
+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]
+ for p in procs:
+ p.start()
+ results = sorted(out.get(timeout = 300) for _ in procs)
+ for p in procs:
+ p.join(timeout = 60)
+ assert p.exitcode == 0
+ assert results[0][1] == "error" and "another rank failed to prepare" in results[0][2]
+ assert results[1][1] == "error"
+
+
+def test_group_metadata_must_be_complete():
+ model = _qwen3_moe()
+ model.config.n_group = 2
+ with pytest.raises(ValueError, match = "no topk_group"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")
+ # n_group 1 (or a lone topk_group) routes ungrouped and prunes globally.
+ model.config.n_group = 1
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")
+ assert model._unsloth_reap_report
+
+
+def test_one_rank_failing_at_capture_stops_every_rank(tmp_path):
+ # Rank 1's shard has an out-of-range token id: its embedding lookup fails while capturing.
+ import multiprocessing
+
+ ctx = multiprocessing.get_context("spawn")
+ out = ctx.Queue()
+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, False, "out_of_range")
+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]
+ for p in procs:
+ p.start()
+ results = sorted(out.get(timeout = 300) for _ in procs)
+ for p in procs:
+ p.join(timeout = 60)
+ assert p.exitcode == 0
+ assert results[0][1] == "error" and "another rank failed to capture" in results[0][2]
+ assert results[1][1] == "error"
+
+
+def test_metadata_free_streaming_token_rows_need_no_tokenizer():
+ datasets = pytest.importorskip("datasets")
+ samples = _samples(n = 6, seed = 13)
+
+ def rows():
+ for ids in samples:
+ yield {"input_ids": ids}
+
+ stream = datasets.IterableDataset.from_generator(rows)
+ model = _qwen3_moe()
+ a, b = copy.deepcopy(model), copy.deepcopy(model)
+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, device = "cpu")
+ reap.reap_prune(b, None, sparsity = 0.25, dataset = stream, device = "cpu")
+ assert a._unsloth_reap_report == b._unsloth_reap_report
+ # Text rows still need one, now reported when the first text row is read.
+ with pytest.raises(ValueError, match = "needs a tokenizer"):
+ reap.reap_prune(
+ copy.deepcopy(model), None, sparsity = 0.25, dataset = [{"text": "hi"}], device = "cpu"
+ )
+
+
+def test_failure_after_pruning_a_layer_marks_the_model_unsaved(monkeypatch):
+ model = _qwen3_moe()
+ model._unsloth_reap_saved = "/stale/earlier/save"
+ original = reap._prune_block
+ calls = []
+
+ def flaky(*args, **kwargs):
+ calls.append(1)
+ if len(calls) == 2:
+ raise RuntimeError("out of memory")
+ return original(*args, **kwargs)
+
+ monkeypatch.setattr(reap, "_prune_block", flaky)
+ with pytest.raises(RuntimeError, match = "out of memory"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")
+ # The completed layer keeps its experts; the one cut short is recorded too, both unsaved.
+ done, cut = list(model._unsloth_reap_report.values())
+ assert done and cut == [] and model._unsloth_reap_saved is None
+
+
+def test_one_rank_failing_after_pruning_stops_every_rank(tmp_path):
+ import multiprocessing
+
+ ctx = multiprocessing.get_context("spawn")
+ out = ctx.Queue()
+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, False, "post")
+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]
+ for p in procs:
+ p.start()
+ results = sorted(out.get(timeout = 300) for _ in procs)
+ for p in procs:
+ p.join(timeout = 60)
+ assert p.exitcode == 0
+ assert results[0][1] == "error" and "another rank failed to run decoder layer" in results[0][2]
+ assert results[1][1] == "error" and "post-prune" in results[1][2]
+
+
+def test_failure_inside_the_first_slice_still_marks_the_model(monkeypatch):
+ model = _qwen3_moe()
+ model._unsloth_reap_saved = "/stale/earlier/save"
+
+ def half_sliced(block, experts, router, retained, num_experts):
+ experts.gate_up_proj.data = experts.gate_up_proj.data[retained]
+ raise RuntimeError("failed after slicing one tensor")
+
+ monkeypatch.setattr(reap, "_prune_block", half_sliced)
+ with pytest.raises(RuntimeError, match = "one tensor"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")
+ assert model._unsloth_reap_report and model._unsloth_reap_saved is None
+
+
+def test_one_rank_failing_to_slice_stops_every_rank(tmp_path):
+ import multiprocessing
+
+ ctx = multiprocessing.get_context("spawn")
+ out = ctx.Queue()
+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, False, "slice")
+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]
+ for p in procs:
+ p.start()
+ results = sorted(out.get(timeout = 300) for _ in procs)
+ for p in procs:
+ p.join(timeout = 60)
+ assert p.exitcode == 0
+ assert results[0][1] == "error" and "another rank failed to run decoder layer" in results[0][2]
+ assert results[1][1] == "error" and "slicing experts" in results[1][2]
+
+
+def test_several_moe_decoder_stacks_are_refused():
+ class TwoStacks(torch.nn.Module):
+ def __init__(self, a, b):
+ super().__init__()
+ self.config = a.config
+ self.text = a.model
+ self.other = b.model
+
+ a, b = _qwen3_moe(seed = 0), _qwen3_moe(seed = 1)
+ model = TwoStacks(a, b)
+ experts = next(m for n, m in model.other.named_modules() if n.endswith("mlp.experts"))
+ before = experts.gate_up_proj.shape
+ with pytest.raises(ValueError, match = "several decoder stacks"):
+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")
+ # Refused before anything changed.
+ assert experts.gate_up_proj.shape == before
+ assert model.config.num_experts == before[0]
+
+
+def test_one_rank_failing_to_stream_a_layer_in_stops_every_rank(tmp_path):
+ import multiprocessing
+
+ ctx = multiprocessing.get_context("spawn")
+ out = ctx.Queue()
+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, False, "move")
+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]
+ for p in procs:
+ p.start()
+ results = sorted(out.get(timeout = 300) for _ in procs)
+ for p in procs:
+ p.join(timeout = 60)
+ assert p.exitcode == 0
+ assert (
+ results[0][1] == "error" and "another rank failed to run decoder layer 1" in results[0][2]
+ )
+ assert results[1][1] == "error" and "streaming the layer in" in results[1][2]
+
+
+def test_shared_layer_state_follows_each_layer_without_offloading(monkeypatch):
+ # Gemma 4 passes one mutable `shared_kv_states` mapping to every layer. Under a device_map the
+ # layers sit on different GPUs, so it must be moved to each layer's device even when the
+ # activations stay put (no offload store).
+ from collections import UserDict
+
+ model = _qwen3_moe(seed = 0)
+ shared = UserDict({"kv": torch.zeros(1)})
+ capture = reap._capture_inputs
+
+ def capture_with_shared(*args, **kwargs):
+ h, per_layer = capture(*args, **kwargs)
+ return h, [(a, {**kw, "shared_kv_states": shared}) for a, kw in per_layer]
+
+ moved = []
+ to = reap._to
+
+ def recording_to(obj, device, *rest):
+ if isinstance(obj, dict) and obj.keys() == {"kv"}:
+ moved.append(torch.device(device))
+ return to(obj, device, *rest)
+
+ monkeypatch.setattr(reap, "_capture_inputs", capture_with_shared)
+ monkeypatch.setattr(reap, "_to", recording_to)
+ reap.reap_prune(
+ model,
+ None,
+ sparsity = 0.5,
+ dataset = _samples(),
+ device = "cpu",
+ offload_hidden_states = False,
+ )
+ assert moved and set(moved) == {torch.device("cpu")}
+
+
+def test_mistral_common_backend_is_not_unwrapped():
+ # MistralCommonBackend is an HF tokenizer whose `.tokenizer` is the raw mistral_common one,
+ # which has neither __call__ nor apply_chat_template. Only processors are unwrapped.
+ from transformers import ProcessorMixin
+
+ class Raw:
+ pass
+
+ class Backend:
+ tokenizer = Raw()
+ chat_template = "{{ messages[0]['content'] }}"
+ pad_token_id = 0
+
+ def __call__(
+ self,
+ text,
+ truncation = True,
+ max_length = None,
+ ):
+ return {"input_ids": [5, 6, 7][:max_length]}
+
+ def apply_chat_template(
+ self,
+ messages,
+ tokenize = False,
+ **kwargs,
+ ):
+ return messages[0]["content"]
+
+ backend = Backend()
+ assert reap._text_tokenizer(backend) is backend
+ ids = reap._calibration_ids([{"messages": [{"role": "user", "content": "hi"}]}], backend, 4, 8)
+ assert ids == [[5, 6, 7]]
+ processor = ProcessorMixin.__new__(ProcessorMixin)
+ processor.tokenizer = backend
+ assert reap._text_tokenizer(processor) is backend
+
+
+def test_saved_pruned_checkpoint_carries_its_provenance(tmp_path):
+ import json
+
+ model = _qwen3_moe(seed = 0)
+ source = model.config._name_or_path
+ reap.reap_prune(
+ model, None, sparsity = 0.5, dataset = _samples(), device = "cpu", save_directory = str(tmp_path)
+ )
+ data = json.loads((tmp_path / reap.REAP_PROVENANCE_FILE).read_text(encoding = "utf-8"))
+ assert data["source_model"] == (source or None)
+ assert data["kept_experts"] == {k: list(v) for k, v in model._unsloth_reap_report.items()}
+
+
+def test_provenance_names_a_local_source_without_its_path(tmp_path):
+ import json
+
+ model = _qwen3_moe(seed = 0)
+ local = tmp_path / "home" / "me" / "my-moe"
+ local.mkdir(parents = True)
+ model.config._name_or_path = str(local)
+ out = tmp_path / "pruned"
+ reap.reap_prune(
+ model, None, sparsity = 0.5, dataset = _samples(), device = "cpu", save_directory = str(out)
+ )
+ data = json.loads((out / reap.REAP_PROVENANCE_FILE).read_text(encoding = "utf-8"))
+ assert data["source_model"] == "my-moe"
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 04bf15cd070..6389535a8df 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -28,6 +28,7 @@ from .mistral import FastMistralModel
from .qwen2 import FastQwen2Model
from .qwen3 import FastQwen3Model
from .qwen3_moe import FastQwen3MoeModel
+from .reap import reap_prune as _reap_prune, raise_if_unsaved_reap
from .cohere import FastCohereModel
from transformers import AutoConfig
from transformers import __version__ as transformers_version
@@ -1089,10 +1090,13 @@ def _drop_vllm_engine_kwargs(kwargs):
class FastLanguageModel(FastLlamaModel):
+ reap_prune = staticmethod(_reap_prune)
+
# Text-diffusion models load through FastModel's slow path; these route them there for the PEFT / mode helpers.
@staticmethod
@functools.wraps(FastLlamaModel.get_peft_model)
def get_peft_model(model, *args, **kwargs):
+ raise_if_unsaved_reap(model)
if getattr(model, "_unsloth_slow_diffusion", False):
return FastDiffusionModel.get_peft_model(
model, **_diffusion_peft_kwargs(model, args, kwargs)
@@ -2007,6 +2011,8 @@ except:
class FastModel(FastBaseModel):
+ reap_prune = staticmethod(_reap_prune)
+
@staticmethod
def get_mtp_head(model, *args, **kwargs):
from .speculative import get_mtp_head
@@ -2019,6 +2025,7 @@ class FastModel(FastBaseModel):
@staticmethod
def get_peft_model(model, *args, **kwargs):
+ raise_if_unsaved_reap(model)
# Route text-diffusion models (slow path) to the transformers-only PEFT helper.
if getattr(model, "_unsloth_slow_diffusion", False):
return FastDiffusionModel.get_peft_model(
diff --git a/unsloth/models/reap.py b/unsloth/models/reap.py
new file mode 100644
index 00000000000..05ddf37dc13
--- /dev/null
+++ b/unsloth/models/reap.py
@@ -0,0 +1,1123 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""REAP expert pruning (Router-weighted Expert Activation Pruning, arXiv:2510.13999).
+
+Same saliency, expert selection and schedule as llm-compressor's ``REAPPruningModifier``: each
+decoder layer is calibrated on the outputs of the already pruned layers before it, its lowest
+saliency experts are dropped, and the pruned layer produces the next layer's inputs. Expert
+outputs come from the model's own fused experts kernel (one call with every routed (token, slot)
+pair as its own row) instead of per-expert Linear modules, so no extra dependency is needed.
+"""
+
+import inspect
+import json
+from collections.abc import MutableMapping
+import os
+import time
+
+import torch
+import torch.nn as nn
+
+__all__ = ["reap_prune", "raise_if_unsaved_reap"]
+
+_NUM_EXPERTS_KEYS = ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts")
+_NUM_EXPERTS_MODULE_KEYS = ("num_experts", "n_experts", "n_routed_experts", "num_local_experts")
+_TOP_K_KEYS = ("num_experts_per_tok", "top_k", "moe_top_k", "top_k_experts")
+_ROUTER_ATTRS = ("gate", "router")
+_DEFAULT_DATASET = "mlabonne/open-perfectblend"
+_TAP_CHUNK_BYTES = 256 << 20
+_STATEFUL_LAYERS = (
+ "Unsloth: REAP calibrates one decoder layer at a time, so it does not support decoder layers "
+ "that hand extra state to the next layer (GLM-5 / HY-V4 sparse-attention top-k indices, ZAYA "
+ "router states)."
+)
+# llm-compressor's fallback for tokenizers without a chat template (PerfectBlendDataset).
+_DEFAULT_CHAT_TEMPLATE = (
+ "{% for message in messages %}\n"
+ "{% if message['role'] == 'user' %}\n"
+ "{{ '<|user|>\n' + message['content'] + eos_token }}\n"
+ "{% elif message['role'] == 'system' %}\n"
+ "{{ '<|system|>\n' + message['content'] + eos_token }}\n"
+ "{% elif message['role'] == 'assistant' %}\n"
+ "{{ '<|assistant|>\n' + message['content'] + eos_token }}\n"
+ "{% endif %}\n"
+ "{% if loop.last and add_generation_prompt %}\n"
+ "{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}"
+)
+
+
+class _StopForward(Exception):
+ pass
+
+
+def _configs(model):
+ config = model.config
+ text_config = getattr(config, "text_config", None)
+ return [c for c in (config, text_config) if c is not None]
+
+
+def _is_int(value):
+ return isinstance(value, int) and not isinstance(value, bool)
+
+
+def _first_attr(
+ configs,
+ keys,
+ per_layer = False,
+):
+ for config in reversed(configs): # text_config first
+ for key in keys:
+ value = getattr(config, key, None)
+ if _is_int(value):
+ return value
+ # HunYuan stores `moe_topk` per layer (`[8] * 32`); the largest bounds every layer.
+ if (
+ per_layer
+ and isinstance(value, (list, tuple))
+ and value
+ and all(map(_is_int, value))
+ ):
+ return max(value)
+ return None
+
+
+def _moe_parts(module, num_experts):
+ """(experts, router) when ``module`` is a fused-experts MoE block, else None."""
+ experts = getattr(module, "experts", None)
+ if not isinstance(experts, nn.Module) or isinstance(experts, nn.ModuleList):
+ return None
+ router = next(
+ (
+ getattr(module, a)
+ for a in _ROUTER_ATTRS
+ if isinstance(getattr(module, a, None), nn.Module)
+ ),
+ None,
+ )
+ if router is None:
+ return None
+ try:
+ params = list(inspect.signature(experts.forward).parameters.values())
+ except (TypeError, ValueError):
+ params = []
+ # The tap needs forward(hidden_states, top_k_index, top_k_weights); Llama4 passes one tensor.
+ positional = [p for p in params if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)]
+ if params and len(positional) < 3 and not any(p.kind == p.VAR_POSITIONAL for p in params):
+ return None
+ if not any(
+ p.dim() == 3 and p.shape[0] == num_experts for p in experts.parameters(recurse = False)
+ ):
+ return None
+ return experts, router
+
+
+def _find_layers(model, num_experts):
+ """(owner, layers): the ModuleList of decoder layers holding the MoE blocks and its parent."""
+ best, stacks = None, set()
+ for name, module in model.named_modules():
+ layers = getattr(module, "layers", None)
+ if not isinstance(layers, nn.ModuleList) or len(layers) == 0:
+ continue
+ n_moe = sum(
+ any(_moe_parts(m, num_experts) is not None for m in layer.modules()) for layer in layers
+ )
+ if n_moe and (best is None or n_moe > best[0]):
+ best = (n_moe, module, layers)
+ if n_moe:
+ stacks.add(id(layers))
+ if best is None:
+ return None, None
+ if len(stacks) > 1:
+ # Pruning one stack would still rewrite the expert counts the other stacks read.
+ raise ValueError(
+ "Unsloth: REAP found MoE layers in several decoder stacks (e.g. separate multimodal "
+ "decoders); it prunes models with one MoE decoder stack."
+ )
+ return best[1], best[2]
+
+
+def _layer_moes(layer, num_experts):
+ found, seen = [], set()
+ for name, module in layer.named_modules():
+ parts = _moe_parts(module, num_experts)
+ if parts is not None and id(parts[0]) not in seen:
+ seen.add(id(parts[0]))
+ found.append((name, module, parts[0], parts[1]))
+ return found
+
+
+def _layer_moe(layer, num_experts):
+ found = _layer_moes(layer, num_experts)
+ return found[0] if found else None
+
+
+def _check_weights(experts, router, layer_name):
+ # The router recursively: HunYuan / AfMoE keep its Linear in a child module (bnb would quantize it).
+ for owner, kind, recurse in ((experts, "experts", False), (router, "router", True)):
+ for name, p in owner.named_parameters(recurse = recurse):
+ if not p.is_floating_point() or p.element_size() < 2 or p.is_meta:
+ raise RuntimeError(
+ f"Unsloth: REAP needs 16-bit or 32-bit MoE weights, but {layer_name} {kind}.{name} "
+ f"is {p.dtype}{' on the meta device' if p.is_meta else ''}. Load the model with "
+ "`load_in_4bit = False` (and without fp8 / offloaded weights), then run REAP."
+ )
+
+
+def _row_text(row, tokenizer, chat):
+ if "text" in row and isinstance(row["text"], str):
+ return row["text"]
+ messages = row.get("messages", row.get("conversations"))
+ if not isinstance(messages, list):
+ raise ValueError(
+ "Unsloth: each REAP calibration row needs `input_ids`, `text`, `messages` or "
+ f"`conversations`; got columns {list(row)}."
+ )
+ messages = [
+ {
+ "role": {"human": "user", "gpt": "assistant"}.get(
+ m.get("role", m.get("from")), m.get("role", m.get("from"))
+ ),
+ "content": m.get("content", m.get("value")),
+ }
+ for m in messages
+ ]
+ kwargs = (
+ {} if getattr(chat, "chat_template", None) else {"chat_template": _DEFAULT_CHAT_TEMPLATE}
+ )
+ return chat.apply_chat_template(messages, tokenize = False, add_generation_prompt = False, **kwargs)
+
+
+def _text_tokenizer(tokenizer):
+ """The text tokenizer of a processor. Only a processor is unwrapped: MistralCommonBackend is
+ itself an HF tokenizer but keeps its raw mistral_common backend in ``.tokenizer``."""
+ try:
+ from transformers import ProcessorMixin
+ except ImportError:
+ return tokenizer
+ return tokenizer.tokenizer if isinstance(tokenizer, ProcessorMixin) else tokenizer
+
+
+def _calibration_ids(dataset, tokenizer, num_samples, max_seq_length):
+ """Token id lists, truncated to ``max_seq_length`` like llm-compressor's tokenize step."""
+ text_tokenizer = _text_tokenizer(tokenizer)
+ chat = tokenizer if getattr(tokenizer, "chat_template", None) else text_tokenizer
+ if dataset is None:
+ dataset = _DEFAULT_DATASET
+ if isinstance(dataset, str):
+ from datasets import load_dataset
+ dataset = load_dataset(dataset, split = "train", streaming = True)
+ ids = []
+ for row in dataset:
+ if len(ids) >= num_samples:
+ break
+ if isinstance(row, str):
+ row = {"text": row}
+ elif isinstance(row, (list, tuple)):
+ row = {"input_ids": row}
+ if "input_ids" in row:
+ sample = list(row["input_ids"])
+ mask = row.get("attention_mask")
+ if mask is not None: # padded pretokenized rows: keep the real tokens only
+ sample = [t for t, m in zip(sample, list(mask)) if m]
+ sample = sample[:max_seq_length]
+ else:
+ if tokenizer is None:
+ raise ValueError(
+ "Unsloth: REAP needs a tokenizer unless the dataset has `input_ids`."
+ )
+ sample = text_tokenizer(
+ _row_text(row, tokenizer, chat), truncation = True, max_length = max_seq_length
+ )["input_ids"]
+ if len(sample):
+ ids.append(sample)
+ if not ids:
+ raise ValueError("Unsloth: REAP got no calibration samples from the dataset.")
+ return ids
+
+
+def _batches(ids, max_batch_tokens, pad_id):
+ """Length-sorted right-padded batches. Right padding cannot change real tokens in a causal
+ model, so no attention mask is passed and padded positions are only masked out of saliency."""
+ order = sorted(range(len(ids)), key = lambda i: -len(ids[i]))
+ batches, current = [], []
+ for i in order:
+ width = len(ids[current[0]]) if current else len(ids[i])
+ if current and (len(current) + 1) * width > max_batch_tokens:
+ batches.append(current)
+ current = []
+ current.append(i)
+ if current:
+ batches.append(current)
+ out = []
+ for batch in batches:
+ width = len(ids[batch[0]])
+ input_ids = torch.full((len(batch), width), pad_id, dtype = torch.long)
+ valid = torch.zeros((len(batch), width), dtype = torch.bool)
+ for row, i in enumerate(batch):
+ input_ids[row, : len(ids[i])] = torch.tensor(ids[i], dtype = torch.long)
+ valid[row, : len(ids[i])] = True
+ out.append((input_ids, valid))
+ return out
+
+
+def _to(
+ obj,
+ device,
+ memo = None,
+):
+ """Moves nested tensors; ``memo`` keeps tensors shared across layers (rotary tables) shared."""
+ if torch.is_tensor(obj):
+ if memo is None:
+ return obj.to(device, non_blocking = True)
+ if id(obj) not in memo:
+ memo[id(obj)] = obj.to(device)
+ return memo[id(obj)]
+ if isinstance(obj, tuple):
+ return tuple(_to(o, device, memo) for o in obj)
+ if isinstance(obj, list):
+ return [_to(o, device, memo) for o in obj]
+ if isinstance(obj, dict):
+ return {k: _to(v, device, memo) for k, v in obj.items()}
+ return obj
+
+
+def _tensor_bytes(obj, seen = None):
+ """Bytes of the distinct tensors nested in ``obj`` (masks shared by every layer count once)."""
+ seen = set() if seen is None else seen
+ if torch.is_tensor(obj):
+ if id(obj) in seen:
+ return 0
+ seen.add(id(obj))
+ return obj.numel() * obj.element_size()
+ if isinstance(obj, (tuple, list)):
+ return sum(_tensor_bytes(o, seen) for o in obj)
+ if isinstance(obj, dict):
+ return sum(_tensor_bytes(o, seen) for o in obj.values())
+ return 0
+
+
+def _capture_inputs(owner, layers, input_ids):
+ """Embedding output plus the per-layer kwargs (masks, rotary embeddings, ...) the model
+ passes to each decoder layer, by running the model with every layer stubbed out."""
+ captured = [None] * len(layers)
+ first = {}
+ originals = []
+ for i, layer in enumerate(layers):
+ originals.append(layer.__dict__.get("forward"))
+
+ def stub(
+ *args,
+ _i = i,
+ **kwargs,
+ ):
+ hidden = args[0] if args else kwargs.pop("hidden_states")
+ if _i == 0:
+ first["hidden"] = hidden
+ captured[_i] = (tuple(args[1:]), kwargs)
+ if _i == len(layers) - 1:
+ raise _StopForward
+ return hidden
+
+ layer.forward = stub
+ try:
+ owner(input_ids = input_ids, use_cache = False)
+ except _StopForward:
+ pass
+ except (TypeError, ValueError) as error:
+ if "unpack" not in str(error):
+ raise
+ raise ValueError(_STATEFUL_LAYERS) from error
+ finally:
+ for layer, original in zip(layers, originals):
+ if original is None:
+ del layer.forward
+ else:
+ layer.forward = original
+ return first["hidden"], captured
+
+
+def _layer_device(layer, default):
+ for p in layer.parameters():
+ return p.device
+ return default
+
+
+def _hidden(out):
+ return out[0] if isinstance(out, (tuple, list)) else out
+
+
+def _collective_device():
+ """(device, accumulation dtype) for a collective on the current process group: CUDA for NCCL,
+ CPU for gloo, else the current accelerator (HCCL on Ascend, XCCL on Intel XPU). MPS, XPU and
+ NPU have no (or slow) float64, so they accumulate in float32 like ``_Saliency.add``."""
+ backend = str(torch.distributed.get_backend())
+ if "nccl" in backend:
+ device = torch.device("cuda", torch.cuda.current_device())
+ elif backend == "gloo":
+ device = torch.device("cpu")
+ else:
+ # torch 2.6 names the index getter current_device_idx.
+ accel = torch.accelerator
+ index = getattr(accel, "current_device_index", None) or accel.current_device_idx
+ device = torch.device(accel.current_accelerator().type, index())
+ return device, torch.float64 if device.type in ("cuda", "cpu") else torch.float32
+
+
+# Written next to a pruned checkpoint saved by reap_prune(..., save_directory).
+REAP_PROVENANCE_FILE = "unsloth_reap.json"
+
+
+def _stream_in(layer, device):
+ """``(device the layer runs on, whether it was moved)``: a CPU-resident layer (not under
+ accelerate hooks) is moved to ``device`` for its turn."""
+ layer_device = _layer_device(layer, device)
+ if layer_device.type == "cpu" and device.type != "cpu" and not hasattr(layer, "_hf_hook"):
+ layer.to(device)
+ return device, True
+ return layer_device, False
+
+
+def _ranks_agree(error, what):
+ """Every rank learns whether any rank failed ``what``, so all stop together instead of some
+ waiting in a later collective. Re-raises this rank's own error. No-op without a process group."""
+ dist = torch.distributed
+ if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() < 2:
+ if error is not None:
+ raise error
+ return
+ device, acc = _collective_device()
+ failed = torch.tensor([1.0 if error is not None else 0.0], dtype = acc, device = device)
+ dist.all_reduce(failed)
+ if error is not None:
+ raise error
+ if failed.item():
+ raise RuntimeError(f"Unsloth: REAP stopped because another rank failed to {what}.")
+
+
+class _Saliency:
+ """Running sum of ``gate * ||expert output||`` and routed-token count per expert (float64)."""
+
+ def __init__(self, num_experts):
+ self.num_experts = num_experts
+ self.total = None
+ self.count = None
+
+ def add(
+ self,
+ expert_ids,
+ weights,
+ norms,
+ mask = None,
+ ):
+ # MPS, consumer Intel XPUs and Ascend NPUs have no (or slow, partial) float64.
+ acc = torch.float64 if norms.device.type in ("cuda", "cpu") else torch.float32
+ if self.total is None:
+ self.total = torch.zeros(self.num_experts, dtype = acc, device = norms.device)
+ self.count = torch.zeros(self.num_experts, dtype = acc, device = norms.device)
+ ones = torch.ones_like(norms, dtype = acc) if mask is None else mask.to(acc)
+ self.total.index_add_(0, expert_ids, weights.to(acc) * norms.to(acc) * ones)
+ self.count.index_add_(0, expert_ids, ones)
+
+ def all_reduce(self):
+ """Sum over the ranks of a distributed job, so every rank keeps the same experts (a DDP
+ job with sharded calibration data would otherwise pair unrelated experts across ranks)."""
+ dist = torch.distributed
+ if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() < 2:
+ return
+ device, acc = _collective_device()
+ both = torch.zeros(2, self.num_experts, dtype = acc, device = device)
+ if self.total is not None:
+ both[0] = self.total.to(device, acc)
+ both[1] = self.count.to(device, acc)
+ dist.all_reduce(both)
+ self.total, self.count = both[0], both[1]
+
+ def mean(self):
+ if self.total is None:
+ return torch.zeros(self.num_experts, dtype = torch.float64)
+ return self.total.cpu().double() / self.count.cpu().double().clamp(min = 1.0)
+
+
+def _retained(saliency, n_drop, n_drop_per_group, n_group):
+ """Experts kept after dropping the lowest saliency ones (per group for group-limited routers)."""
+ num_experts = saliency.numel()
+ if n_drop_per_group is None:
+ drop = set(torch.topk(saliency, n_drop, largest = False).indices.tolist())
+ return [i for i in range(num_experts) if i not in drop]
+ group_size = num_experts // n_group
+ kept = []
+ for g in range(n_group):
+ lo = g * group_size
+ drop = {
+ lo + i
+ for i in torch.topk(
+ saliency[lo : lo + group_size], n_drop_per_group, largest = False
+ ).indices.tolist()
+ }
+ kept.extend(i for i in range(lo, lo + group_size) if i not in drop)
+ return kept
+
+
+def _prune_block(block, experts, router, retained, num_experts):
+ index = torch.tensor(retained, dtype = torch.long)
+ # Experts on dim 0; anything per expert inside the router (HunYuan `gate.wg`, AfMoE `router.gate`,
+ # Ernie's [1, E] `moe_statics` bias); the block's own expert biases.
+ targets = [(experts, None)] + [(m, None) for m in router.modules()]
+ targets.append((block, ("e_score_correction_bias", "expert_bias")))
+ for module, only in targets:
+ tensors = list(module.named_parameters(recurse = False))
+ tensors += list(module.named_buffers(recurse = False))
+ # Inkling's router weight carries the shared experts' rows after the routed ones.
+ shared = getattr(module, "n_shared_experts", 0) if module is not experts else 0
+ shared = shared if isinstance(shared, int) else 0
+ for name, tensor in tensors:
+ if only is not None and name not in only or not tensor.dim():
+ continue
+ if shared and tensor.shape[0] == num_experts + shared:
+ kept = torch.cat([index, torch.arange(num_experts, num_experts + shared)])
+ new = tensor.data.index_select(0, kept.to(tensor.device)).contiguous()
+ setattr(module, name, nn.Parameter(new, requires_grad = tensor.requires_grad))
+ continue
+ if tensor.shape[0] == num_experts:
+ dim = 0
+ elif module is not experts and tensor.shape[-1] == num_experts:
+ dim = -1
+ else:
+ continue
+ new = tensor.data.index_select(dim, index.to(tensor.device)).contiguous()
+ if isinstance(tensor, nn.Parameter):
+ setattr(module, name, nn.Parameter(new, requires_grad = tensor.requires_grad))
+ else:
+ module._buffers[name] = new
+ for key in _NUM_EXPERTS_MODULE_KEYS:
+ value = getattr(module, key, None)
+ if isinstance(value, int) and value == num_experts:
+ setattr(module, key, len(retained))
+ if shared and getattr(module, "n_total_experts", None) == num_experts + shared:
+ module.n_total_experts = len(retained) + shared
+ if module is not experts and getattr(module, "out_features", None) == num_experts:
+ module.out_features = len(retained)
+ # Kernel configs Unsloth's MoE backend cached for the old expert count.
+ if getattr(experts, "_unsloth_moe_configs", None) is not None:
+ experts._unsloth_moe_configs = None
+
+
+class _ExpertsTap:
+ """Wraps an experts module's forward to record per (token, slot) expert output norms."""
+
+ def __init__(
+ self,
+ experts,
+ saliency,
+ post = None,
+ ):
+ self.experts = experts
+ self.saliency = saliency
+ # Nemotron-H latent MoE maps the summed expert output back to the hidden size after the
+ # experts, so ||f_j|| is measured after that shared Linear (without its bias, added once).
+ self.post = post
+ self.valid = None
+ self.stop = False
+ self.original = experts.__dict__.get("forward")
+ self.forward = experts.forward
+ experts.forward = self
+
+ def __call__(self, hidden_states, top_k_index, top_k_weights, *args, **kwargs):
+ shape = hidden_states.shape
+ hidden_states = hidden_states.reshape(-1, shape[-1])
+ top_k_index = top_k_index.reshape(hidden_states.shape[0], -1)
+ top_k_weights = top_k_weights.reshape(hidden_states.shape[0], -1)
+ n_tokens, top_k = top_k_index.shape
+ valid = self.valid if self.valid is not None and self.valid.numel() == n_tokens else None
+ step = max(
+ 1, _TAP_CHUNK_BYTES // (top_k * hidden_states.shape[-1] * hidden_states.element_size())
+ )
+ outs = []
+ for start in range(0, n_tokens, step):
+ x = hidden_states[start : start + step]
+ index = top_k_index[start : start + step]
+ gate = top_k_weights[start : start + step]
+ n = x.shape[0]
+ # One row per routed (token, slot) pair with gate 1 gives each expert's raw output f_j(t).
+ rows = self.forward(
+ x.repeat_interleave(top_k, dim = 0),
+ index.reshape(-1, 1),
+ torch.ones((n * top_k, 1), dtype = gate.dtype, device = gate.device),
+ *args,
+ **kwargs,
+ )
+ scored = rows if self.post is None else nn.functional.linear(rows, self.post.weight)
+ norms = torch.linalg.vector_norm(
+ scored,
+ dim = -1,
+ dtype = torch.float64 if rows.dtype == torch.float64 else torch.float32,
+ )
+ mask = None
+ if valid is not None:
+ # Masking by multiplication avoids the host sync boolean indexing needs.
+ mask = (
+ valid[start : start + step]
+ .to(norms.device, non_blocking = True)
+ .repeat_interleave(top_k)
+ )
+ self.saliency.add(index.reshape(-1).long(), gate.reshape(-1), norms, mask)
+ if not self.stop:
+ out = rows.view(n, top_k, -1) * gate.unsqueeze(-1).to(rows.dtype)
+ outs.append(out.sum(1).to(rows.dtype))
+ if self.stop:
+ raise _StopForward
+ return (outs[0] if len(outs) == 1 else torch.cat(outs)).reshape(shape)
+
+ def remove(self):
+ if self.original is None:
+ del self.experts.forward
+ else:
+ self.experts.forward = self.original
+
+
+def _residual_split(layer, block):
+ """(norm, block) when the layer ends ``residual + block(norm(residual))`` so a pruned layer's
+ output can be rebuilt from the residual alone; verified bitwise per layer before use."""
+ norm = getattr(layer, "post_attention_layernorm", None)
+ if (
+ isinstance(norm, nn.Module)
+ and not isinstance(norm, nn.ModuleList)
+ and getattr(layer, "mlp", None) is block
+ ):
+ return norm
+ return None
+
+
+def _resolve_dataset(dataset, tokenizer):
+ """The calibration rows to read: a DatasetDict's train (or only) split. Raises when text rows
+ come without a tokenizer. Data dependent, so it runs where every rank agrees on failures."""
+ if isinstance(dataset, dict):
+ # load_dataset(...) without split= gives a DatasetDict, which iterates its split names.
+ if "train" in dataset or len(dataset) == 1:
+ dataset = dataset["train"] if "train" in dataset else next(iter(dataset.values()))
+ else:
+ raise ValueError(
+ f"Unsloth: REAP got a dataset with the splits {sorted(dataset)}; pass one of them, "
+ "e.g. `dataset['validation']`."
+ )
+ # A dataset name (or the default) is text; other rows are checked as they are read, since a
+ # streaming dataset built from a generator declares no columns.
+ if (dataset is None or isinstance(dataset, str)) and tokenizer is None:
+ raise ValueError("Unsloth: REAP needs a tokenizer unless the dataset has `input_ids`.")
+ return dataset
+
+
+def _model_parallel_layout(model):
+ """Name of a model-parallel layout (expert / tensor / pipeline / FSDP / sequence), else None."""
+ distributed = getattr(getattr(model, "config", None), "distributed_config", None)
+
+ def field(name):
+ if isinstance(distributed, dict):
+ return distributed.get(name)
+ return getattr(distributed, name, None)
+
+ if field("enable_expert_parallel"):
+ return "expert parallel"
+ for name, label in (
+ ("tp_size", "tensor parallel"),
+ ("pp_size", "pipeline parallel"),
+ ("fsdp_size", "FSDP sharded"),
+ ):
+ size = field(name)
+ if isinstance(size, int) and size > 1:
+ return label
+ if field("enable_sequence_parallel"):
+ return "sequence parallel"
+ tp_size = getattr(model, "_tp_size", None)
+ if isinstance(tp_size, int) and tp_size > 1:
+ return "tensor parallel"
+ return None
+
+
+def reap_prune(
+ model,
+ tokenizer = None,
+ sparsity = 0.25,
+ dataset = None,
+ num_calibration_samples = 512,
+ max_seq_length = 2048,
+ max_batch_tokens = 16384,
+ report_path = None,
+ device = None,
+ offload_hidden_states = None,
+ save_directory = None,
+):
+ """Prune ``sparsity`` of the routed experts in every MoE layer with REAP, in place.
+
+ Mirrors llm-compressor's ``REAPPruningModifier`` (same saliency ``S_j = mean(g_j * ||f_j||)``,
+ selection and layer-by-layer calibration), on Unsloth's fused MoE kernels.
+
+ Args:
+ model: a 16-bit MoE model (e.g. from ``FastLanguageModel.from_pretrained(..., load_in_4bit = False)``),
+ before ``get_peft_model`` or after merging LoRA.
+ tokenizer: tokenizer or processor, used to format and tokenize calibration text.
+ sparsity: fraction of experts removed per MoE layer, in (0, 1).
+ dataset: Hugging Face dataset name, ``datasets`` dataset, or list of strings / token id lists.
+ Rows may carry ``input_ids``, ``text``, ``messages`` or ``conversations``.
+ Defaults to llm-compressor's REAP example dataset ``mlabonne/open-perfectblend``.
+ num_calibration_samples: number of calibration rows.
+ max_seq_length: truncation length per row.
+ max_batch_tokens: padded tokens per calibration batch.
+ report_path: optional ``.json`` file for ``{layer name: [kept expert ids]}``.
+ device: device layers run on; layers stored on the CPU are moved there one at a time.
+ offload_hidden_states: keep calibration activations on the CPU (default: only when they
+ would take more than a quarter of the free GPU memory).
+ save_directory: also save the pruned model (and tokenizer) here and load later merges from
+ it. Needed before ``get_peft_model`` when the LoRA model will be saved merged or to GGUF,
+ since those rebuild the base weights from the checkpoint on disk.
+
+ Returns:
+ The pruned model. ``model.config`` carries the new expert count, so ``save_pretrained``,
+ ``save_pretrained_merged`` and GGUF export write the smaller model.
+ """
+ if not 0.0 < float(sparsity) < 1.0:
+ raise ValueError(f"Unsloth: REAP sparsity must be in (0, 1), got {sparsity}.")
+ if hasattr(model, "peft_config"):
+ raise RuntimeError(
+ "Unsloth: run REAP on the base model before `get_peft_model`, or merge the LoRA "
+ "adapters first (`model = model.merge_and_unload()`)."
+ )
+ if getattr(model, "vllm_engine", None) is not None:
+ raise RuntimeError(
+ "Unsloth: REAP cannot update the attached vLLM engine, so `fast_generate` would keep the "
+ "unpruned experts. Load the model without `fast_inference = True` to prune it."
+ )
+ if model.__dict__.get("_unsloth_mtp_head") is not None:
+ raise RuntimeError(
+ "Unsloth: REAP does not prune the attached MTP head, so a save would write it with the "
+ "old expert count. Run `reap_prune` before `get_mtp_head`."
+ )
+ layout = _model_parallel_layout(model)
+ if layout is not None:
+ # Each rank would hold a slice of the experts (or, pipelined, other layers); plain DP is fine.
+ raise RuntimeError(
+ f"Unsloth: REAP does not support {layout} models. Load the model on one device (or with "
+ "`device_map`, or plain data parallel) to prune it."
+ )
+ configs = _configs(model)
+ num_experts = _first_attr(configs, _NUM_EXPERTS_KEYS)
+ top_k = _first_attr(configs, _TOP_K_KEYS, per_layer = True)
+ if num_experts is None or top_k is None:
+ raise ValueError(
+ "Unsloth: REAP needs an MoE model with an expert count and top_k in its config."
+ )
+ n_group = _first_attr(configs, ("n_group", "num_expert_group"))
+ topk_group = _first_attr(configs, ("topk_group", "top_k_group"))
+ if n_group is not None and n_group > 1 and topk_group is None:
+ # The router still splits experts into groups; pruning them globally would shift the
+ # group boundaries. A lone topk_group (or n_group 1) routes ungrouped.
+ raise ValueError(
+ f"Unsloth: REAP found n_group = {n_group} but no topk_group in the config, so it cannot "
+ "prune per router group."
+ )
+ if n_group is None or n_group <= 1 or topk_group is None:
+ n_group = topk_group = None
+
+ # Calibration runs each decoder stack alone on right padded text, which needs causal attention.
+ if getattr(model, "_unsloth_slow_diffusion", False) or any(
+ getattr(c, "use_bidirectional_attention", None) == "all"
+ or str(getattr(c, "model_type", "")).startswith("diffusion")
+ for c in configs
+ ):
+ raise ValueError(
+ "Unsloth: REAP supports causal language models, not bidirectional or diffusion ones."
+ )
+ for config in configs:
+ if getattr(config, "zero_expert_num", None):
+ raise ValueError(
+ "Unsloth: REAP does not support MoE routers with zero (identity) experts."
+ )
+ owner, layers = _find_layers(model, num_experts)
+ if layers is None:
+ raise RuntimeError(
+ "Unsloth: REAP found no MoE layers with fused experts. It needs transformers 5.x MoE "
+ "modules (an `experts` module with 3D expert weights next to a `gate` / `router`)."
+ )
+ if any(p.startswith("prev_") for p in inspect.signature(layers[0].forward).parameters):
+ raise ValueError(_STATEFUL_LAYERS)
+ # Non-causal text attention (openai/privacy-filter) carries no config flag.
+ if any(getattr(m, "is_causal", None) is False for layer in layers for m in layer.modules()):
+ raise ValueError(
+ "Unsloth: REAP supports causal language models, not bidirectional or diffusion ones."
+ )
+
+ layer_names = {id(m): n for n, m in model.named_modules()}
+ for layer in layers:
+ moes = _layer_moes(layer, num_experts)
+ if len(moes) > 1:
+ raise ValueError(
+ "Unsloth: REAP supports one MoE block per decoder layer; "
+ f"{layer_names[id(layer)]} has {len(moes)} (e.g. separate text and vision experts)."
+ )
+ moe = moes[0] if moes else None
+ if moe is not None:
+ _check_weights(moe[2], moe[3], layer_names[id(moe[1])])
+ if moe is not None and any(hasattr(m, "tid2eid") for m in moe[3].modules()):
+ raise ValueError(
+ "Unsloth: REAP does not support hash-routed MoE layers (fixed token to expert "
+ "tables, e.g. DeepSeek-V4), since dropped experts cannot be remapped."
+ )
+
+ n_drop = int(num_experts * float(sparsity))
+ if n_drop == 0:
+ raise ValueError(
+ f"Unsloth: REAP sparsity={sparsity} drops 0 of {num_experts} experts; raise the sparsity."
+ )
+ n_drop_per_group = None
+ if n_group is not None:
+ if num_experts % n_group:
+ raise ValueError(
+ f"Unsloth: {num_experts} experts are not divisible by n_group={n_group}."
+ )
+ n_drop_per_group = round(n_drop / n_group)
+ if n_drop_per_group == 0:
+ raise ValueError(
+ f"Unsloth: REAP sparsity={sparsity} drops 0 experts per router group "
+ f"({num_experts // n_group} experts in each of {n_group} groups); raise the sparsity."
+ )
+ if n_drop_per_group * n_group != n_drop:
+ print(
+ f"Unsloth: REAP group-limited routing drops {n_drop_per_group} experts per group, "
+ f"{n_drop_per_group * n_group} in total instead of {n_drop}."
+ )
+ n_drop = n_drop_per_group * n_group
+ if num_experts // n_group - n_drop_per_group < 2:
+ # DeepSeek style routers score each group by the sum of its top 2 experts.
+ raise ValueError(
+ f"Unsloth: REAP sparsity={sparsity} leaves fewer than 2 experts in each of the "
+ f"{n_group} router groups; lower the sparsity."
+ )
+ available = topk_group * (num_experts // n_group - n_drop_per_group)
+ else:
+ available = num_experts - n_drop
+ if top_k > available:
+ raise ValueError(
+ f"Unsloth: REAP sparsity={sparsity} is too aggressive: the router picks top_k={top_k} "
+ f"experts per token but only {available} would stay reachable. Lower the sparsity."
+ )
+ kept_count = num_experts - n_drop
+
+ if device is None:
+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ device = torch.device(device)
+
+ pad_id = 0
+ text_tokenizer = _text_tokenizer(tokenizer)
+ if text_tokenizer is not None:
+ pad_id = text_tokenizer.pad_token_id
+ if pad_id is None:
+ pad_id = text_tokenizer.eos_token_id or 0
+ error = None
+ try:
+ dataset = _resolve_dataset(dataset, tokenizer)
+ ids = _calibration_ids(dataset, tokenizer, num_calibration_samples, max_seq_length)
+ batches = _batches(ids, max(max_batch_tokens, max(len(x) for x in ids)), pad_id)
+ except Exception as e:
+ error = e
+ # One rank's empty or malformed shard must stop every rank before the per-layer collectives.
+ _ranks_agree(error, "prepare its calibration data")
+ n_tokens = sum(len(x) for x in ids)
+
+ report, saliencies = {}, {}
+ pruning = None
+ modes = [(m, m.training) for m in model.modules()]
+ model.eval()
+ start = time.time()
+ try:
+ hidden, kwargs_per_batch, valid_per_batch = [], [], []
+ embed_device = _layer_device(owner, device)
+ store = None
+ capture_error = None
+ try:
+ for input_ids, valid in batches:
+ h, kwargs = _capture_inputs(owner, layers, input_ids.to(embed_device))
+ if not hidden:
+ # Decided from the first batch, so activations never pile up on the GPU first.
+ if offload_hidden_states is None and device.type == "cuda":
+ padded = sum(x.numel() for x, _ in batches) * h.shape[-1] * h.element_size()
+ # Eager attention keeps a [batch, 1, seq, seq] mask per batch in the kwargs.
+ padded += _tensor_bytes(kwargs) * len(batches)
+ offload_hidden_states = padded > torch.cuda.mem_get_info(device)[0] // 4
+ store = torch.device("cpu") if offload_hidden_states else None
+ if store is not None:
+ h = h.to(store).pin_memory() if torch.cuda.is_available() else h.to(store)
+ # Rotary tables and any materialized masks go too, or they pile up per batch.
+ kwargs = _to(kwargs, store, {})
+ hidden.append(h)
+ kwargs_per_batch.append(kwargs)
+ valid_per_batch.append(valid.reshape(-1))
+ except Exception as e:
+ capture_error = e
+ _ranks_agree(capture_error, "capture its calibration activations")
+ width = hidden[0].shape[-1]
+
+ print(
+ f"Unsloth: REAP pruning {n_drop} of {num_experts} experts per MoE layer "
+ f"({len(ids)} samples, {n_tokens} tokens, {len(batches)} batches, hidden {width})."
+ )
+ rebuilt_layers = 0
+ for index, layer in enumerate(layers):
+ move_error = None
+ try:
+ layer_device, moved = _stream_in(layer, device)
+ except Exception as e:
+ layer_device, moved, move_error = device, True, e
+ post_error = move_error
+
+ try:
+
+ def run(b, h):
+ args, kwargs = kwargs_per_batch[b][index]
+ # Gemma 4's shared_kv_states (a UserDict shared across layers) follows offloaded activations.
+ shared = [
+ v
+ for v in kwargs.values()
+ if isinstance(v, MutableMapping) and not isinstance(v, dict)
+ ]
+ # Moved every call: under a device_map the previous layer may sit on another GPU.
+ for m in shared:
+ m.update(_to(dict(m), layer_device))
+ try:
+ return _hidden(
+ layer(h, *_to(args, layer_device), **_to(kwargs, layer_device))
+ )
+ finally:
+ if store is not None:
+ for m in shared:
+ m.update(_to(dict(m), store))
+
+ moe = _layer_moe(layer, num_experts)
+ norm = None
+ if moe is not None:
+ _, block, experts, router = moe
+ block_name = layer_names[id(block)]
+ stage_error = None
+ try:
+ if move_error is not None:
+ raise move_error
+ norm = _residual_split(layer, block)
+ residuals = []
+ if norm is not None:
+ hook = norm.register_forward_pre_hook(
+ lambda module, args: residuals.append(args[0])
+ )
+ try:
+ h0 = hidden[0].to(layer_device, non_blocking = True)
+ reference = run(0, h0)
+ try:
+ same = len(residuals) == 1 and torch.equal(
+ reference,
+ residuals[0]
+ + _hidden(block(norm(residuals[0]))).reshape(
+ residuals[0].shape
+ ),
+ )
+ except (
+ Exception
+ ): # block needs more than hidden states (DeepSeek-V4)
+ same = False
+ if not same:
+ norm = None
+ finally:
+ residuals.clear()
+ hook.remove()
+
+ saliency = _Saliency(num_experts)
+ post = getattr(block, "fc2_latent_proj", None)
+ tap = _ExpertsTap(
+ experts, saliency, post if isinstance(post, nn.Linear) else None
+ )
+ # Pass 2 recomputes the layer output, so pass 1 stops right after the experts
+ # (not under accelerate hooks, whose post-forward offload must run).
+ tap.stop = not any(hasattr(m, "_hf_hook") for m in layer.modules())
+ hook = None
+ if norm is not None:
+ hook = norm.register_forward_pre_hook(
+ lambda module, args: residuals.append(args[0])
+ )
+ try:
+ for b in range(len(batches)):
+ tap.valid = valid_per_batch[b]
+ try:
+ run(b, hidden[b].to(layer_device, non_blocking = True))
+ except _StopForward:
+ pass
+ if norm is not None:
+ # The residual is all a verified layer needs later, so it replaces the input.
+ r = residuals.pop()
+ hidden[b] = (
+ r if store is None else r.to(store, non_blocking = True)
+ )
+ finally:
+ tap.remove()
+ if hook is not None:
+ hook.remove()
+ except Exception as e:
+ stage_error = e
+ _ranks_agree(stage_error, f"score {block_name}")
+
+ saliency.all_reduce()
+ if saliency.total is None or not saliency.count.sum().item():
+ raise RuntimeError(
+ f"Unsloth: REAP saw no tokens reach {block_name}'s experts module; its MoE "
+ "forward computes the experts itself, so REAP cannot score them."
+ )
+ scores = saliency.mean()
+ saliencies[block_name] = scores.tolist()
+ retained = _retained(scores, n_drop, n_drop_per_group, n_group)
+ rebuilt_layers += norm is not None
+ assert len(retained) == kept_count
+ # Tensors change from here on: no longer the checkpoint any earlier save wrote.
+ model._unsloth_reap_saved = None
+ pruning = block_name
+ try:
+ _prune_block(block, experts, router, retained, num_experts)
+ report[block_name] = retained
+ pruning = None
+ except Exception as e:
+ post_error = e
+
+ try:
+ for b in range(len(batches) if post_error is None else 0):
+ h = hidden[b].to(layer_device, non_blocking = True)
+ if norm is not None:
+ out = h + _hidden(block(norm(h))).reshape(h.shape)
+ else:
+ out = run(b, h)
+ hidden[b] = out if store is None else out.to(store, non_blocking = True)
+ except Exception as e:
+ post_error = e
+ _ranks_agree(post_error, f"run decoder layer {index + 1}")
+ finally:
+ if moved:
+ layer.to("cpu")
+ print(
+ f"Unsloth: REAP layer {index + 1}/{len(layers)} done ({time.time() - start:.1f}s)."
+ )
+ except BaseException:
+ if report or pruning is not None:
+ # Partly pruned: mark it REAP modified and unsaved so the PEFT / save guards refuse it.
+ partial = dict(report)
+ if pruning is not None:
+ partial.setdefault(pruning, [])
+ model._unsloth_reap_report = partial
+ model._unsloth_reap_saved = None
+ print(
+ f"Unsloth: REAP stopped after pruning {len(report)} MoE layers, so the model now mixes "
+ "pruned and unpruned layers. Reload it before running REAP again."
+ )
+ raise
+ finally:
+ for module, training in modes:
+ module.training = training
+
+ if not report:
+ raise RuntimeError("Unsloth: REAP found no MoE layers to prune.")
+ for config in configs:
+ for key in _NUM_EXPERTS_KEYS:
+ if getattr(config, key, None) == num_experts:
+ setattr(config, key, kept_count)
+ # MTP layers live outside `layers` and are not pruned; they would be rebuilt at the new count.
+ for key in (
+ "num_nextn_predict_layers",
+ "num_mtp_layers",
+ "mtp_num_layers",
+ "mtp_num_hidden_layers",
+ ):
+ value = getattr(config, key, None)
+ if isinstance(value, int) and value > 0:
+ setattr(config, key, 0)
+ print(
+ f"Unsloth: REAP set {key} = 0, since the MTP layers keep the old expert count."
+ )
+ # Wrappers cache the count too (Mixtral / Jamba pass model.num_experts to the router aux loss).
+ for module in model.modules():
+ for key in _NUM_EXPERTS_MODULE_KEYS:
+ value = getattr(module, key, None)
+ if isinstance(value, int) and value == num_experts:
+ setattr(module, key, kept_count)
+ model._unsloth_reap_report = report
+ model._unsloth_reap_saliency = saliencies
+ dist = torch.distributed
+ multi_rank = dist.is_available() and dist.is_initialized() and dist.get_world_size() > 1
+ if save_directory is not None:
+ save_directory = os.path.abspath(save_directory)
+ source_model = getattr(model.config, "_name_or_path", None) or None
+ if source_model and os.path.exists(source_model):
+ # This folder may be pushed to the Hub: name a local source without its path.
+ source_model = os.path.basename(os.path.normpath(source_model))
+ error = None
+ if not multi_rank or dist.get_rank() == 0:
+ try:
+ model.save_pretrained(save_directory)
+ if tokenizer is not None:
+ tokenizer.save_pretrained(save_directory)
+ # Survives a reload, unlike the in-process marker: the adapter push guard reads it.
+ with open(
+ os.path.join(save_directory, REAP_PROVENANCE_FILE), "w", encoding = "utf-8"
+ ) as f:
+ json.dump({"source_model": source_model, "kept_experts": report}, f, indent = 1)
+ except Exception as e:
+ error = e
+ _ranks_agree(error, f"save the pruned model to {save_directory}")
+ # Merged / GGUF saves read config._name_or_path; PEFT's adapter config reads model.name_or_path.
+ for config in configs:
+ config._name_or_path = save_directory
+ for module in model.modules():
+ if "name_or_path" in module.__dict__:
+ module.name_or_path = save_directory
+ if getattr(model, "_unsloth_modelscope_snapshot", None):
+ model._unsloth_modelscope_snapshot = save_directory
+ model._unsloth_reap_saved = save_directory
+ if report_path is not None:
+ error = None
+ if not multi_rank or dist.get_rank() == 0:
+ try:
+ os.makedirs(os.path.dirname(os.path.abspath(report_path)), exist_ok = True)
+ with open(report_path, "w", encoding = "utf-8") as f:
+ json.dump(report, f)
+ except Exception as e:
+ error = e
+ _ranks_agree(error, f"write the REAP report to {report_path}")
+ print(
+ f"Unsloth: REAP kept {kept_count} of {num_experts} experts in {len(report)} MoE layers "
+ f"in {time.time() - start:.1f}s ({rebuilt_layers} reused their attention output)."
+ )
+ if save_directory is None:
+ print(
+ "Unsloth: to LoRA finetune the pruned model, pass `save_directory` (or save the pruned "
+ "model and load it again) before `get_peft_model`."
+ )
+ return model
+
+
+def raise_if_unsaved_reap(model):
+ """LoRA adapters and merged saves record the base checkpoint on disk, which still holds every
+ expert after an in-memory prune."""
+ if getattr(model, "_unsloth_reap_report", None) and not getattr(
+ model, "_unsloth_reap_saved", None
+ ):
+ raise RuntimeError(
+ "Unsloth: this model was REAP pruned in memory only, but LoRA adapters and merged saves "
+ f"point at the base checkpoint `{model.config._name_or_path}`, which still has every "
+ 'expert. Call `reap_prune(..., save_directory = "pruned_model")` (or save the pruned '
+ "model and load it again) before `get_peft_model`."
+ )
+
+
+def _is_token_rows(dataset):
+ if isinstance(dataset, (list, tuple)) and dataset:
+ first = dataset[0]
+ return isinstance(first, (list, tuple)) or (
+ isinstance(first, dict) and "input_ids" in first
+ )
+ column_names = getattr(dataset, "column_names", None)
+ return bool(column_names) and "input_ids" in column_names
diff --git a/unsloth/save.py b/unsloth/save.py
index 9f6f23a503e..5a39918ee5d 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -3621,6 +3621,11 @@ def _gguf_reuses_loaded_checkpoint(model, state_dict = None):
return False
if state_dict is not None or getattr(model, "_unsloth_full_finetuning", False):
return False
+ # REAP pruned in memory: the loaded folder still holds every expert.
+ if getattr(model, "_unsloth_reap_report", None) and not getattr(
+ model, "_unsloth_reap_saved", None
+ ):
+ return False
# The loaded folder holds the stock MTP head; a trained one needs a fresh checkpoint.
if _attached_mtp_head(model) is not None:
return False
@@ -6125,6 +6130,48 @@ def unsloth_generic_save(
# this function's own keywords.
safe_serialization = _normalize_safe_serialization(safe_serialization)
+ # Inline, builtins only (no new locals: they are forwarded below), and before the Hub revision
+ # route, which creates the repo / branch first. After an in-memory REAP prune the base
+ # checkpoint on disk still has every expert, and both a merge (which rebuilds from it) and an
+ # adapter save (which records it as the base) would use it. get_peft_model refuses this too;
+ # raw PEFT does not.
+ if (
+ isinstance(model, PeftModel)
+ and getattr(model, "_unsloth_reap_report", None)
+ and not getattr(model, "_unsloth_reap_saved", None)
+ ):
+ raise RuntimeError(
+ "Unsloth: this model was REAP pruned in memory only, but this save would use the base "
+ f"checkpoint `{model.config._name_or_path}`, which still has every expert. Nothing was "
+ 'written. Call `reap_prune(..., save_directory = "pruned_model")` (or save the pruned '
+ "model and load it again) before `get_peft_model`."
+ )
+ # The pruned base REAP saved is a local folder: an adapter pushed against it names a path no one
+ # else can load. In this process the marker says so (even if the folder moved since); after a
+ # reload from that folder, the provenance file reap_prune wrote there does.
+ if (
+ push_to_hub
+ and isinstance(model, PeftModel)
+ and _is_adapter_save_method(save_method)
+ and (
+ getattr(model, "_unsloth_reap_saved", None)
+ or any(
+ os.path.isfile(
+ os.path.join(
+ str(getattr(c, "base_model_name_or_path", None) or ""), "unsloth_reap.json"
+ )
+ )
+ for c in (getattr(model, "peft_config", None) or {}).values()
+ )
+ )
+ ):
+ raise RuntimeError(
+ "Unsloth: this adapter's base is a REAP pruned model saved to a local folder, so a "
+ "pushed adapter could not be loaded anywhere else. Nothing was uploaded. Push the pruned base "
+ "to the Hub first and load it from there before finetuning, or push a merged model "
+ '(`save_method = "merged_16bit"`).'
+ )
+
# A trained MTP head can only be stitched into a local copy, which this route makes first.
if push_to_hub and (
create_pr
@@ -6488,6 +6535,19 @@ def unsloth_generic_save_pretrained_merged(
asks for a pickle.
"""
_assert_export_target_is_not_base_with_lora_layers(self)
+ # Inline, builtins only, like unsloth_generic_save's REAP guard, but before the compressed /
+ # torchao dispatch below, whose helpers install their runtime and create the Hub repo first.
+ if (
+ isinstance(self, PeftModel)
+ and getattr(self, "_unsloth_reap_report", None)
+ and not getattr(self, "_unsloth_reap_saved", None)
+ ):
+ raise RuntimeError(
+ "Unsloth: this model was REAP pruned in memory only, but this save would use the base "
+ f"checkpoint `{self.config._name_or_path}`, which still has every expert. Nothing was "
+ 'written. Call `reap_prune(..., save_directory = "pruned_model")` (or save the pruned '
+ "model and load it again) before `get_peft_model`."
+ )
if tokenizer is None:
logger.warning_once(
"Unsloth: You're not saving a tokenizer as well?\n"
@@ -6635,6 +6695,19 @@ def unsloth_generic_push_to_hub_merged(
asks for a pickle.
"""
_assert_export_target_is_not_base_with_lora_layers(self)
+ # Inline, builtins only, like unsloth_generic_save's REAP guard, but before the compressed /
+ # torchao dispatch below, whose helpers install their runtime and create the Hub repo first.
+ if (
+ isinstance(self, PeftModel)
+ and getattr(self, "_unsloth_reap_report", None)
+ and not getattr(self, "_unsloth_reap_saved", None)
+ ):
+ raise RuntimeError(
+ "Unsloth: this model was REAP pruned in memory only, but this save would use the base "
+ f"checkpoint `{self.config._name_or_path}`, which still has every expert. Nothing was "
+ 'written. Call `reap_prune(..., save_directory = "pruned_model")` (or save the pruned '
+ "model and load it again) before `get_peft_model`."
+ )
if tokenizer is None:
logger.warning_once(
"Unsloth: You're not saving a tokenizer as well?\n"