Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add REAP expert pruning for MoE models (#13834)
Daniel Han093390e4997f
diff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.pyindex c61157b567f..9081ab2db53 100644--- a/tests/test_merged_hub_destination.py+++ b/tests/test_merged_hub_destination.py@@ -346,6 +346,19 @@ def test_unforced_4bit_rejection_does_not_create_hub_resources(saving, options):assert records["repos"] == records["branches"] == records["merges"] == records["uploads"] == [][email protected]("options", [{"revision": "candidate"}, {"create_pr": True}, {}])+def test_in_memory_reap_rejection_does_not_create_hub_resources(saving, options):+ # A raw-PEFT wrap of a model REAP pruned without save_directory: the base on disk is unpruned.+ env, records, _ = saving+ model = PeftModel()+ model._unsloth_reap_report = {"model.layers.0.mlp": [0, 2]}+ with pytest.raises(RuntimeError, match = "pruned in memory only"):+ env["unsloth_generic_push_to_hub_merged"](+ model, "owner/new-model", save_method = "merged_16bit", **options+ )+ assert records["repos"] == records["branches"] == records["merges"] == records["uploads"] == []++def test_forced_4bit_mode_still_reaches_staged_save(saving):env, records, _ = savingenv["unsloth_generic_push_to_hub_merged"](@@ -716,3 +729,69 @@ def test_merged_dtype_refuses_an_older_zoo(saving):)# Refused before the base model is downloaded into the Hub cache.assert records["merges"] == [] and prewarmed == []+++def test_adapter_push_against_a_local_reap_base_uploads_nothing(saving, tmp_path):+ # reap_prune(..., save_directory) then LoRA: the adapter's base is a folder only this machine has.+ env, records, _ = saving+ model = PeftModel()+ model._unsloth_reap_report = {"model.layers.0.mlp": [0, 2]}+ model._unsloth_reap_saved = str(tmp_path)+ with pytest.raises(RuntimeError, match = "local folder"):+ env["unsloth_generic_push_to_hub_merged"](model, "owner/adapter", save_method = "lora")+ assert records["repos"] == records["branches"] == records["merges"] == records["uploads"] == []+++def test_adapter_push_against_a_moved_local_reap_base_uploads_nothing(saving, tmp_path):+ # The pruned base folder was moved or deleted after LoRA setup: the adapter still names it.+ env, records, _ = saving+ model = PeftModel()+ model._unsloth_reap_report = {"model.layers.0.mlp": [0, 2]}+ model._unsloth_reap_saved = str(tmp_path / "moved_away")+ with pytest.raises(RuntimeError, match = "local folder"):+ env["unsloth_generic_push_to_hub_merged"](model, "owner/adapter", save_method = "lora")+ assert records["repos"] == records["branches"] == records["merges"] == records["uploads"] == []+++def test_adapter_push_after_reloading_a_local_reap_checkpoint_uploads_nothing(saving, tmp_path):+ # reap_prune(..., save_directory) then a fresh process loads that folder: no in-process marker,+ # but the provenance file reap_prune wrote there marks the adapter's base as a local REAP model.+ env, records, _ = saving+ (tmp_path / "unsloth_reap.json").write_text('{"kept_experts": {}}')+ model = PeftModel()+ model.peft_config = {"default": SimpleNamespace(base_model_name_or_path = str(tmp_path))}+ with pytest.raises(RuntimeError, match = "REAP pruned model saved to a local folder"):+ env["unsloth_generic_push_to_hub_merged"](model, "owner/adapter", save_method = "lora")+ assert (+ records["repos"]+ == records["branches"]+ == records["uploads"]+ == records["adapter_saves"]+ == []+ )+++def test_adapter_push_on_an_ordinary_local_base_is_not_refused(saving, tmp_path):+ env, records, _ = saving+ model = PeftModel()+ model.peft_config = {"default": SimpleNamespace(base_model_name_or_path = str(tmp_path))}+ env["unsloth_generic_push_to_hub_merged"](model, "owner/adapter", save_method = "lora")+ assert len(records["adapter_saves"]) == 1++[email protected]("kind", ["compressed", "torchao"])+def test_in_memory_reap_is_refused_before_quantized_export_dispatch(saving, kind):+ # Those export helpers install their runtime and create the Hub repo before any nested save runs.+ env, records, _ = saving+ called = []+ if kind == "compressed":+ env["_normalize_compressed_method"] = lambda method: ("FP8_DYNAMIC", False, "FP8")+ env["_unsloth_save_compressed_tensors"] = lambda **kwargs: called.append(kwargs)+ else:+ env["_normalize_torchao_method"] = lambda method: ("int8", "INT8")+ env["_unsloth_save_torchao"] = lambda **kwargs: called.append(kwargs)+ model = PeftModel()+ model._unsloth_reap_report = {"model.layers.0.mlp": [0, 2]}+ with pytest.raises(RuntimeError, match = "pruned in memory only"):+ env["unsloth_generic_push_to_hub_merged"](model, "owner/new-model", save_method = "fp8")+ assert called == [] and records["repos"] == []diff --git a/tests/test_reap_prune.py b/tests/test_reap_prune.pynew file mode 100644index 00000000000..b994dc56dda--- /dev/null+++ b/tests/test_reap_prune.py@@ -0,0 +1,1053 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""REAP expert pruning (unsloth/models/reap.py) against a brute-force oracle, on CPU tiny MoEs."""++import copy+import importlib.util+import json+import sys+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+transformers = pytest.importorskip("transformers")+if int(transformers.__version__.split(".")[0]) < 5:+ pytest.skip(+ reason = "REAP prunes transformers 5 fused MoE experts (3D expert weights); 4.x keeps a ModuleList",+ allow_module_level = True,+ )++_MODULE = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "reap.py"+_spec = importlib.util.spec_from_file_location("_unsloth_reap", _MODULE)+reap = importlib.util.module_from_spec(_spec)+_spec.loader.exec_module(reap)++[email protected](autouse = True)+def _one_thread():+ # Tiny CPU ops thrash on a loaded many-core host; one thread is faster and deterministic.+ threads = torch.get_num_threads()+ torch.set_num_threads(1)+ yield+ torch.set_num_threads(threads)++[email protected](autouse = True)+def _float64_moe_backend(monkeypatch):+ # These models are float64; Unsloth's grouped_mm MoE backend (picked when a GPU is visible) has no+ # float64 kernel, so pin the looped backend. The backend choice is lru_cached per module copy.+ monkeypatch.setenv("UNSLOTH_MOE_BACKEND", "native_torch")+ caches = [+ getattr(vars(module).get("select_moe_backend"), "cache_clear", None)+ for module in list(sys.modules.values())+ if module is not None and hasattr(module, "__dict__")+ ]+ caches = [clear for clear in caches if callable(clear)]+ for clear in caches:+ clear()+ yield+ for clear in caches:+ clear()+++def _qwen3_moe(+ num_experts = 16,+ top_k = 4,+ seed = 0,+):+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM++ config = Qwen3MoeConfig(+ vocab_size = 96,+ hidden_size = 32,+ intermediate_size = 48,+ moe_intermediate_size = 16,+ num_hidden_layers = 3,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 8,+ num_experts = num_experts,+ num_experts_per_tok = top_k,+ norm_topk_prob = True,+ mlp_only_layers = [0], # a dense layer in front, like GLM / DeepSeek+ max_position_embeddings = 64,+ )+ torch.manual_seed(seed)+ model = Qwen3MoeForCausalLM._from_config(+ config, dtype = torch.float64, experts_implementation = "eager"+ )+ return model.eval()+++def _deepseek_v3(+ n_group = 4,+ topk_group = 2,+ seed = 0,+):+ from transformers import DeepseekV3Config, DeepseekV3ForCausalLM++ config = DeepseekV3Config(+ vocab_size = 96,+ hidden_size = 32,+ intermediate_size = 48,+ moe_intermediate_size = 16,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 4,+ n_routed_experts = 16,+ n_shared_experts = 1,+ num_experts_per_tok = 4,+ n_group = n_group,+ topk_group = topk_group,+ first_k_dense_replace = 0,+ q_lora_rank = 16,+ kv_lora_rank = 16,+ qk_rope_head_dim = 8,+ qk_nope_head_dim = 8,+ v_head_dim = 8,+ max_position_embeddings = 64,+ )+ torch.manual_seed(seed)+ model = DeepseekV3ForCausalLM._from_config(+ config, dtype = torch.float64, experts_implementation = "eager"+ )+ with torch.no_grad():+ for module in model.modules():+ bias = getattr(module, "e_score_correction_bias", None)+ if bias is not None:+ bias.copy_(torch.randn_like(bias) * 0.1)+ return model.eval()+++def _samples(n = 7, seed = 1):+ g = torch.Generator().manual_seed(seed)+ return [+ torch.randint(1, 96, (int(torch.randint(3, 19, (1,), generator = g)),), generator = g).tolist()+ for _ in range(n)+ ]+++def _oracle(model, samples, n_drop):+ """Sequential REAP written from the paper: per sample (no padding), per (token, slot) eager+ expert matmuls in float64, each layer pruned before the next one is scored."""+ model = copy.deepcopy(model)+ kept, scores = {}, {}+ for index, layer in enumerate(model.model.layers):+ block = layer.mlp+ if not hasattr(block, "experts"):+ continue+ experts = block.experts+ num_experts = experts.gate_up_proj.shape[0]+ total = torch.zeros(num_experts, dtype = torch.float64)+ count = torch.zeros(num_experts, dtype = torch.float64)+ seen = {}+ hook = experts.register_forward_hook(+ lambda m, args, out: seen.update(x = args[0], idx = args[1], w = args[2])+ )+ with torch.no_grad():+ for sample in samples:+ model.model(input_ids = torch.tensor([sample]), use_cache = False)+ for t in range(seen["x"].shape[0]):+ for s in range(seen["idx"].shape[1]):+ e = int(seen["idx"][t, s])+ gate, up = torch.nn.functional.linear(+ seen["x"][t], experts.gate_up_proj[e]+ ).chunk(2)+ f = torch.nn.functional.linear(+ experts.act_fn(gate) * up, experts.down_proj[e]+ )+ total[e] += seen["w"][t, s].double() * f.double().norm()+ count[e] += 1+ hook.remove()+ saliency = total / count.clamp(min = 1)+ drop = set(torch.topk(saliency, n_drop, largest = False).indices.tolist())+ keep = [i for i in range(num_experts) if i not in drop]+ name = f"model.layers.{index}.mlp"+ kept[name], scores[name] = keep, saliency+ idx = torch.tensor(keep)+ experts.gate_up_proj = torch.nn.Parameter(experts.gate_up_proj.data[idx])+ experts.down_proj = torch.nn.Parameter(experts.down_proj.data[idx])+ experts.num_experts = len(keep)+ block.gate.weight = torch.nn.Parameter(block.gate.weight.data[idx])+ block.gate.num_experts = len(keep)+ return kept, scores+++def test_matches_sequential_oracle_with_padded_batches():+ model = _qwen3_moe()+ samples = _samples()+ kept, scores = _oracle(model, samples, n_drop = 4)+ grad_modes = []+ hook = model.model.layers[1].register_forward_hook(+ lambda *_: grad_modes.append(torch.is_grad_enabled())+ )+ # A small token budget forces several right-padded batches.+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, max_batch_tokens = 40, device = "cpu")+ hook.remove()+ # No autograd graph may keep calibration activations alive, and weights stay trainable.+ assert grad_modes and not any(grad_modes)+ assert model.model.layers[1].mlp.experts.gate_up_proj.requires_grad+ assert model._unsloth_reap_report == kept+ for name, oracle in scores.items():+ torch.testing.assert_close(+ torch.tensor(model._unsloth_reap_saliency[name], dtype = torch.float64),+ oracle,+ rtol = 1e-9,+ atol = 1e-12,+ )+ assert model.config.num_experts == 12+ for layer in model.model.layers[1:]:+ assert layer.mlp.experts.gate_up_proj.shape[0] == 12+ assert layer.mlp.gate.weight.shape[0] == 12+ assert layer.mlp.experts.num_experts == 12+++def test_batching_does_not_change_the_result():+ samples = _samples(n = 9, seed = 3)+ a, b = _qwen3_moe(seed = 5), _qwen3_moe(seed = 5)+ reap.reap_prune(a, None, sparsity = 0.5, dataset = samples, max_batch_tokens = 1, device = "cpu")+ reap.reap_prune(b, None, sparsity = 0.5, dataset = samples, max_batch_tokens = 10_000, device = "cpu")+ assert a._unsloth_reap_report == b._unsloth_reap_report+ for name in a._unsloth_reap_saliency:+ torch.testing.assert_close(+ torch.tensor(a._unsloth_reap_saliency[name]),+ torch.tensor(b._unsloth_reap_saliency[name]),+ rtol = 1e-9,+ atol = 1e-12,+ )+++def test_pruned_model_saves_reloads_and_writes_report(tmp_path):+ from transformers import AutoModelForCausalLM++ model = _qwen3_moe()+ report = tmp_path / "sub" / "reap.json"+ reap.reap_prune(+ model, None, sparsity = 0.25, dataset = _samples(), device = "cpu", report_path = str(report)+ )+ assert json.loads(report.read_text(encoding = "utf-8")) == model._unsloth_reap_report+ model.save_pretrained(tmp_path / "pruned")+ reloaded = AutoModelForCausalLM.from_pretrained(+ tmp_path / "pruned", dtype = torch.float64, experts_implementation = "eager"+ ).eval()+ assert reloaded.config.num_experts == 12+ ids = torch.tensor([_samples(n = 1, seed = 9)[0]])+ with torch.no_grad():+ torch.testing.assert_close(reloaded(ids).logits, model(ids).logits)+++def test_group_limited_router_drops_per_group():+ model = _deepseek_v3()+ biases = [layer.mlp.gate.e_score_correction_bias.clone() for layer in model.model.layers]+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")+ for index, (layer, bias) in enumerate(zip(model.model.layers, biases)):+ kept = model._unsloth_reap_report[f"model.layers.{index}.mlp"]+ # 16 experts in 4 groups of 4, one dropped per group.+ assert [sum(1 for e in kept if e // 4 == g) for g in range(4)] == [3, 3, 3, 3]+ torch.testing.assert_close(layer.mlp.gate.e_score_correction_bias, bias[kept])+ assert layer.mlp.experts.gate_up_proj.shape[0] == 12+ assert model.config.n_routed_experts == 12+ with torch.no_grad():+ model(torch.tensor([_samples(n = 1)[0]]))+++def test_rejects_bad_requests():+ model = _qwen3_moe()+ samples = _samples()+ with pytest.raises(ValueError, match = "drops 0"):+ reap.reap_prune(model, None, sparsity = 0.01, dataset = samples, device = "cpu")+ with pytest.raises(ValueError, match = "too aggressive"):+ reap.reap_prune(model, None, sparsity = 0.9, dataset = samples, device = "cpu")+ with pytest.raises(ValueError, match = r"in \(0, 1\)"):+ reap.reap_prune(model, None, sparsity = 1.0, dataset = samples, device = "cpu")+ model.vllm_engine = object()+ with pytest.raises(RuntimeError, match = "vLLM engine"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")+ del model.vllm_engine+ object.__setattr__(model, "_unsloth_mtp_head", torch.nn.Linear(1, 1))+ with pytest.raises(RuntimeError, match = "MTP head"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")+ del model.__dict__["_unsloth_mtp_head"]+ for layout, match in (+ ({"enable_expert_parallel": True}, "expert parallel"),+ ({"tp_size": 2}, "tensor parallel"),+ ({"pp_size": 2}, "pipeline parallel"),+ ({"fsdp_size": 2}, "FSDP"),+ ({"enable_sequence_parallel": True}, "sequence parallel"),+ ):+ model.config.distributed_config = layout+ with pytest.raises(RuntimeError, match = match):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")+ # Size 1 is not model parallel.+ model.config.distributed_config = {"tp_size": 1, "pp_size": 1}+ assert reap._model_parallel_layout(model) is None+ del model.config.distributed_config+ model.config.use_bidirectional_attention = "all"+ with pytest.raises(ValueError, match = "causal"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")+ model.config.use_bidirectional_attention = None+ model.model.layers[1].self_attn.is_causal = False+ with pytest.raises(ValueError, match = "causal"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")+ model.model.layers[1].self_attn.is_causal = True+ layer0 = model.model.layers[0]+ original = layer0.forward+ layer0.forward = lambda hidden_states, prev_topk_indices = None, **kwargs: original(+ hidden_states, **kwargs+ ) # GLM-5 / HY-V4 hand top-k indices from layer to layer+ with pytest.raises(ValueError, match = "extra state to the next layer"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")+ del layer0.forward+ model.peft_config = {}+ with pytest.raises(RuntimeError, match = "get_peft_model"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")+ # Nothing was touched by the refused calls.+ del model.peft_config+ assert model.model.layers[1].mlp.experts.gate_up_proj.shape[0] == 16+++def test_full_layer_rerun_matches_residual_shortcut(monkeypatch):+ samples = _samples(n = 5, seed = 4)+ a, b = _qwen3_moe(seed = 2), _qwen3_moe(seed = 2)+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, device = "cpu")+ monkeypatch.setattr(reap, "_residual_split", lambda layer, block: None)+ reap.reap_prune(b, None, sparsity = 0.25, dataset = samples, device = "cpu")+ assert a._unsloth_reap_report == b._unsloth_reap_report+ ids = torch.tensor([samples[0]])+ with torch.no_grad():+ torch.testing.assert_close(a(ids).logits, b(ids).logits)+++def test_save_directory_becomes_the_merge_base(tmp_path):+ model = _qwen3_moe()+ reap.reap_prune(+ model,+ None,+ sparsity = 0.25,+ dataset = _samples(),+ device = "cpu",+ save_directory = str(tmp_path / "pruned"),+ )+ # Later merged / GGUF saves of a LoRA on top read the base checkpoint from config._name_or_path.+ assert model.config._name_or_path == str((tmp_path / "pruned").resolve())+ assert model._unsloth_reap_saved == model.config._name_or_path == model.name_or_path+ saved = json.loads((tmp_path / "pruned" / "config.json").read_text(encoding = "utf-8"))+ assert saved.get("num_experts", saved.get("num_local_experts")) == 12+++ "build",+ [+ # Router weight in a child module (`gate.wg`); top_k per layer, as in Hunyuan-A13B.+ lambda T, base: T.HunYuanMoEV1Config(+ **{**base, "num_experts_per_tok": [2] * base["num_hidden_layers"]}, num_experts = 8+ ),+ # [1, E] score bias in `gate.moe_statics`.+ lambda T, base: T.Ernie4_5_MoeConfig(+ **base, moe_num_experts = 8, moe_layer_start_index = 0, moe_k = 2+ ),+ ],+ ids = ["hunyuan_v1_moe", "ernie4_5_moe"],+)+def test_per_expert_router_tensors_in_child_modules_are_sliced(build, tmp_path):+ from transformers import AutoModelForCausalLM++ base = dict(+ vocab_size = 96,+ hidden_size = 32,+ intermediate_size = 48,+ moe_intermediate_size = 16,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 4,+ head_dim = 8,+ max_position_embeddings = 64,+ num_experts_per_tok = 2,+ )+ torch.manual_seed(0)+ model = AutoModelForCausalLM.from_config(+ build(transformers, base), dtype = torch.float64, experts_implementation = "eager"+ ).eval()+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")+ for name, tensor in list(model.named_parameters()) + list(model.named_buffers()):+ if ".mlp." in name and "shared" not in name:+ assert 8 not in tuple(tensor.shape), (name, tuple(tensor.shape))+ model.save_pretrained(tmp_path)+ reloaded = AutoModelForCausalLM.from_pretrained(+ tmp_path, dtype = torch.float64, experts_implementation = "eager"+ ).eval()+ ids = torch.tensor([_samples(n = 1, seed = 9)[0]])+ with torch.no_grad():+ torch.testing.assert_close(reloaded(ids).logits, model(ids).logits)+++def test_unsupported_weights_refuse_before_any_layer_is_pruned():+ model = _qwen3_moe()+ experts = model.model.layers[2].mlp.experts+ experts.down_proj = torch.nn.Parameter(+ torch.zeros(experts.down_proj.shape, dtype = torch.int8), requires_grad = False+ )+ with pytest.raises(RuntimeError, match = "16-bit or 32-bit"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")+ assert model.model.layers[1].mlp.experts.gate_up_proj.shape[0] == 16+++def test_offloaded_activations_give_the_same_result():+ samples = _samples(n = 6, seed = 7)+ a, b = _qwen3_moe(seed = 3), _qwen3_moe(seed = 3)+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, max_batch_tokens = 30, device = "cpu")+ reap.reap_prune(+ b,+ None,+ sparsity = 0.25,+ dataset = samples,+ max_batch_tokens = 30,+ device = "cpu",+ offload_hidden_states = True,+ )+ assert a._unsloth_reap_report == b._unsloth_reap_report+ assert a._unsloth_reap_saliency == b._unsloth_reap_saliency+++def test_pretokenized_padding_is_dropped_via_attention_mask():+ samples = _samples(n = 5, seed = 11)+ width = max(len(x) for x in samples)+ padded = [+ {+ "input_ids": [0] * (width - len(x)) + x,+ "attention_mask": [0] * (width - len(x)) + [1] * len(x),+ }+ for x in samples+ ]+ a, b = _qwen3_moe(seed = 4), _qwen3_moe(seed = 4)+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, device = "cpu")+ reap.reap_prune(b, None, sparsity = 0.25, dataset = padded, device = "cpu")+ assert a._unsloth_reap_saliency == b._unsloth_reap_saliency+++def test_latent_moe_scores_expert_outputs_in_hidden_space():+ # Nemotron-H with moe_latent_size: experts run in a latent space and fc2_latent_proj maps+ # their summed output back, so ||f_j|| is taken after that projection.+ NemotronHConfig = getattr(transformers, "NemotronHConfig", None)+ if NemotronHConfig is None:+ pytest.skip("this transformers has no Nemotron-H")+ from transformers import NemotronHForCausalLM++ config = NemotronHConfig(+ vocab_size = 96,+ hidden_size = 32,+ layers_block_type = ["moe", "attention", "moe"],+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 8,+ intermediate_size = 48,+ n_routed_experts = 8,+ n_shared_experts = 1,+ moe_intermediate_size = 16,+ moe_shared_expert_intermediate_size = 16,+ moe_latent_size = 12,+ num_experts_per_tok = 2,+ n_group = 1,+ topk_group = 1,+ mamba_num_heads = 4,+ mamba_head_dim = 8,+ n_groups = 1,+ )+ torch.manual_seed(0)+ model = NemotronHForCausalLM._from_config(+ config, dtype = torch.float64, experts_implementation = "eager"+ ).eval()+ samples = _samples()+ block = model.model.layers[0].mixer+ # The experts call's own inputs (latent states, routing): the router's return tuple varies by version.+ calls = []+ hook = block.experts.register_forward_pre_hook(lambda module, args: calls.append(args[:3]))+ with torch.no_grad():+ for x in samples:+ model(torch.tensor([x]))+ hook.remove()+ total = torch.zeros(8, dtype = torch.float64)+ count = torch.zeros(8, dtype = torch.float64)+ with torch.no_grad():+ for latent, index, weights in calls:+ for t in range(latent.shape[0]):+ for s in range(index.shape[1]):+ e = int(index[t, s])+ f = block.experts(+ latent[t : t + 1], index[t : t + 1, s : s + 1], torch.ones(1, 1)+ )+ total[e] += weights[t, s] * (f @ block.fc2_latent_proj.weight.T).norm()+ count[e] += 1+ expected = total / count.clamp(min = 1)++ reap.reap_prune(model, None, sparsity = 0.5, dataset = samples, device = "cpu")+ torch.testing.assert_close(+ torch.tensor(model._unsloth_reap_saliency["model.layers.0.mixer"], dtype = torch.float64),+ expected,+ rtol = 1e-9,+ atol = 1e-12,+ )+ assert model.model.layers[0].mixer.experts.up_proj.shape[0] == 4+++def test_refuses_layers_with_several_moe_blocks_and_keeps_training_modes():+ model = _qwen3_moe()+ samples = _samples()+ layer = model.model.layers[1]+ layer.vision_mlp = copy.deepcopy(layer.mlp) # e.g. Ernie 4.5 VL's text_moe + vision_moe+ with pytest.raises(ValueError, match = "one MoE block per decoder layer"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")+ assert layer.mlp.experts.gate_up_proj.shape[0] == 16+ del layer.vision_mlp++ model.train()+ model.model.layers[0].eval() # e.g. a frozen tower kept in eval mode+ reap.reap_prune(model, None, sparsity = 0.25, dataset = samples, device = "cpu")+ assert model.training and model.model.layers[1].training+ assert not model.model.layers[0].training+++def test_chunked_scoring_matches_one_pass(monkeypatch):+ samples = _samples(n = 6, seed = 5)+ a, b = _qwen3_moe(seed = 6), _qwen3_moe(seed = 6)+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, device = "cpu")+ monkeypatch.setattr(reap, "_TAP_CHUNK_BYTES", 1) # one token per experts call+ reap.reap_prune(b, None, sparsity = 0.25, dataset = samples, device = "cpu")+ assert a._unsloth_reap_report == b._unsloth_reap_report+ for name in a._unsloth_reap_saliency:+ torch.testing.assert_close(+ torch.tensor(a._unsloth_reap_saliency[name], dtype = torch.float64),+ torch.tensor(b._unsloth_reap_saliency[name], dtype = torch.float64),+ rtol = 1e-9,+ atol = 1e-12,+ )+++def _saliency_rank(rank, init_file, shards, out):+ import torch.distributed as dist+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = 2)+ try:+ saliency = reap._Saliency(4)+ ids, weights, norms = shards[rank]+ if ids is not None:+ saliency.add(ids, weights, norms)+ saliency.all_reduce()+ out.put((rank, saliency.total.tolist(), saliency.count.tolist()))+ finally:+ dist.destroy_process_group()+++def test_saliency_is_summed_across_ranks(tmp_path):+ # Two gloo ranks with different calibration shards (rank 1 routed nothing) must both end with+ # the one-process sums, so every rank of a DDP job prunes the same experts.+ import multiprocessing++ ids = torch.tensor([0, 1, 1, 3])+ weights = torch.tensor([0.5, 1.0, 0.25, 2.0])+ norms = torch.tensor([2.0, 1.0, 4.0, 0.5])+ one = reap._Saliency(4)+ one.add(ids, weights, norms)+ shards = [(ids, weights, norms), (None, None, None)]+ ctx = multiprocessing.get_context("spawn")+ out = ctx.Queue()+ procs = [+ ctx.Process(target = _saliency_rank, args = (r, str(tmp_path / "pg"), shards, out))+ for r in range(2)+ ]+ for p in procs:+ p.start()+ results = sorted(out.get(timeout = 120) for _ in procs)+ for p in procs:+ p.join(timeout = 60)+ assert p.exitcode == 0+ for _, total, count in results:+ assert total == one.total.tolist() and count == one.count.tolist()+ # Single process (no process group): unchanged.+ before = one.total.clone()+ one.all_reduce()+ assert torch.equal(one.total, before)+++def test_saliency_reduces_on_the_accelerator_for_xccl_and_hccl(monkeypatch):+ # Accelerator-only backends cannot reduce a CPU tensor, and their devices have no float64.+ dist = torch.distributed+ seen = []+ monkeypatch.setattr(dist, "is_available", lambda: True)+ monkeypatch.setattr(dist, "is_initialized", lambda: True)+ monkeypatch.setattr(dist, "get_world_size", lambda *a, **k: 2)+ monkeypatch.setattr(dist, "get_backend", lambda *a, **k: "xccl")+ monkeypatch.setattr(+ dist, "all_reduce", lambda t, *a, **k: seen.append((t.device.type, t.dtype))+ )+ monkeypatch.setattr(+ torch.accelerator, "current_accelerator", lambda *a, **k: torch.device("meta")+ )+ monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 0, raising = False)+ saliency = reap._Saliency(4)+ saliency.all_reduce()+ assert seen == [("meta", torch.float32)]+++def test_dataset_dict_uses_a_split_not_its_names():+ # load_dataset(...) without split= returns a DatasetDict; its iteration yields split names.+ datasets = pytest.importorskip("datasets")+ samples = _samples(n = 6, seed = 9)+ split = datasets.Dataset.from_list([{"input_ids": ids} for ids in samples])+ model = _qwen3_moe()+ a = copy.deepcopy(model)+ b = copy.deepcopy(model)+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, device = "cpu")+ reap.reap_prune(b, None, sparsity = 0.25, dataset = datasets.DatasetDict(train = split), device = "cpu")+ assert a._unsloth_reap_report == b._unsloth_reap_report+ two = datasets.DatasetDict(validation = split, test = split)+ with pytest.raises(ValueError, match = "splits"):+ reap.reap_prune(copy.deepcopy(model), None, sparsity = 0.25, dataset = two, device = "cpu")+++def _prune_rank(+ rank,+ init_file,+ save_dir,+ out,+ fail_save = False,+ bad_shard = False,+ report = False,+):+ import json+ import os+ import torch.distributed as dist++ os.environ["UNSLOTH_MOE_BACKEND"] = "native_torch"+ torch.set_num_threads(1)+ dist.init_process_group("gloo", init_method = f"file://{init_file}", rank = rank, world_size = 2)+ try:+ model = _qwen3_moe(seed = 0)+ original = model.save_pretrained++ def save(*args, **kwargs):+ if fail_save:+ raise OSError("No space left on device")+ with open(os.path.join(os.path.dirname(save_dir), f"saved_by_{rank}"), "w") as f:+ f.write("1")+ return original(*args, **kwargs)++ model.save_pretrained = save+ report_path = None+ if report:+ report_path = os.path.join(os.path.dirname(save_dir), "report.json")+ original_dump = json.dump++ def dump(obj, f, *args, **kwargs):+ if getattr(f, "name", "") == report_path:+ if report == "fail":+ raise OSError("Read-only file system")+ with open(+ os.path.join(os.path.dirname(save_dir), f"report_by_{rank}"), "w"+ ) as m:+ m.write("1")+ return original_dump(obj, f, *args, **kwargs)++ json.dump = dump+ if bad_shard == "move" and rank == 1:++ def fail_move(*args, **kwargs):+ raise RuntimeError("CUDA out of memory while streaming the layer in")++ reap._stream_in = fail_move+ if bad_shard == "slice" and rank == 1:++ def fail_slice(*args, **kwargs):+ raise RuntimeError("out of memory while slicing experts")++ reap._prune_block = fail_slice+ if bad_shard == "post" and rank == 1:+ # Rank 1 fails in the post-prune pass (an OOM on a larger shard, say).+ original_prune = reap._prune_block++ def prune_then_fail(block, *args, **kwargs):+ original_prune(block, *args, **kwargs)++ def boom(*_):+ raise RuntimeError("out of memory in the post-prune pass")++ block.register_forward_hook(boom)++ reap._prune_block = prune_then_fail+ # Each rank calibrates on its own shard, as a DDP job would.+ try:+ reap.reap_prune(+ model,+ None,+ sparsity = 0.25,+ dataset = (+ ([[10**6] * 6] if bad_shard == "out_of_range" else [])+ if bad_shard in (True, "out_of_range") and rank == 1+ else _samples(n = 5, seed = 20 + rank)+ ),+ device = "cpu",+ save_directory = save_dir,+ report_path = report_path,+ )+ except Exception as e:+ out.put((rank, "error", str(e)))+ return+ out.put((rank, model._unsloth_reap_report, model.config._name_or_path))+ finally:+ dist.destroy_process_group()+++def test_ddp_ranks_keep_the_same_experts_and_one_writes(tmp_path):+ import multiprocessing++ save_dir = str(tmp_path / "pruned")+ ctx = multiprocessing.get_context("spawn")+ out = ctx.Queue()+ procs = [+ ctx.Process(target = _prune_rank, args = (r, str(tmp_path / "pg"), save_dir, out))+ for r in range(2)+ ]+ for p in procs:+ p.start()+ results = sorted(out.get(timeout = 300) for _ in procs)+ for p in procs:+ p.join(timeout = 60)+ assert p.exitcode == 0+ (_, report0, path0), (_, report1, path1) = results+ assert report0 == report1 and path0 == path1 == save_dir+ assert (tmp_path / "saved_by_0").exists() and not (tmp_path / "saved_by_1").exists()+ # The one shared-data answer: both shards together, in one process.+ one = _qwen3_moe(seed = 0)+ reap.reap_prune(+ one,+ None,+ sparsity = 0.25,+ dataset = _samples(n = 5, seed = 20) + _samples(n = 5, seed = 21),+ device = "cpu",+ )+ assert one._unsloth_reap_report == report0+++def test_a_failed_rank_zero_save_stops_every_rank(tmp_path):+ # Rank 0's write fails (disk full): every rank must raise, none may wait forever.+ import multiprocessing++ ctx = multiprocessing.get_context("spawn")+ out = ctx.Queue()+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, True)+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]+ for p in procs:+ p.start()+ results = sorted(out.get(timeout = 300) for _ in procs)+ for p in procs:+ p.join(timeout = 60)+ assert p.exitcode == 0+ assert results[0][1] == "error" and "No space left" in results[0][2]+ assert results[1][1] == "error" and "another rank failed to save" in results[1][2]+++def _run_ranks(tmp_path, **kwargs):+ import multiprocessing++ ctx = multiprocessing.get_context("spawn")+ out = ctx.Queue()+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out)+ procs = [ctx.Process(target = _prune_rank, args = (r, *args), kwargs = kwargs) for r in range(2)]+ for p in procs:+ p.start()+ results = sorted(out.get(timeout = 300) for _ in procs)+ for p in procs:+ p.join(timeout = 60)+ assert p.exitcode == 0+ return results+++def test_ddp_report_is_written_by_rank_zero_only(tmp_path):+ (_, report0, _), (_, report1, _) = _run_ranks(tmp_path, report = True)+ assert report0 == report1+ assert (tmp_path / "report_by_0").exists() and not (tmp_path / "report_by_1").exists()+ assert json.loads((tmp_path / "report.json").read_text(encoding = "utf-8")) == report0+++def test_a_failed_rank_zero_report_write_stops_every_rank(tmp_path):+ results = _run_ranks(tmp_path, report = "fail")+ assert results[0][1] == "error" and "Read-only file system" in results[0][2]+ assert (+ results[1][1] == "error" and "another rank failed to write the REAP report" in results[1][2]+ )+++def test_one_bad_calibration_shard_stops_every_rank(tmp_path):+ # Rank 1's shard is empty: it must not leave rank 0 waiting in the first per-layer collective.+ import multiprocessing++ ctx = multiprocessing.get_context("spawn")+ out = ctx.Queue()+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, False, True)+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]+ for p in procs:+ p.start()+ results = sorted(out.get(timeout = 300) for _ in procs)+ for p in procs:+ p.join(timeout = 60)+ assert p.exitcode == 0+ assert results[0][1] == "error" and "another rank failed to prepare" in results[0][2]+ assert results[1][1] == "error"+++def test_group_metadata_must_be_complete():+ model = _qwen3_moe()+ model.config.n_group = 2+ with pytest.raises(ValueError, match = "no topk_group"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")+ # n_group 1 (or a lone topk_group) routes ungrouped and prunes globally.+ model.config.n_group = 1+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")+ assert model._unsloth_reap_report+++def test_one_rank_failing_at_capture_stops_every_rank(tmp_path):+ # Rank 1's shard has an out-of-range token id: its embedding lookup fails while capturing.+ import multiprocessing++ ctx = multiprocessing.get_context("spawn")+ out = ctx.Queue()+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, False, "out_of_range")+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]+ for p in procs:+ p.start()+ results = sorted(out.get(timeout = 300) for _ in procs)+ for p in procs:+ p.join(timeout = 60)+ assert p.exitcode == 0+ assert results[0][1] == "error" and "another rank failed to capture" in results[0][2]+ assert results[1][1] == "error"+++def test_metadata_free_streaming_token_rows_need_no_tokenizer():+ datasets = pytest.importorskip("datasets")+ samples = _samples(n = 6, seed = 13)++ def rows():+ for ids in samples:+ yield {"input_ids": ids}++ stream = datasets.IterableDataset.from_generator(rows)+ model = _qwen3_moe()+ a, b = copy.deepcopy(model), copy.deepcopy(model)+ reap.reap_prune(a, None, sparsity = 0.25, dataset = samples, device = "cpu")+ reap.reap_prune(b, None, sparsity = 0.25, dataset = stream, device = "cpu")+ assert a._unsloth_reap_report == b._unsloth_reap_report+ # Text rows still need one, now reported when the first text row is read.+ with pytest.raises(ValueError, match = "needs a tokenizer"):+ reap.reap_prune(+ copy.deepcopy(model), None, sparsity = 0.25, dataset = [{"text": "hi"}], device = "cpu"+ )+++def test_failure_after_pruning_a_layer_marks_the_model_unsaved(monkeypatch):+ model = _qwen3_moe()+ model._unsloth_reap_saved = "/stale/earlier/save"+ original = reap._prune_block+ calls = []++ def flaky(*args, **kwargs):+ calls.append(1)+ if len(calls) == 2:+ raise RuntimeError("out of memory")+ return original(*args, **kwargs)++ monkeypatch.setattr(reap, "_prune_block", flaky)+ with pytest.raises(RuntimeError, match = "out of memory"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")+ # The completed layer keeps its experts; the one cut short is recorded too, both unsaved.+ done, cut = list(model._unsloth_reap_report.values())+ assert done and cut == [] and model._unsloth_reap_saved is None+++def test_one_rank_failing_after_pruning_stops_every_rank(tmp_path):+ import multiprocessing++ ctx = multiprocessing.get_context("spawn")+ out = ctx.Queue()+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, False, "post")+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]+ for p in procs:+ p.start()+ results = sorted(out.get(timeout = 300) for _ in procs)+ for p in procs:+ p.join(timeout = 60)+ assert p.exitcode == 0+ assert results[0][1] == "error" and "another rank failed to run decoder layer" in results[0][2]+ assert results[1][1] == "error" and "post-prune" in results[1][2]+++def test_failure_inside_the_first_slice_still_marks_the_model(monkeypatch):+ model = _qwen3_moe()+ model._unsloth_reap_saved = "/stale/earlier/save"++ def half_sliced(block, experts, router, retained, num_experts):+ experts.gate_up_proj.data = experts.gate_up_proj.data[retained]+ raise RuntimeError("failed after slicing one tensor")++ monkeypatch.setattr(reap, "_prune_block", half_sliced)+ with pytest.raises(RuntimeError, match = "one tensor"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")+ assert model._unsloth_reap_report and model._unsloth_reap_saved is None+++def test_one_rank_failing_to_slice_stops_every_rank(tmp_path):+ import multiprocessing++ ctx = multiprocessing.get_context("spawn")+ out = ctx.Queue()+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, False, "slice")+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]+ for p in procs:+ p.start()+ results = sorted(out.get(timeout = 300) for _ in procs)+ for p in procs:+ p.join(timeout = 60)+ assert p.exitcode == 0+ assert results[0][1] == "error" and "another rank failed to run decoder layer" in results[0][2]+ assert results[1][1] == "error" and "slicing experts" in results[1][2]+++def test_several_moe_decoder_stacks_are_refused():+ class TwoStacks(torch.nn.Module):+ def __init__(self, a, b):+ super().__init__()+ self.config = a.config+ self.text = a.model+ self.other = b.model++ a, b = _qwen3_moe(seed = 0), _qwen3_moe(seed = 1)+ model = TwoStacks(a, b)+ experts = next(m for n, m in model.other.named_modules() if n.endswith("mlp.experts"))+ before = experts.gate_up_proj.shape+ with pytest.raises(ValueError, match = "several decoder stacks"):+ reap.reap_prune(model, None, sparsity = 0.25, dataset = _samples(), device = "cpu")+ # Refused before anything changed.+ assert experts.gate_up_proj.shape == before+ assert model.config.num_experts == before[0]+++def test_one_rank_failing_to_stream_a_layer_in_stops_every_rank(tmp_path):+ import multiprocessing++ ctx = multiprocessing.get_context("spawn")+ out = ctx.Queue()+ args = (str(tmp_path / "pg"), str(tmp_path / "pruned"), out, False, "move")+ procs = [ctx.Process(target = _prune_rank, args = (r, *args)) for r in range(2)]+ for p in procs:+ p.start()+ results = sorted(out.get(timeout = 300) for _ in procs)+ for p in procs:+ p.join(timeout = 60)+ assert p.exitcode == 0+ assert (+ results[0][1] == "error" and "another rank failed to run decoder layer 1" in results[0][2]+ )+ assert results[1][1] == "error" and "streaming the layer in" in results[1][2]+++def test_shared_layer_state_follows_each_layer_without_offloading(monkeypatch):+ # Gemma 4 passes one mutable `shared_kv_states` mapping to every layer. Under a device_map the+ # layers sit on different GPUs, so it must be moved to each layer's device even when the+ # activations stay put (no offload store).+ from collections import UserDict++ model = _qwen3_moe(seed = 0)+ shared = UserDict({"kv": torch.zeros(1)})+ capture = reap._capture_inputs++ def capture_with_shared(*args, **kwargs):+ h, per_layer = capture(*args, **kwargs)+ return h, [(a, {**kw, "shared_kv_states": shared}) for a, kw in per_layer]++ moved = []+ to = reap._to++ def recording_to(obj, device, *rest):+ if isinstance(obj, dict) and obj.keys() == {"kv"}:+ moved.append(torch.device(device))+ return to(obj, device, *rest)++ monkeypatch.setattr(reap, "_capture_inputs", capture_with_shared)+ monkeypatch.setattr(reap, "_to", recording_to)+ reap.reap_prune(+ model,+ None,+ sparsity = 0.5,+ dataset = _samples(),+ device = "cpu",+ offload_hidden_states = False,+ )+ assert moved and set(moved) == {torch.device("cpu")}+++def test_mistral_common_backend_is_not_unwrapped():+ # MistralCommonBackend is an HF tokenizer whose `.tokenizer` is the raw mistral_common one,+ # which has neither __call__ nor apply_chat_template. Only processors are unwrapped.+ from transformers import ProcessorMixin++ class Raw:+ pass++ class Backend:+ tokenizer = Raw()+ chat_template = "{{ messages[0]['content'] }}"+ pad_token_id = 0++ def __call__(+ self,+ text,+ truncation = True,+ max_length = None,+ ):+ return {"input_ids": [5, 6, 7][:max_length]}++ def apply_chat_template(+ self,+ messages,+ tokenize = False,+ **kwargs,+ ):+ return messages[0]["content"]++ backend = Backend()+ assert reap._text_tokenizer(backend) is backend+ ids = reap._calibration_ids([{"messages": [{"role": "user", "content": "hi"}]}], backend, 4, 8)+ assert ids == [[5, 6, 7]]+ processor = ProcessorMixin.__new__(ProcessorMixin)+ processor.tokenizer = backend+ assert reap._text_tokenizer(processor) is backend+++def test_saved_pruned_checkpoint_carries_its_provenance(tmp_path):+ import json++ model = _qwen3_moe(seed = 0)+ source = model.config._name_or_path+ reap.reap_prune(+ model, None, sparsity = 0.5, dataset = _samples(), device = "cpu", save_directory = str(tmp_path)+ )+ data = json.loads((tmp_path / reap.REAP_PROVENANCE_FILE).read_text(encoding = "utf-8"))+ assert data["source_model"] == (source or None)+ assert data["kept_experts"] == {k: list(v) for k, v in model._unsloth_reap_report.items()}+++def test_provenance_names_a_local_source_without_its_path(tmp_path):+ import json++ model = _qwen3_moe(seed = 0)+ local = tmp_path / "home" / "me" / "my-moe"+ local.mkdir(parents = True)+ model.config._name_or_path = str(local)+ out = tmp_path / "pruned"+ reap.reap_prune(+ model, None, sparsity = 0.5, dataset = _samples(), device = "cpu", save_directory = str(out)+ )+ data = json.loads((out / reap.REAP_PROVENANCE_FILE).read_text(encoding = "utf-8"))+ assert data["source_model"] == "my-moe"diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 04bf15cd070..6389535a8df 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -28,6 +28,7 @@ from .mistral import FastMistralModelfrom .qwen2 import FastQwen2Modelfrom .qwen3 import FastQwen3Modelfrom .qwen3_moe import FastQwen3MoeModel+from .reap import reap_prune as _reap_prune, raise_if_unsaved_reapfrom .cohere import FastCohereModelfrom transformers import AutoConfigfrom transformers import __version__ as transformers_version@@ -1089,10 +1090,13 @@ def _drop_vllm_engine_kwargs(kwargs):class FastLanguageModel(FastLlamaModel):+ reap_prune = staticmethod(_reap_prune)+# Text-diffusion models load through FastModel's slow path; these route them there for the PEFT / mode helpers.@staticmethod@functools.wraps(FastLlamaModel.get_peft_model)def get_peft_model(model, *args, **kwargs):+ raise_if_unsaved_reap(model)if getattr(model, "_unsloth_slow_diffusion", False):return FastDiffusionModel.get_peft_model(model, **_diffusion_peft_kwargs(model, args, kwargs)@@ -2007,6 +2011,8 @@ except:class FastModel(FastBaseModel):+ reap_prune = staticmethod(_reap_prune)+@staticmethoddef get_mtp_head(model, *args, **kwargs):from .speculative import get_mtp_head@@ -2019,6 +2025,7 @@ class FastModel(FastBaseModel):@staticmethoddef get_peft_model(model, *args, **kwargs):+ raise_if_unsaved_reap(model)# Route text-diffusion models (slow path) to the transformers-only PEFT helper.if getattr(model, "_unsloth_slow_diffusion", False):return FastDiffusionModel.get_peft_model(diff --git a/unsloth/models/reap.py b/unsloth/models/reap.pynew file mode 100644index 00000000000..05ddf37dc13--- /dev/null+++ b/unsloth/models/reap.py@@ -0,0 +1,1123 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""REAP expert pruning (Router-weighted Expert Activation Pruning, arXiv:2510.13999).++Same saliency, expert selection and schedule as llm-compressor's ``REAPPruningModifier``: each+decoder layer is calibrated on the outputs of the already pruned layers before it, its lowest+saliency experts are dropped, and the pruned layer produces the next layer's inputs. Expert+outputs come from the model's own fused experts kernel (one call with every routed (token, slot)+pair as its own row) instead of per-expert Linear modules, so no extra dependency is needed.+"""++import inspect+import json+from collections.abc import MutableMapping+import os+import time++import torch+import torch.nn as nn++__all__ = ["reap_prune", "raise_if_unsaved_reap"]++_NUM_EXPERTS_KEYS = ("num_experts", "num_local_experts", "n_routed_experts", "moe_num_experts")+_NUM_EXPERTS_MODULE_KEYS = ("num_experts", "n_experts", "n_routed_experts", "num_local_experts")+_TOP_K_KEYS = ("num_experts_per_tok", "top_k", "moe_top_k", "top_k_experts")+_ROUTER_ATTRS = ("gate", "router")+_DEFAULT_DATASET = "mlabonne/open-perfectblend"+_TAP_CHUNK_BYTES = 256 << 20+_STATEFUL_LAYERS = (+ "Unsloth: REAP calibrates one decoder layer at a time, so it does not support decoder layers "+ "that hand extra state to the next layer (GLM-5 / HY-V4 sparse-attention top-k indices, ZAYA "+ "router states)."+)+# llm-compressor's fallback for tokenizers without a chat template (PerfectBlendDataset).+_DEFAULT_CHAT_TEMPLATE = (+ "{% for message in messages %}\n"+ "{% if message['role'] == 'user' %}\n"+ "{{ '<|user|>\n' + message['content'] + eos_token }}\n"+ "{% elif message['role'] == 'system' %}\n"+ "{{ '<|system|>\n' + message['content'] + eos_token }}\n"+ "{% elif message['role'] == 'assistant' %}\n"+ "{{ '<|assistant|>\n' + message['content'] + eos_token }}\n"+ "{% endif %}\n"+ "{% if loop.last and add_generation_prompt %}\n"+ "{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}"+)+++class _StopForward(Exception):+ pass+++def _configs(model):+ config = model.config+ text_config = getattr(config, "text_config", None)+ return [c for c in (config, text_config) if c is not None]+++def _is_int(value):+ return isinstance(value, int) and not isinstance(value, bool)+++def _first_attr(+ configs,+ keys,+ per_layer = False,+):+ for config in reversed(configs): # text_config first+ for key in keys:+ value = getattr(config, key, None)+ if _is_int(value):+ return value+ # HunYuan stores `moe_topk` per layer (`[8] * 32`); the largest bounds every layer.+ if (+ per_layer+ and isinstance(value, (list, tuple))+ and value+ and all(map(_is_int, value))+ ):+ return max(value)+ return None+++def _moe_parts(module, num_experts):+ """(experts, router) when ``module`` is a fused-experts MoE block, else None."""+ experts = getattr(module, "experts", None)+ if not isinstance(experts, nn.Module) or isinstance(experts, nn.ModuleList):+ return None+ router = next(+ (+ getattr(module, a)+ for a in _ROUTER_ATTRS+ if isinstance(getattr(module, a, None), nn.Module)+ ),+ None,+ )+ if router is None:+ return None+ try:+ params = list(inspect.signature(experts.forward).parameters.values())+ except (TypeError, ValueError):+ params = []+ # The tap needs forward(hidden_states, top_k_index, top_k_weights); Llama4 passes one tensor.+ positional = [p for p in params if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)]+ if params and len(positional) < 3 and not any(p.kind == p.VAR_POSITIONAL for p in params):+ return None+ if not any(+ p.dim() == 3 and p.shape[0] == num_experts for p in experts.parameters(recurse = False)+ ):+ return None+ return experts, router+++def _find_layers(model, num_experts):+ """(owner, layers): the ModuleList of decoder layers holding the MoE blocks and its parent."""+ best, stacks = None, set()+ for name, module in model.named_modules():+ layers = getattr(module, "layers", None)+ if not isinstance(layers, nn.ModuleList) or len(layers) == 0:+ continue+ n_moe = sum(+ any(_moe_parts(m, num_experts) is not None for m in layer.modules()) for layer in layers+ )+ if n_moe and (best is None or n_moe > best[0]):+ best = (n_moe, module, layers)+ if n_moe:+ stacks.add(id(layers))+ if best is None:+ return None, None+ if len(stacks) > 1:+ # Pruning one stack would still rewrite the expert counts the other stacks read.+ raise ValueError(+ "Unsloth: REAP found MoE layers in several decoder stacks (e.g. separate multimodal "+ "decoders); it prunes models with one MoE decoder stack."+ )+ return best[1], best[2]+++def _layer_moes(layer, num_experts):+ found, seen = [], set()+ for name, module in layer.named_modules():+ parts = _moe_parts(module, num_experts)+ if parts is not None and id(parts[0]) not in seen:+ seen.add(id(parts[0]))+ found.append((name, module, parts[0], parts[1]))+ return found+++def _layer_moe(layer, num_experts):+ found = _layer_moes(layer, num_experts)+ return found[0] if found else None+++def _check_weights(experts, router, layer_name):+ # The router recursively: HunYuan / AfMoE keep its Linear in a child module (bnb would quantize it).+ for owner, kind, recurse in ((experts, "experts", False), (router, "router", True)):+ for name, p in owner.named_parameters(recurse = recurse):+ if not p.is_floating_point() or p.element_size() < 2 or p.is_meta:+ raise RuntimeError(+ f"Unsloth: REAP needs 16-bit or 32-bit MoE weights, but {layer_name} {kind}.{name} "+ f"is {p.dtype}{' on the meta device' if p.is_meta else ''}. Load the model with "+ "`load_in_4bit = False` (and without fp8 / offloaded weights), then run REAP."+ )+++def _row_text(row, tokenizer, chat):+ if "text" in row and isinstance(row["text"], str):+ return row["text"]+ messages = row.get("messages", row.get("conversations"))+ if not isinstance(messages, list):+ raise ValueError(+ "Unsloth: each REAP calibration row needs `input_ids`, `text`, `messages` or "+ f"`conversations`; got columns {list(row)}."+ )+ messages = [+ {+ "role": {"human": "user", "gpt": "assistant"}.get(+ m.get("role", m.get("from")), m.get("role", m.get("from"))+ ),+ "content": m.get("content", m.get("value")),+ }+ for m in messages+ ]+ kwargs = (+ {} if getattr(chat, "chat_template", None) else {"chat_template": _DEFAULT_CHAT_TEMPLATE}+ )+ return chat.apply_chat_template(messages, tokenize = False, add_generation_prompt = False, **kwargs)+++def _text_tokenizer(tokenizer):+ """The text tokenizer of a processor. Only a processor is unwrapped: MistralCommonBackend is+ itself an HF tokenizer but keeps its raw mistral_common backend in ``.tokenizer``."""+ try:+ from transformers import ProcessorMixin+ except ImportError:+ return tokenizer+ return tokenizer.tokenizer if isinstance(tokenizer, ProcessorMixin) else tokenizer+++def _calibration_ids(dataset, tokenizer, num_samples, max_seq_length):+ """Token id lists, truncated to ``max_seq_length`` like llm-compressor's tokenize step."""+ text_tokenizer = _text_tokenizer(tokenizer)+ chat = tokenizer if getattr(tokenizer, "chat_template", None) else text_tokenizer+ if dataset is None:+ dataset = _DEFAULT_DATASET+ if isinstance(dataset, str):+ from datasets import load_dataset+ dataset = load_dataset(dataset, split = "train", streaming = True)+ ids = []+ for row in dataset:+ if len(ids) >= num_samples:+ break+ if isinstance(row, str):+ row = {"text": row}+ elif isinstance(row, (list, tuple)):+ row = {"input_ids": row}+ if "input_ids" in row:+ sample = list(row["input_ids"])+ mask = row.get("attention_mask")+ if mask is not None: # padded pretokenized rows: keep the real tokens only+ sample = [t for t, m in zip(sample, list(mask)) if m]+ sample = sample[:max_seq_length]+ else:+ if tokenizer is None:+ raise ValueError(+ "Unsloth: REAP needs a tokenizer unless the dataset has `input_ids`."+ )+ sample = text_tokenizer(+ _row_text(row, tokenizer, chat), truncation = True, max_length = max_seq_length+ )["input_ids"]+ if len(sample):+ ids.append(sample)+ if not ids:+ raise ValueError("Unsloth: REAP got no calibration samples from the dataset.")+ return ids+++def _batches(ids, max_batch_tokens, pad_id):+ """Length-sorted right-padded batches. Right padding cannot change real tokens in a causal+ model, so no attention mask is passed and padded positions are only masked out of saliency."""+ order = sorted(range(len(ids)), key = lambda i: -len(ids[i]))+ batches, current = [], []+ for i in order:+ width = len(ids[current[0]]) if current else len(ids[i])+ if current and (len(current) + 1) * width > max_batch_tokens:+ batches.append(current)+ current = []+ current.append(i)+ if current:+ batches.append(current)+ out = []+ for batch in batches:+ width = len(ids[batch[0]])+ input_ids = torch.full((len(batch), width), pad_id, dtype = torch.long)+ valid = torch.zeros((len(batch), width), dtype = torch.bool)+ for row, i in enumerate(batch):+ input_ids[row, : len(ids[i])] = torch.tensor(ids[i], dtype = torch.long)+ valid[row, : len(ids[i])] = True+ out.append((input_ids, valid))+ return out+++def _to(+ obj,+ device,+ memo = None,+):+ """Moves nested tensors; ``memo`` keeps tensors shared across layers (rotary tables) shared."""+ if torch.is_tensor(obj):+ if memo is None:+ return obj.to(device, non_blocking = True)+ if id(obj) not in memo:+ memo[id(obj)] = obj.to(device)+ return memo[id(obj)]+ if isinstance(obj, tuple):+ return tuple(_to(o, device, memo) for o in obj)+ if isinstance(obj, list):+ return [_to(o, device, memo) for o in obj]+ if isinstance(obj, dict):+ return {k: _to(v, device, memo) for k, v in obj.items()}+ return obj+++def _tensor_bytes(obj, seen = None):+ """Bytes of the distinct tensors nested in ``obj`` (masks shared by every layer count once)."""+ seen = set() if seen is None else seen+ if torch.is_tensor(obj):+ if id(obj) in seen:+ return 0+ seen.add(id(obj))+ return obj.numel() * obj.element_size()+ if isinstance(obj, (tuple, list)):+ return sum(_tensor_bytes(o, seen) for o in obj)+ if isinstance(obj, dict):+ return sum(_tensor_bytes(o, seen) for o in obj.values())+ return 0+++def _capture_inputs(owner, layers, input_ids):+ """Embedding output plus the per-layer kwargs (masks, rotary embeddings, ...) the model+ passes to each decoder layer, by running the model with every layer stubbed out."""+ captured = [None] * len(layers)+ first = {}+ originals = []+ for i, layer in enumerate(layers):+ originals.append(layer.__dict__.get("forward"))++ def stub(+ *args,+ _i = i,+ **kwargs,+ ):+ hidden = args[0] if args else kwargs.pop("hidden_states")+ if _i == 0:+ first["hidden"] = hidden+ captured[_i] = (tuple(args[1:]), kwargs)+ if _i == len(layers) - 1:+ raise _StopForward+ return hidden++ layer.forward = stub+ try:+ owner(input_ids = input_ids, use_cache = False)+ except _StopForward:+ pass+ except (TypeError, ValueError) as error:+ if "unpack" not in str(error):+ raise+ raise ValueError(_STATEFUL_LAYERS) from error+ finally:+ for layer, original in zip(layers, originals):+ if original is None:+ del layer.forward+ else:+ layer.forward = original+ return first["hidden"], captured+++def _layer_device(layer, default):+ for p in layer.parameters():+ return p.device+ return default+++def _hidden(out):+ return out[0] if isinstance(out, (tuple, list)) else out+++def _collective_device():+ """(device, accumulation dtype) for a collective on the current process group: CUDA for NCCL,+ CPU for gloo, else the current accelerator (HCCL on Ascend, XCCL on Intel XPU). MPS, XPU and+ NPU have no (or slow) float64, so they accumulate in float32 like ``_Saliency.add``."""+ backend = str(torch.distributed.get_backend())+ if "nccl" in backend:+ device = torch.device("cuda", torch.cuda.current_device())+ elif backend == "gloo":+ device = torch.device("cpu")+ else:+ # torch 2.6 names the index getter current_device_idx.+ accel = torch.accelerator+ index = getattr(accel, "current_device_index", None) or accel.current_device_idx+ device = torch.device(accel.current_accelerator().type, index())+ return device, torch.float64 if device.type in ("cuda", "cpu") else torch.float32+++# Written next to a pruned checkpoint saved by reap_prune(..., save_directory).+REAP_PROVENANCE_FILE = "unsloth_reap.json"+++def _stream_in(layer, device):+ """``(device the layer runs on, whether it was moved)``: a CPU-resident layer (not under+ accelerate hooks) is moved to ``device`` for its turn."""+ layer_device = _layer_device(layer, device)+ if layer_device.type == "cpu" and device.type != "cpu" and not hasattr(layer, "_hf_hook"):+ layer.to(device)+ return device, True+ return layer_device, False+++def _ranks_agree(error, what):+ """Every rank learns whether any rank failed ``what``, so all stop together instead of some+ waiting in a later collective. Re-raises this rank's own error. No-op without a process group."""+ dist = torch.distributed+ if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() < 2:+ if error is not None:+ raise error+ return+ device, acc = _collective_device()+ failed = torch.tensor([1.0 if error is not None else 0.0], dtype = acc, device = device)+ dist.all_reduce(failed)+ if error is not None:+ raise error+ if failed.item():+ raise RuntimeError(f"Unsloth: REAP stopped because another rank failed to {what}.")+++class _Saliency:+ """Running sum of ``gate * ||expert output||`` and routed-token count per expert (float64)."""++ def __init__(self, num_experts):+ self.num_experts = num_experts+ self.total = None+ self.count = None++ def add(+ self,+ expert_ids,+ weights,+ norms,+ mask = None,+ ):+ # MPS, consumer Intel XPUs and Ascend NPUs have no (or slow, partial) float64.+ acc = torch.float64 if norms.device.type in ("cuda", "cpu") else torch.float32+ if self.total is None:+ self.total = torch.zeros(self.num_experts, dtype = acc, device = norms.device)+ self.count = torch.zeros(self.num_experts, dtype = acc, device = norms.device)+ ones = torch.ones_like(norms, dtype = acc) if mask is None else mask.to(acc)+ self.total.index_add_(0, expert_ids, weights.to(acc) * norms.to(acc) * ones)+ self.count.index_add_(0, expert_ids, ones)++ def all_reduce(self):+ """Sum over the ranks of a distributed job, so every rank keeps the same experts (a DDP+ job with sharded calibration data would otherwise pair unrelated experts across ranks)."""+ dist = torch.distributed+ if not (dist.is_available() and dist.is_initialized()) or dist.get_world_size() < 2:+ return+ device, acc = _collective_device()+ both = torch.zeros(2, self.num_experts, dtype = acc, device = device)+ if self.total is not None:+ both[0] = self.total.to(device, acc)+ both[1] = self.count.to(device, acc)+ dist.all_reduce(both)+ self.total, self.count = both[0], both[1]++ def mean(self):+ if self.total is None:+ return torch.zeros(self.num_experts, dtype = torch.float64)+ return self.total.cpu().double() / self.count.cpu().double().clamp(min = 1.0)+++def _retained(saliency, n_drop, n_drop_per_group, n_group):+ """Experts kept after dropping the lowest saliency ones (per group for group-limited routers)."""+ num_experts = saliency.numel()+ if n_drop_per_group is None:+ drop = set(torch.topk(saliency, n_drop, largest = False).indices.tolist())+ return [i for i in range(num_experts) if i not in drop]+ group_size = num_experts // n_group+ kept = []+ for g in range(n_group):+ lo = g * group_size+ drop = {+ lo + i+ for i in torch.topk(+ saliency[lo : lo + group_size], n_drop_per_group, largest = False+ ).indices.tolist()+ }+ kept.extend(i for i in range(lo, lo + group_size) if i not in drop)+ return kept++[email protected]_grad()+def _prune_block(block, experts, router, retained, num_experts):+ index = torch.tensor(retained, dtype = torch.long)+ # Experts on dim 0; anything per expert inside the router (HunYuan `gate.wg`, AfMoE `router.gate`,+ # Ernie's [1, E] `moe_statics` bias); the block's own expert biases.+ targets = [(experts, None)] + [(m, None) for m in router.modules()]+ targets.append((block, ("e_score_correction_bias", "expert_bias")))+ for module, only in targets:+ tensors = list(module.named_parameters(recurse = False))+ tensors += list(module.named_buffers(recurse = False))+ # Inkling's router weight carries the shared experts' rows after the routed ones.+ shared = getattr(module, "n_shared_experts", 0) if module is not experts else 0+ shared = shared if isinstance(shared, int) else 0+ for name, tensor in tensors:+ if only is not None and name not in only or not tensor.dim():+ continue+ if shared and tensor.shape[0] == num_experts + shared:+ kept = torch.cat([index, torch.arange(num_experts, num_experts + shared)])+ new = tensor.data.index_select(0, kept.to(tensor.device)).contiguous()+ setattr(module, name, nn.Parameter(new, requires_grad = tensor.requires_grad))+ continue+ if tensor.shape[0] == num_experts:+ dim = 0+ elif module is not experts and tensor.shape[-1] == num_experts:+ dim = -1+ else:+ continue+ new = tensor.data.index_select(dim, index.to(tensor.device)).contiguous()+ if isinstance(tensor, nn.Parameter):+ setattr(module, name, nn.Parameter(new, requires_grad = tensor.requires_grad))+ else:+ module._buffers[name] = new+ for key in _NUM_EXPERTS_MODULE_KEYS:+ value = getattr(module, key, None)+ if isinstance(value, int) and value == num_experts:+ setattr(module, key, len(retained))+ if shared and getattr(module, "n_total_experts", None) == num_experts + shared:+ module.n_total_experts = len(retained) + shared+ if module is not experts and getattr(module, "out_features", None) == num_experts:+ module.out_features = len(retained)+ # Kernel configs Unsloth's MoE backend cached for the old expert count.+ if getattr(experts, "_unsloth_moe_configs", None) is not None:+ experts._unsloth_moe_configs = None+++class _ExpertsTap:+ """Wraps an experts module's forward to record per (token, slot) expert output norms."""++ def __init__(+ self,+ experts,+ saliency,+ post = None,+ ):+ self.experts = experts+ self.saliency = saliency+ # Nemotron-H latent MoE maps the summed expert output back to the hidden size after the+ # experts, so ||f_j|| is measured after that shared Linear (without its bias, added once).+ self.post = post+ self.valid = None+ self.stop = False+ self.original = experts.__dict__.get("forward")+ self.forward = experts.forward+ experts.forward = self++ def __call__(self, hidden_states, top_k_index, top_k_weights, *args, **kwargs):+ shape = hidden_states.shape+ hidden_states = hidden_states.reshape(-1, shape[-1])+ top_k_index = top_k_index.reshape(hidden_states.shape[0], -1)+ top_k_weights = top_k_weights.reshape(hidden_states.shape[0], -1)+ n_tokens, top_k = top_k_index.shape+ valid = self.valid if self.valid is not None and self.valid.numel() == n_tokens else None+ step = max(+ 1, _TAP_CHUNK_BYTES // (top_k * hidden_states.shape[-1] * hidden_states.element_size())+ )+ outs = []+ for start in range(0, n_tokens, step):+ x = hidden_states[start : start + step]+ index = top_k_index[start : start + step]+ gate = top_k_weights[start : start + step]+ n = x.shape[0]+ # One row per routed (token, slot) pair with gate 1 gives each expert's raw output f_j(t).+ rows = self.forward(+ x.repeat_interleave(top_k, dim = 0),+ index.reshape(-1, 1),+ torch.ones((n * top_k, 1), dtype = gate.dtype, device = gate.device),+ *args,+ **kwargs,+ )+ scored = rows if self.post is None else nn.functional.linear(rows, self.post.weight)+ norms = torch.linalg.vector_norm(+ scored,+ dim = -1,+ dtype = torch.float64 if rows.dtype == torch.float64 else torch.float32,+ )+ mask = None+ if valid is not None:+ # Masking by multiplication avoids the host sync boolean indexing needs.+ mask = (+ valid[start : start + step]+ .to(norms.device, non_blocking = True)+ .repeat_interleave(top_k)+ )+ self.saliency.add(index.reshape(-1).long(), gate.reshape(-1), norms, mask)+ if not self.stop:+ out = rows.view(n, top_k, -1) * gate.unsqueeze(-1).to(rows.dtype)+ outs.append(out.sum(1).to(rows.dtype))+ if self.stop:+ raise _StopForward+ return (outs[0] if len(outs) == 1 else torch.cat(outs)).reshape(shape)++ def remove(self):+ if self.original is None:+ del self.experts.forward+ else:+ self.experts.forward = self.original+++def _residual_split(layer, block):+ """(norm, block) when the layer ends ``residual + block(norm(residual))`` so a pruned layer's+ output can be rebuilt from the residual alone; verified bitwise per layer before use."""+ norm = getattr(layer, "post_attention_layernorm", None)+ if (+ isinstance(norm, nn.Module)+ and not isinstance(norm, nn.ModuleList)+ and getattr(layer, "mlp", None) is block+ ):+ return norm+ return None+++def _resolve_dataset(dataset, tokenizer):+ """The calibration rows to read: a DatasetDict's train (or only) split. Raises when text rows+ come without a tokenizer. Data dependent, so it runs where every rank agrees on failures."""+ if isinstance(dataset, dict):+ # load_dataset(...) without split= gives a DatasetDict, which iterates its split names.+ if "train" in dataset or len(dataset) == 1:+ dataset = dataset["train"] if "train" in dataset else next(iter(dataset.values()))+ else:+ raise ValueError(+ f"Unsloth: REAP got a dataset with the splits {sorted(dataset)}; pass one of them, "+ "e.g. `dataset['validation']`."+ )+ # A dataset name (or the default) is text; other rows are checked as they are read, since a+ # streaming dataset built from a generator declares no columns.+ if (dataset is None or isinstance(dataset, str)) and tokenizer is None:+ raise ValueError("Unsloth: REAP needs a tokenizer unless the dataset has `input_ids`.")+ return dataset+++def _model_parallel_layout(model):+ """Name of a model-parallel layout (expert / tensor / pipeline / FSDP / sequence), else None."""+ distributed = getattr(getattr(model, "config", None), "distributed_config", None)++ def field(name):+ if isinstance(distributed, dict):+ return distributed.get(name)+ return getattr(distributed, name, None)++ if field("enable_expert_parallel"):+ return "expert parallel"+ for name, label in (+ ("tp_size", "tensor parallel"),+ ("pp_size", "pipeline parallel"),+ ("fsdp_size", "FSDP sharded"),+ ):+ size = field(name)+ if isinstance(size, int) and size > 1:+ return label+ if field("enable_sequence_parallel"):+ return "sequence parallel"+ tp_size = getattr(model, "_tp_size", None)+ if isinstance(tp_size, int) and tp_size > 1:+ return "tensor parallel"+ return None++[email protected]_grad()+def reap_prune(+ model,+ tokenizer = None,+ sparsity = 0.25,+ dataset = None,+ num_calibration_samples = 512,+ max_seq_length = 2048,+ max_batch_tokens = 16384,+ report_path = None,+ device = None,+ offload_hidden_states = None,+ save_directory = None,+):+ """Prune ``sparsity`` of the routed experts in every MoE layer with REAP, in place.++ Mirrors llm-compressor's ``REAPPruningModifier`` (same saliency ``S_j = mean(g_j * ||f_j||)``,+ selection and layer-by-layer calibration), on Unsloth's fused MoE kernels.++ Args:+ model: a 16-bit MoE model (e.g. from ``FastLanguageModel.from_pretrained(..., load_in_4bit = False)``),+ before ``get_peft_model`` or after merging LoRA.+ tokenizer: tokenizer or processor, used to format and tokenize calibration text.+ sparsity: fraction of experts removed per MoE layer, in (0, 1).+ dataset: Hugging Face dataset name, ``datasets`` dataset, or list of strings / token id lists.+ Rows may carry ``input_ids``, ``text``, ``messages`` or ``conversations``.+ Defaults to llm-compressor's REAP example dataset ``mlabonne/open-perfectblend``.+ num_calibration_samples: number of calibration rows.+ max_seq_length: truncation length per row.+ max_batch_tokens: padded tokens per calibration batch.+ report_path: optional ``.json`` file for ``{layer name: [kept expert ids]}``.+ device: device layers run on; layers stored on the CPU are moved there one at a time.+ offload_hidden_states: keep calibration activations on the CPU (default: only when they+ would take more than a quarter of the free GPU memory).+ save_directory: also save the pruned model (and tokenizer) here and load later merges from+ it. Needed before ``get_peft_model`` when the LoRA model will be saved merged or to GGUF,+ since those rebuild the base weights from the checkpoint on disk.++ Returns:+ The pruned model. ``model.config`` carries the new expert count, so ``save_pretrained``,+ ``save_pretrained_merged`` and GGUF export write the smaller model.+ """+ if not 0.0 < float(sparsity) < 1.0:+ raise ValueError(f"Unsloth: REAP sparsity must be in (0, 1), got {sparsity}.")+ if hasattr(model, "peft_config"):+ raise RuntimeError(+ "Unsloth: run REAP on the base model before `get_peft_model`, or merge the LoRA "+ "adapters first (`model = model.merge_and_unload()`)."+ )+ if getattr(model, "vllm_engine", None) is not None:+ raise RuntimeError(+ "Unsloth: REAP cannot update the attached vLLM engine, so `fast_generate` would keep the "+ "unpruned experts. Load the model without `fast_inference = True` to prune it."+ )+ if model.__dict__.get("_unsloth_mtp_head") is not None:+ raise RuntimeError(+ "Unsloth: REAP does not prune the attached MTP head, so a save would write it with the "+ "old expert count. Run `reap_prune` before `get_mtp_head`."+ )+ layout = _model_parallel_layout(model)+ if layout is not None:+ # Each rank would hold a slice of the experts (or, pipelined, other layers); plain DP is fine.+ raise RuntimeError(+ f"Unsloth: REAP does not support {layout} models. Load the model on one device (or with "+ "`device_map`, or plain data parallel) to prune it."+ )+ configs = _configs(model)+ num_experts = _first_attr(configs, _NUM_EXPERTS_KEYS)+ top_k = _first_attr(configs, _TOP_K_KEYS, per_layer = True)+ if num_experts is None or top_k is None:+ raise ValueError(+ "Unsloth: REAP needs an MoE model with an expert count and top_k in its config."+ )+ n_group = _first_attr(configs, ("n_group", "num_expert_group"))+ topk_group = _first_attr(configs, ("topk_group", "top_k_group"))+ if n_group is not None and n_group > 1 and topk_group is None:+ # The router still splits experts into groups; pruning them globally would shift the+ # group boundaries. A lone topk_group (or n_group 1) routes ungrouped.+ raise ValueError(+ f"Unsloth: REAP found n_group = {n_group} but no topk_group in the config, so it cannot "+ "prune per router group."+ )+ if n_group is None or n_group <= 1 or topk_group is None:+ n_group = topk_group = None++ # Calibration runs each decoder stack alone on right padded text, which needs causal attention.+ if getattr(model, "_unsloth_slow_diffusion", False) or any(+ getattr(c, "use_bidirectional_attention", None) == "all"+ or str(getattr(c, "model_type", "")).startswith("diffusion")+ for c in configs+ ):+ raise ValueError(+ "Unsloth: REAP supports causal language models, not bidirectional or diffusion ones."+ )+ for config in configs:+ if getattr(config, "zero_expert_num", None):+ raise ValueError(+ "Unsloth: REAP does not support MoE routers with zero (identity) experts."+ )+ owner, layers = _find_layers(model, num_experts)+ if layers is None:+ raise RuntimeError(+ "Unsloth: REAP found no MoE layers with fused experts. It needs transformers 5.x MoE "+ "modules (an `experts` module with 3D expert weights next to a `gate` / `router`)."+ )+ if any(p.startswith("prev_") for p in inspect.signature(layers[0].forward).parameters):+ raise ValueError(_STATEFUL_LAYERS)+ # Non-causal text attention (openai/privacy-filter) carries no config flag.+ if any(getattr(m, "is_causal", None) is False for layer in layers for m in layer.modules()):+ raise ValueError(+ "Unsloth: REAP supports causal language models, not bidirectional or diffusion ones."+ )++ layer_names = {id(m): n for n, m in model.named_modules()}+ for layer in layers:+ moes = _layer_moes(layer, num_experts)+ if len(moes) > 1:+ raise ValueError(+ "Unsloth: REAP supports one MoE block per decoder layer; "+ f"{layer_names[id(layer)]} has {len(moes)} (e.g. separate text and vision experts)."+ )+ moe = moes[0] if moes else None+ if moe is not None:+ _check_weights(moe[2], moe[3], layer_names[id(moe[1])])+ if moe is not None and any(hasattr(m, "tid2eid") for m in moe[3].modules()):+ raise ValueError(+ "Unsloth: REAP does not support hash-routed MoE layers (fixed token to expert "+ "tables, e.g. DeepSeek-V4), since dropped experts cannot be remapped."+ )++ n_drop = int(num_experts * float(sparsity))+ if n_drop == 0:+ raise ValueError(+ f"Unsloth: REAP sparsity={sparsity} drops 0 of {num_experts} experts; raise the sparsity."+ )+ n_drop_per_group = None+ if n_group is not None:+ if num_experts % n_group:+ raise ValueError(+ f"Unsloth: {num_experts} experts are not divisible by n_group={n_group}."+ )+ n_drop_per_group = round(n_drop / n_group)+ if n_drop_per_group == 0:+ raise ValueError(+ f"Unsloth: REAP sparsity={sparsity} drops 0 experts per router group "+ f"({num_experts // n_group} experts in each of {n_group} groups); raise the sparsity."+ )+ if n_drop_per_group * n_group != n_drop:+ print(+ f"Unsloth: REAP group-limited routing drops {n_drop_per_group} experts per group, "+ f"{n_drop_per_group * n_group} in total instead of {n_drop}."+ )+ n_drop = n_drop_per_group * n_group+ if num_experts // n_group - n_drop_per_group < 2:+ # DeepSeek style routers score each group by the sum of its top 2 experts.+ raise ValueError(+ f"Unsloth: REAP sparsity={sparsity} leaves fewer than 2 experts in each of the "+ f"{n_group} router groups; lower the sparsity."+ )+ available = topk_group * (num_experts // n_group - n_drop_per_group)+ else:+ available = num_experts - n_drop+ if top_k > available:+ raise ValueError(+ f"Unsloth: REAP sparsity={sparsity} is too aggressive: the router picks top_k={top_k} "+ f"experts per token but only {available} would stay reachable. Lower the sparsity."+ )+ kept_count = num_experts - n_drop++ if device is None:+ device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")+ device = torch.device(device)++ pad_id = 0+ text_tokenizer = _text_tokenizer(tokenizer)+ if text_tokenizer is not None:+ pad_id = text_tokenizer.pad_token_id+ if pad_id is None:+ pad_id = text_tokenizer.eos_token_id or 0+ error = None+ try:+ dataset = _resolve_dataset(dataset, tokenizer)+ ids = _calibration_ids(dataset, tokenizer, num_calibration_samples, max_seq_length)+ batches = _batches(ids, max(max_batch_tokens, max(len(x) for x in ids)), pad_id)+ except Exception as e:+ error = e+ # One rank's empty or malformed shard must stop every rank before the per-layer collectives.+ _ranks_agree(error, "prepare its calibration data")+ n_tokens = sum(len(x) for x in ids)++ report, saliencies = {}, {}+ pruning = None+ modes = [(m, m.training) for m in model.modules()]+ model.eval()+ start = time.time()+ try:+ hidden, kwargs_per_batch, valid_per_batch = [], [], []+ embed_device = _layer_device(owner, device)+ store = None+ capture_error = None+ try:+ for input_ids, valid in batches:+ h, kwargs = _capture_inputs(owner, layers, input_ids.to(embed_device))+ if not hidden:+ # Decided from the first batch, so activations never pile up on the GPU first.+ if offload_hidden_states is None and device.type == "cuda":+ padded = sum(x.numel() for x, _ in batches) * h.shape[-1] * h.element_size()+ # Eager attention keeps a [batch, 1, seq, seq] mask per batch in the kwargs.+ padded += _tensor_bytes(kwargs) * len(batches)+ offload_hidden_states = padded > torch.cuda.mem_get_info(device)[0] // 4+ store = torch.device("cpu") if offload_hidden_states else None+ if store is not None:+ h = h.to(store).pin_memory() if torch.cuda.is_available() else h.to(store)+ # Rotary tables and any materialized masks go too, or they pile up per batch.+ kwargs = _to(kwargs, store, {})+ hidden.append(h)+ kwargs_per_batch.append(kwargs)+ valid_per_batch.append(valid.reshape(-1))+ except Exception as e:+ capture_error = e+ _ranks_agree(capture_error, "capture its calibration activations")+ width = hidden[0].shape[-1]++ print(+ f"Unsloth: REAP pruning {n_drop} of {num_experts} experts per MoE layer "+ f"({len(ids)} samples, {n_tokens} tokens, {len(batches)} batches, hidden {width})."+ )+ rebuilt_layers = 0+ for index, layer in enumerate(layers):+ move_error = None+ try:+ layer_device, moved = _stream_in(layer, device)+ except Exception as e:+ layer_device, moved, move_error = device, True, e+ post_error = move_error++ try:++ def run(b, h):+ args, kwargs = kwargs_per_batch[b][index]+ # Gemma 4's shared_kv_states (a UserDict shared across layers) follows offloaded activations.+ shared = [+ v+ for v in kwargs.values()+ if isinstance(v, MutableMapping) and not isinstance(v, dict)+ ]+ # Moved every call: under a device_map the previous layer may sit on another GPU.+ for m in shared:+ m.update(_to(dict(m), layer_device))+ try:+ return _hidden(+ layer(h, *_to(args, layer_device), **_to(kwargs, layer_device))+ )+ finally:+ if store is not None:+ for m in shared:+ m.update(_to(dict(m), store))++ moe = _layer_moe(layer, num_experts)+ norm = None+ if moe is not None:+ _, block, experts, router = moe+ block_name = layer_names[id(block)]+ stage_error = None+ try:+ if move_error is not None:+ raise move_error+ norm = _residual_split(layer, block)+ residuals = []+ if norm is not None:+ hook = norm.register_forward_pre_hook(+ lambda module, args: residuals.append(args[0])+ )+ try:+ h0 = hidden[0].to(layer_device, non_blocking = True)+ reference = run(0, h0)+ try:+ same = len(residuals) == 1 and torch.equal(+ reference,+ residuals[0]+ + _hidden(block(norm(residuals[0]))).reshape(+ residuals[0].shape+ ),+ )+ except (+ Exception+ ): # block needs more than hidden states (DeepSeek-V4)+ same = False+ if not same:+ norm = None+ finally:+ residuals.clear()+ hook.remove()++ saliency = _Saliency(num_experts)+ post = getattr(block, "fc2_latent_proj", None)+ tap = _ExpertsTap(+ experts, saliency, post if isinstance(post, nn.Linear) else None+ )+ # Pass 2 recomputes the layer output, so pass 1 stops right after the experts+ # (not under accelerate hooks, whose post-forward offload must run).+ tap.stop = not any(hasattr(m, "_hf_hook") for m in layer.modules())+ hook = None+ if norm is not None:+ hook = norm.register_forward_pre_hook(+ lambda module, args: residuals.append(args[0])+ )+ try:+ for b in range(len(batches)):+ tap.valid = valid_per_batch[b]+ try:+ run(b, hidden[b].to(layer_device, non_blocking = True))+ except _StopForward:+ pass+ if norm is not None:+ # The residual is all a verified layer needs later, so it replaces the input.+ r = residuals.pop()+ hidden[b] = (+ r if store is None else r.to(store, non_blocking = True)+ )+ finally:+ tap.remove()+ if hook is not None:+ hook.remove()+ except Exception as e:+ stage_error = e+ _ranks_agree(stage_error, f"score {block_name}")++ saliency.all_reduce()+ if saliency.total is None or not saliency.count.sum().item():+ raise RuntimeError(+ f"Unsloth: REAP saw no tokens reach {block_name}'s experts module; its MoE "+ "forward computes the experts itself, so REAP cannot score them."+ )+ scores = saliency.mean()+ saliencies[block_name] = scores.tolist()+ retained = _retained(scores, n_drop, n_drop_per_group, n_group)+ rebuilt_layers += norm is not None+ assert len(retained) == kept_count+ # Tensors change from here on: no longer the checkpoint any earlier save wrote.+ model._unsloth_reap_saved = None+ pruning = block_name+ try:+ _prune_block(block, experts, router, retained, num_experts)+ report[block_name] = retained+ pruning = None+ except Exception as e:+ post_error = e++ try:+ for b in range(len(batches) if post_error is None else 0):+ h = hidden[b].to(layer_device, non_blocking = True)+ if norm is not None:+ out = h + _hidden(block(norm(h))).reshape(h.shape)+ else:+ out = run(b, h)+ hidden[b] = out if store is None else out.to(store, non_blocking = True)+ except Exception as e:+ post_error = e+ _ranks_agree(post_error, f"run decoder layer {index + 1}")+ finally:+ if moved:+ layer.to("cpu")+ print(+ f"Unsloth: REAP layer {index + 1}/{len(layers)} done ({time.time() - start:.1f}s)."+ )+ except BaseException:+ if report or pruning is not None:+ # Partly pruned: mark it REAP modified and unsaved so the PEFT / save guards refuse it.+ partial = dict(report)+ if pruning is not None:+ partial.setdefault(pruning, [])+ model._unsloth_reap_report = partial+ model._unsloth_reap_saved = None+ print(+ f"Unsloth: REAP stopped after pruning {len(report)} MoE layers, so the model now mixes "+ "pruned and unpruned layers. Reload it before running REAP again."+ )+ raise+ finally:+ for module, training in modes:+ module.training = training++ if not report:+ raise RuntimeError("Unsloth: REAP found no MoE layers to prune.")+ for config in configs:+ for key in _NUM_EXPERTS_KEYS:+ if getattr(config, key, None) == num_experts:+ setattr(config, key, kept_count)+ # MTP layers live outside `layers` and are not pruned; they would be rebuilt at the new count.+ for key in (+ "num_nextn_predict_layers",+ "num_mtp_layers",+ "mtp_num_layers",+ "mtp_num_hidden_layers",+ ):+ value = getattr(config, key, None)+ if isinstance(value, int) and value > 0:+ setattr(config, key, 0)+ print(+ f"Unsloth: REAP set {key} = 0, since the MTP layers keep the old expert count."+ )+ # Wrappers cache the count too (Mixtral / Jamba pass model.num_experts to the router aux loss).+ for module in model.modules():+ for key in _NUM_EXPERTS_MODULE_KEYS:+ value = getattr(module, key, None)+ if isinstance(value, int) and value == num_experts:+ setattr(module, key, kept_count)+ model._unsloth_reap_report = report+ model._unsloth_reap_saliency = saliencies+ dist = torch.distributed+ multi_rank = dist.is_available() and dist.is_initialized() and dist.get_world_size() > 1+ if save_directory is not None:+ save_directory = os.path.abspath(save_directory)+ source_model = getattr(model.config, "_name_or_path", None) or None+ if source_model and os.path.exists(source_model):+ # This folder may be pushed to the Hub: name a local source without its path.+ source_model = os.path.basename(os.path.normpath(source_model))+ error = None+ if not multi_rank or dist.get_rank() == 0:+ try:+ model.save_pretrained(save_directory)+ if tokenizer is not None:+ tokenizer.save_pretrained(save_directory)+ # Survives a reload, unlike the in-process marker: the adapter push guard reads it.+ with open(+ os.path.join(save_directory, REAP_PROVENANCE_FILE), "w", encoding = "utf-8"+ ) as f:+ json.dump({"source_model": source_model, "kept_experts": report}, f, indent = 1)+ except Exception as e:+ error = e+ _ranks_agree(error, f"save the pruned model to {save_directory}")+ # Merged / GGUF saves read config._name_or_path; PEFT's adapter config reads model.name_or_path.+ for config in configs:+ config._name_or_path = save_directory+ for module in model.modules():+ if "name_or_path" in module.__dict__:+ module.name_or_path = save_directory+ if getattr(model, "_unsloth_modelscope_snapshot", None):+ model._unsloth_modelscope_snapshot = save_directory+ model._unsloth_reap_saved = save_directory+ if report_path is not None:+ error = None+ if not multi_rank or dist.get_rank() == 0:+ try:+ os.makedirs(os.path.dirname(os.path.abspath(report_path)), exist_ok = True)+ with open(report_path, "w", encoding = "utf-8") as f:+ json.dump(report, f)+ except Exception as e:+ error = e+ _ranks_agree(error, f"write the REAP report to {report_path}")+ print(+ f"Unsloth: REAP kept {kept_count} of {num_experts} experts in {len(report)} MoE layers "+ f"in {time.time() - start:.1f}s ({rebuilt_layers} reused their attention output)."+ )+ if save_directory is None:+ print(+ "Unsloth: to LoRA finetune the pruned model, pass `save_directory` (or save the pruned "+ "model and load it again) before `get_peft_model`."+ )+ return model+++def raise_if_unsaved_reap(model):+ """LoRA adapters and merged saves record the base checkpoint on disk, which still holds every+ expert after an in-memory prune."""+ if getattr(model, "_unsloth_reap_report", None) and not getattr(+ model, "_unsloth_reap_saved", None+ ):+ raise RuntimeError(+ "Unsloth: this model was REAP pruned in memory only, but LoRA adapters and merged saves "+ f"point at the base checkpoint `{model.config._name_or_path}`, which still has every "+ 'expert. Call `reap_prune(..., save_directory = "pruned_model")` (or save the pruned '+ "model and load it again) before `get_peft_model`."+ )+++def _is_token_rows(dataset):+ if isinstance(dataset, (list, tuple)) and dataset:+ first = dataset[0]+ return isinstance(first, (list, tuple)) or (+ isinstance(first, dict) and "input_ids" in first+ )+ column_names = getattr(dataset, "column_names", None)+ return bool(column_names) and "input_ids" in column_namesdiff --git a/unsloth/save.py b/unsloth/save.pyindex 9f6f23a503e..5a39918ee5d 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -3621,6 +3621,11 @@ def _gguf_reuses_loaded_checkpoint(model, state_dict = None):return Falseif state_dict is not None or getattr(model, "_unsloth_full_finetuning", False):return False+ # REAP pruned in memory: the loaded folder still holds every expert.+ if getattr(model, "_unsloth_reap_report", None) and not getattr(+ model, "_unsloth_reap_saved", None+ ):+ return False# The loaded folder holds the stock MTP head; a trained one needs a fresh checkpoint.if _attached_mtp_head(model) is not None:return False@@ -6125,6 +6130,48 @@ def unsloth_generic_save(# this function's own keywords.safe_serialization = _normalize_safe_serialization(safe_serialization)+ # Inline, builtins only (no new locals: they are forwarded below), and before the Hub revision+ # route, which creates the repo / branch first. After an in-memory REAP prune the base+ # checkpoint on disk still has every expert, and both a merge (which rebuilds from it) and an+ # adapter save (which records it as the base) would use it. get_peft_model refuses this too;+ # raw PEFT does not.+ if (+ isinstance(model, PeftModel)+ and getattr(model, "_unsloth_reap_report", None)+ and not getattr(model, "_unsloth_reap_saved", None)+ ):+ raise RuntimeError(+ "Unsloth: this model was REAP pruned in memory only, but this save would use the base "+ f"checkpoint `{model.config._name_or_path}`, which still has every expert. Nothing was "+ 'written. Call `reap_prune(..., save_directory = "pruned_model")` (or save the pruned '+ "model and load it again) before `get_peft_model`."+ )+ # The pruned base REAP saved is a local folder: an adapter pushed against it names a path no one+ # else can load. In this process the marker says so (even if the folder moved since); after a+ # reload from that folder, the provenance file reap_prune wrote there does.+ if (+ push_to_hub+ and isinstance(model, PeftModel)+ and _is_adapter_save_method(save_method)+ and (+ getattr(model, "_unsloth_reap_saved", None)+ or any(+ os.path.isfile(+ os.path.join(+ str(getattr(c, "base_model_name_or_path", None) or ""), "unsloth_reap.json"+ )+ )+ for c in (getattr(model, "peft_config", None) or {}).values()+ )+ )+ ):+ raise RuntimeError(+ "Unsloth: this adapter's base is a REAP pruned model saved to a local folder, so a "+ "pushed adapter could not be loaded anywhere else. Nothing was uploaded. Push the pruned base "+ "to the Hub first and load it from there before finetuning, or push a merged model "+ '(`save_method = "merged_16bit"`).'+ )+# A trained MTP head can only be stitched into a local copy, which this route makes first.if push_to_hub and (create_pr@@ -6488,6 +6535,19 @@ def unsloth_generic_save_pretrained_merged(asks for a pickle."""_assert_export_target_is_not_base_with_lora_layers(self)+ # Inline, builtins only, like unsloth_generic_save's REAP guard, but before the compressed /+ # torchao dispatch below, whose helpers install their runtime and create the Hub repo first.+ if (+ isinstance(self, PeftModel)+ and getattr(self, "_unsloth_reap_report", None)+ and not getattr(self, "_unsloth_reap_saved", None)+ ):+ raise RuntimeError(+ "Unsloth: this model was REAP pruned in memory only, but this save would use the base "+ f"checkpoint `{self.config._name_or_path}`, which still has every expert. Nothing was "+ 'written. Call `reap_prune(..., save_directory = "pruned_model")` (or save the pruned '+ "model and load it again) before `get_peft_model`."+ )if tokenizer is None:logger.warning_once("Unsloth: You're not saving a tokenizer as well?\n"@@ -6635,6 +6695,19 @@ def unsloth_generic_push_to_hub_merged(asks for a pickle."""_assert_export_target_is_not_base_with_lora_layers(self)+ # Inline, builtins only, like unsloth_generic_save's REAP guard, but before the compressed /+ # torchao dispatch below, whose helpers install their runtime and create the Hub repo first.+ if (+ isinstance(self, PeftModel)+ and getattr(self, "_unsloth_reap_report", None)+ and not getattr(self, "_unsloth_reap_saved", None)+ ):+ raise RuntimeError(+ "Unsloth: this model was REAP pruned in memory only, but this save would use the base "+ f"checkpoint `{self.config._name_or_path}`, which still has every expert. Nothing was "+ 'written. Call `reap_prune(..., save_directory = "pruned_model")` (or save the pruned '+ "model and load it again) before `get_peft_model`."+ )if tokenizer is None:logger.warning_once("Unsloth: You're not saving a tokenizer as well?\n"