Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix FastLanguageModel fused CE loss under FSDP2 with an untied lm_head (#13827)
Daniel Han12288e5e43d2
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex a4da695e551..6e377f0d18d 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -38,6 +38,26 @@ from ._utils import (set_module_gradient_checkpointing,switch_training_mode,)++try:+ from unsloth_zoo.fused_losses.cross_entropy_loss import unsloth_fsdp_unshard+except ImportError: # unsloth_zoo predating the helper (unsloth-zoo#1751)++ @torch.compiler.disable+ def _fsdp_unshard(module):+ module.unshard()++ def unsloth_fsdp_unshard(module):+ if not torch.distributed.is_available() or not torch.distributed.is_initialized():+ return+ FSDPModule = getattr(sys.modules.get("torch.distributed.fsdp"), "FSDPModule", None)+ if FSDPModule is None:+ return+ for submodule in module.modules():+ if isinstance(submodule, FSDPModule):+ _fsdp_unshard(submodule)++from .loader_utils import (DEFAULT_DEVICE_MAP,OFFLOAD_EMBEDDING_AUTO,@@ -1762,6 +1782,8 @@ def CausalLM_fast_forward(fast_forward_inference):hidden_states = outputs[0]bsz, q_len, hd = hidden_states.shape+ # FSDP2 shards an untied lm_head in its own group, gathered only when lm_head is called.+ unsloth_fsdp_unshard(self.lm_head)lm_head = self.lm_head.weightlm_head_device = lm_head.devicediff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex 242879e10b0..7b7eda96098 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -38,6 +38,7 @@ from .llama import (resolve_logit_transforms,_cache_as_legacy_tuple,_cached_prefill_defaults,+ unsloth_fsdp_unshard,)from transformers.models.mistral.modeling_mistral import (MistralAttention,@@ -303,6 +304,7 @@ def MistralForCausalLM_fast_forward(hidden_states = outputs[0]bsz, q_len, hd = hidden_states.shape+ unsloth_fsdp_unshard(self.lm_head)lm_head = self.lm_head.weightlm_head_device = lm_head.device