Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix FastLanguageModel fused CE loss under FSDP2 with an untied lm_head (#13827)

Daniel Han12288e5e43d2
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index a4da695e551..6e377f0d18d 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -38,6 +38,26 @@ from ._utils import (
set_module_gradient_checkpointing,
switch_training_mode,
)
+
+try:
+ from unsloth_zoo.fused_losses.cross_entropy_loss import unsloth_fsdp_unshard
+except ImportError: # unsloth_zoo predating the helper (unsloth-zoo#1751)
+
+ @torch.compiler.disable
+ def _fsdp_unshard(module):
+ module.unshard()
+
+ def unsloth_fsdp_unshard(module):
+ if not torch.distributed.is_available() or not torch.distributed.is_initialized():
+ return
+ FSDPModule = getattr(sys.modules.get("torch.distributed.fsdp"), "FSDPModule", None)
+ if FSDPModule is None:
+ return
+ for submodule in module.modules():
+ if isinstance(submodule, FSDPModule):
+ _fsdp_unshard(submodule)
+
+
from .loader_utils import (
DEFAULT_DEVICE_MAP,
OFFLOAD_EMBEDDING_AUTO,
@@ -1762,6 +1782,8 @@ def CausalLM_fast_forward(fast_forward_inference):
hidden_states = outputs[0]
bsz, q_len, hd = hidden_states.shape
+ # FSDP2 shards an untied lm_head in its own group, gathered only when lm_head is called.
+ unsloth_fsdp_unshard(self.lm_head)
lm_head = self.lm_head.weight
lm_head_device = lm_head.device
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index 242879e10b0..7b7eda96098 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -38,6 +38,7 @@ from .llama import (
resolve_logit_transforms,
_cache_as_legacy_tuple,
_cached_prefill_defaults,
+ unsloth_fsdp_unshard,
)
from transformers.models.mistral.modeling_mistral import (
MistralAttention,
@@ -303,6 +304,7 @@ def MistralForCausalLM_fast_forward(
hidden_states = outputs[0]
bsz, q_len, hd = hidden_states.shape
+ unsloth_fsdp_unshard(self.lm_head)
lm_head = self.lm_head.weight
lm_head_device = lm_head.device