Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Generation keeps inference_mode on DeepSpeed ZeRO-1/2; only ZeRO-3 needs no_grad (#14673)

Daniel Hand4a92bb52d14
diff --git a/tests/test_fsdp_generation_grad_mode.py b/tests/test_fsdp_generation_grad_mode.py
index b8cce6f0e35..d98b01059b6 100644
--- a/tests/test_fsdp_generation_grad_mode.py
+++ b/tests/test_fsdp_generation_grad_mode.py
@@ -10,7 +10,7 @@ import torch
from unsloth.models._utils import _get_inference_mode_context_manager
-_FSDP_ENV = ("ACCELERATE_USE_FSDP", "FSDP_VERSION")
+_FSDP_ENV = ("ACCELERATE_USE_FSDP", "FSDP_VERSION", "ACCELERATE_DEEPSPEED_ZERO_STAGE")
@pytest.fixture
@@ -117,3 +117,76 @@ def test_deepspeed_accelerator_state_scores_under_no_grad(no_launcher):
state.AcceleratorState, "_shared_state", {"distributed_type": "DistributedType.DEEPSPEED"}
)
assert _mode_inside(torch.nn.Linear(2, 2)) == (False, False)
+
+
[email protected]("stage, inference", [("0", True), ("1", True), ("2", True), ("3", False)])
+def test_a_known_zero_stage_below_3_keeps_inference_mode(no_launcher, stage, inference):
+ """ZeRO-1/2 keep whole parameters, so only ZeRO-3 needs no_grad for generation."""
+ no_launcher.setenv("ACCELERATE_USE_DEEPSPEED", "true")
+ no_launcher.setenv("ACCELERATE_DEEPSPEED_ZERO_STAGE", stage)
+ assert _mode_inside(torch.nn.Linear(2, 2)) == (inference, False)
+
+
[email protected]("config_stage, inference", [(2, True), (3, False), (-1, True)])
+def test_the_accelerator_plugin_stage_decides(no_launcher, config_stage, inference):
+ import types
+
+ state = pytest.importorskip("accelerate.state")
+ no_launcher.delenv("ACCELERATE_USE_DEEPSPEED", raising = False)
+ plugin = types.SimpleNamespace(
+ zero_stage = 2, hf_ds_config = types.SimpleNamespace(_stage = config_stage)
+ )
+ no_launcher.setattr(
+ state.AcceleratorState,
+ "_shared_state",
+ {"distributed_type": "DistributedType.DEEPSPEED", "deepspeed_plugins": {"default": plugin}},
+ )
+ assert _mode_inside(torch.nn.Linear(2, 2)) == (inference, False)
+
+
+def test_a_zero2_launch_still_honours_a_zero3_trainer_config(no_launcher):
+ hf_deepspeed = pytest.importorskip("transformers.integrations.deepspeed")
+ no_launcher.setenv("ACCELERATE_USE_DEEPSPEED", "true")
+ no_launcher.setenv("ACCELERATE_DEEPSPEED_ZERO_STAGE", "2")
+ no_launcher.setattr(hf_deepspeed, "is_deepspeed_zero3_enabled", lambda: True)
+ assert _mode_inside(torch.nn.Linear(2, 2)) == (False, False)
+
+
+def test_a_zero2_launch_with_partitioned_params_still_gets_no_grad(no_launcher):
+ no_launcher.setenv("ACCELERATE_USE_DEEPSPEED", "true")
+ no_launcher.setenv("ACCELERATE_DEEPSPEED_ZERO_STAGE", "2")
+ model = torch.nn.Linear(2, 2)
+ model.weight.ds_id = 0
+ assert _mode_inside(model) == (False, False)
+
+
+def test_only_the_selected_deepspeed_plugin_counts(no_launcher):
+ import types
+
+ state = pytest.importorskip("accelerate.state")
+ no_launcher.delenv("ACCELERATE_USE_DEEPSPEED", raising = False)
+ active = types.SimpleNamespace(zero_stage = 2, selected = True, hf_ds_config = None)
+ inactive = types.SimpleNamespace(zero_stage = 3, selected = False, hf_ds_config = None)
+ no_launcher.setattr(
+ state.AcceleratorState,
+ "_shared_state",
+ {
+ "distributed_type": "DistributedType.DEEPSPEED",
+ "deepspeed_plugins": {"a": active, "b": inactive},
+ },
+ )
+ assert _mode_inside(torch.nn.Linear(2, 2)) == (True, False)
+
+
+def test_the_grpo_preamble_copy_is_self_contained():
+ # rl_replacements appends only this function's source to the generated GRPO trainer.
+ import ast, inspect
+ from unsloth.models._utils import _get_inference_mode_context_manager as fn
+
+ tree = ast.parse(inspect.getsource(fn))
+ called = {
+ n.func.id
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
+ }
+ assert "_deepspeed_zero_stage" not in called
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 3e9d8720d69..aa56b77e1da 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -9137,13 +9137,42 @@ def _get_inference_mode_context_manager(model: torch.nn.Module):
)
if isinstance(shared_state, dict) and "FSDP" in str(shared_state.get("distributed_type", "")):
return torch.no_grad()
- # ZeRO-3 keeps small persistent gathered params, so the same poisoning applies.
- if os.environ.get("ACCELERATE_USE_DEEPSPEED", "").strip().lower() in ("1", "true", "yes", "on"):
- return torch.no_grad()
- if isinstance(shared_state, dict) and "DEEPSPEED" in str(
- shared_state.get("distributed_type", "")
- ):
- return torch.no_grad()
+ # ZeRO-3 reuses gathered persistent params (same poisoning); a known stage below 3 does not.
+ deepspeed = os.environ.get("ACCELERATE_USE_DEEPSPEED", "").strip().lower() in (
+ "1",
+ "true",
+ "yes",
+ "on",
+ ) or (
+ isinstance(shared_state, dict)
+ and "DEEPSPEED" in str(shared_state.get("distributed_type", ""))
+ )
+ if deepspeed:
+ # Inline, not a helper: rl_replacements copies only this function into the GRPO trainer.
+ stages = []
+ hf_deepspeed = sys.modules.get("transformers.integrations.deepspeed", None)
+ try:
+ if hf_deepspeed is not None and hf_deepspeed.is_deepspeed_zero3_enabled():
+ stages.append(3)
+ except Exception:
+ pass
+ plugins = shared_state.get("deepspeed_plugins") if isinstance(shared_state, dict) else None
+ if plugins is not None and not isinstance(plugins, dict):
+ plugins = {"default": plugins}
+ for plugin in (plugins or {}).values():
+ if not getattr(plugin, "selected", True):
+ continue
+ # A config file's stage lives in hf_ds_config; zero_stage defaults to 2 regardless.
+ stage = getattr(getattr(plugin, "hf_ds_config", None), "_stage", None)
+ if not isinstance(stage, int) or stage < 0:
+ stage = getattr(plugin, "zero_stage", None)
+ stages.append(stage)
+ env_stage = os.environ.get("ACCELERATE_DEEPSPEED_ZERO_STAGE", "").strip()
+ if env_stage.isdigit():
+ stages.append(int(env_stage))
+ stages = [stage for stage in stages if isinstance(stage, int) and stage >= 0]
+ if not stages or 3 in stages:
+ return torch.no_grad()
try:
first = next(iter(model.parameters()), None)
except Exception: