Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Generation keeps inference_mode on DeepSpeed ZeRO-1/2; only ZeRO-3 needs no_grad (#14673)
Daniel Hand4a92bb52d14
diff --git a/tests/test_fsdp_generation_grad_mode.py b/tests/test_fsdp_generation_grad_mode.pyindex b8cce6f0e35..d98b01059b6 100644--- a/tests/test_fsdp_generation_grad_mode.py+++ b/tests/test_fsdp_generation_grad_mode.py@@ -10,7 +10,7 @@ import torchfrom unsloth.models._utils import _get_inference_mode_context_manager-_FSDP_ENV = ("ACCELERATE_USE_FSDP", "FSDP_VERSION")+_FSDP_ENV = ("ACCELERATE_USE_FSDP", "FSDP_VERSION", "ACCELERATE_DEEPSPEED_ZERO_STAGE")@pytest.fixture@@ -117,3 +117,76 @@ def test_deepspeed_accelerator_state_scores_under_no_grad(no_launcher):state.AcceleratorState, "_shared_state", {"distributed_type": "DistributedType.DEEPSPEED"})assert _mode_inside(torch.nn.Linear(2, 2)) == (False, False)++[email protected]("stage, inference", [("0", True), ("1", True), ("2", True), ("3", False)])+def test_a_known_zero_stage_below_3_keeps_inference_mode(no_launcher, stage, inference):+ """ZeRO-1/2 keep whole parameters, so only ZeRO-3 needs no_grad for generation."""+ no_launcher.setenv("ACCELERATE_USE_DEEPSPEED", "true")+ no_launcher.setenv("ACCELERATE_DEEPSPEED_ZERO_STAGE", stage)+ assert _mode_inside(torch.nn.Linear(2, 2)) == (inference, False)++[email protected]("config_stage, inference", [(2, True), (3, False), (-1, True)])+def test_the_accelerator_plugin_stage_decides(no_launcher, config_stage, inference):+ import types++ state = pytest.importorskip("accelerate.state")+ no_launcher.delenv("ACCELERATE_USE_DEEPSPEED", raising = False)+ plugin = types.SimpleNamespace(+ zero_stage = 2, hf_ds_config = types.SimpleNamespace(_stage = config_stage)+ )+ no_launcher.setattr(+ state.AcceleratorState,+ "_shared_state",+ {"distributed_type": "DistributedType.DEEPSPEED", "deepspeed_plugins": {"default": plugin}},+ )+ assert _mode_inside(torch.nn.Linear(2, 2)) == (inference, False)+++def test_a_zero2_launch_still_honours_a_zero3_trainer_config(no_launcher):+ hf_deepspeed = pytest.importorskip("transformers.integrations.deepspeed")+ no_launcher.setenv("ACCELERATE_USE_DEEPSPEED", "true")+ no_launcher.setenv("ACCELERATE_DEEPSPEED_ZERO_STAGE", "2")+ no_launcher.setattr(hf_deepspeed, "is_deepspeed_zero3_enabled", lambda: True)+ assert _mode_inside(torch.nn.Linear(2, 2)) == (False, False)+++def test_a_zero2_launch_with_partitioned_params_still_gets_no_grad(no_launcher):+ no_launcher.setenv("ACCELERATE_USE_DEEPSPEED", "true")+ no_launcher.setenv("ACCELERATE_DEEPSPEED_ZERO_STAGE", "2")+ model = torch.nn.Linear(2, 2)+ model.weight.ds_id = 0+ assert _mode_inside(model) == (False, False)+++def test_only_the_selected_deepspeed_plugin_counts(no_launcher):+ import types++ state = pytest.importorskip("accelerate.state")+ no_launcher.delenv("ACCELERATE_USE_DEEPSPEED", raising = False)+ active = types.SimpleNamespace(zero_stage = 2, selected = True, hf_ds_config = None)+ inactive = types.SimpleNamespace(zero_stage = 3, selected = False, hf_ds_config = None)+ no_launcher.setattr(+ state.AcceleratorState,+ "_shared_state",+ {+ "distributed_type": "DistributedType.DEEPSPEED",+ "deepspeed_plugins": {"a": active, "b": inactive},+ },+ )+ assert _mode_inside(torch.nn.Linear(2, 2)) == (True, False)+++def test_the_grpo_preamble_copy_is_self_contained():+ # rl_replacements appends only this function's source to the generated GRPO trainer.+ import ast, inspect+ from unsloth.models._utils import _get_inference_mode_context_manager as fn++ tree = ast.parse(inspect.getsource(fn))+ called = {+ n.func.id+ for n in ast.walk(tree)+ if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)+ }+ assert "_deepspeed_zero_stage" not in calleddiff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 3e9d8720d69..aa56b77e1da 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -9137,13 +9137,42 @@ def _get_inference_mode_context_manager(model: torch.nn.Module):)if isinstance(shared_state, dict) and "FSDP" in str(shared_state.get("distributed_type", "")):return torch.no_grad()- # ZeRO-3 keeps small persistent gathered params, so the same poisoning applies.- if os.environ.get("ACCELERATE_USE_DEEPSPEED", "").strip().lower() in ("1", "true", "yes", "on"):- return torch.no_grad()- if isinstance(shared_state, dict) and "DEEPSPEED" in str(- shared_state.get("distributed_type", "")- ):- return torch.no_grad()+ # ZeRO-3 reuses gathered persistent params (same poisoning); a known stage below 3 does not.+ deepspeed = os.environ.get("ACCELERATE_USE_DEEPSPEED", "").strip().lower() in (+ "1",+ "true",+ "yes",+ "on",+ ) or (+ isinstance(shared_state, dict)+ and "DEEPSPEED" in str(shared_state.get("distributed_type", ""))+ )+ if deepspeed:+ # Inline, not a helper: rl_replacements copies only this function into the GRPO trainer.+ stages = []+ hf_deepspeed = sys.modules.get("transformers.integrations.deepspeed", None)+ try:+ if hf_deepspeed is not None and hf_deepspeed.is_deepspeed_zero3_enabled():+ stages.append(3)+ except Exception:+ pass+ plugins = shared_state.get("deepspeed_plugins") if isinstance(shared_state, dict) else None+ if plugins is not None and not isinstance(plugins, dict):+ plugins = {"default": plugins}+ for plugin in (plugins or {}).values():+ if not getattr(plugin, "selected", True):+ continue+ # A config file's stage lives in hf_ds_config; zero_stage defaults to 2 regardless.+ stage = getattr(getattr(plugin, "hf_ds_config", None), "_stage", None)+ if not isinstance(stage, int) or stage < 0:+ stage = getattr(plugin, "zero_stage", None)+ stages.append(stage)+ env_stage = os.environ.get("ACCELERATE_DEEPSPEED_ZERO_STAGE", "").strip()+ if env_stage.isdigit():+ stages.append(int(env_stage))+ stages = [stage for stage in stages if isinstance(stage, int) and stage >= 0]+ if not stages or 3 in stages:+ return torch.no_grad()try:first = next(iter(model.parameters()), None)except Exception: