Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Optimizer parking outside inference mode; refuse fp16 params with unquantized adamw_flash moments (#14201)
Daniel Han143e41f43673
diff --git a/tests/test_grpo_park_optimizer_during_rollout.py b/tests/test_grpo_park_optimizer_during_rollout.pyindex ebc09599bef..2b99006db98 100644--- a/tests/test_grpo_park_optimizer_during_rollout.py+++ b/tests/test_grpo_park_optimizer_during_rollout.py@@ -49,7 +49,12 @@ def _load():return torch.empty_like(tensor, **kwargs)namespace["torch"] = types.SimpleNamespace(- Tensor = torch.Tensor, empty_like = empty_like, cuda = cuda, strided = torch.strided+ Tensor = torch.Tensor,+ empty_like = empty_like,+ cuda = cuda,+ strided = torch.strided,+ inference_mode = torch.inference_mode,+ no_grad = torch.no_grad,)namespace["_is_parkable_tensor"] = lambda value: isinstance(value, torch.Tensor)return fns, cuda@@ -209,3 +214,38 @@ def test_nested_or_layerwise_optimizer_state_is_declined():layerwise = types.SimpleNamespace(optimizer_dict = {}, param_groups = [], state = {})trainer = types.SimpleNamespace(model = _Model(), optimizer = layerwise)assert fns["_park_optimizer_to_host"](trainer) is None+++def test_park_under_outer_inference_mode_restores_ordinary_tensors(monkeypatch):+ # An eval loop wrapped in torch.inference_mode() around the rollout must not turn the+ # optimizer moments / grads into inference tensors, or the next optimizer.step() raises.+ fns, _ = _load()+ trainer, model, optimizer = _trainer_after_one_step()+ state_before, grads_before = _snapshot(optimizer, model)++ class Trainer:+ def _generate_and_score_completions(self, inputs):+ return inputs++ fns["_wrap_grpo_generate_and_score"](Trainer)+ instance = Trainer()+ instance.__dict__.update(vars(trainer))+ monkeypatch.setenv("UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT", "1")+ with torch.inference_mode():+ assert instance._generate_and_score_completions("x") == "x"+ assert getattr(instance, "_unsloth_park_count", 0) == 1+ for p, s in optimizer.state.items():+ for k, v in s.items():+ if isinstance(v, torch.Tensor):+ assert not v.is_inference(), k+ assert torch.equal(v, state_before[(id(p), k)])+ for p in model.parameters():+ assert not p.grad.is_inference()+ assert torch.equal(p.grad, grads_before[id(p)])+ for host in instance._unsloth_park_buffers.values():+ assert not host.is_inference()+ optimizer.step()+ # A second rollout reuses the pinned buffers outside inference mode as well.+ with torch.inference_mode():+ instance._generate_and_score_completions("x")+ optimizer.step()diff --git a/tests/utils/test_flash_adamw_optim.py b/tests/utils/test_flash_adamw_optim.pyindex 9fd9272362d..14040f03e9b 100644--- a/tests/utils/test_flash_adamw_optim.py+++ b/tests/utils/test_flash_adamw_optim.py@@ -27,14 +27,40 @@ def stub_flashoptim(monkeypatch):betas = (0.9, 0.999),eps = 1e-8,weight_decay = 1e-2,+ decouple_lr = False,+ quantize = True,+ compress_state_dict = True,master_weight_bits = 24,**kwargs,):calls.append(dict(lr = lr, betas = betas, eps = eps, master_weight_bits = master_weight_bits, **kwargs))- # flashoptim's own defaults carry initial_lr = the constructor lr.- super().__init__(params, dict(lr = lr, initial_lr = lr, weight_decay = weight_decay))+ if quantize is not True:+ calls[-1]["quantize"] = quantize+ if compress_state_dict is not True:+ calls[-1]["compress_state_dict"] = compress_state_dict+ # flashoptim 0.1.4's own defaults carry initial_lr = the constructor lr and quantize,+ # and a group without the key falls back to self._quantize.+ self._quantize = quantize+ super().__init__(+ params, dict(lr = lr, initial_lr = lr, weight_decay = weight_decay, quantize = quantize)+ )++ # flashoptim: step() calls step_param() for every parameter with a gradient, and+ # gradient release calls step_param() directly.+ def step(self, closure = None):+ for group in self.param_groups:+ for p in group["params"]:+ if p.grad is not None:+ self.step_param(p, group)++ def step_param(+ self,+ p,+ group = None,+ ):+ return Nonemodule = types.ModuleType("flashoptim")module.FlashAdamW = FlashAdamW@@ -404,3 +430,90 @@ def test_adamw_flash_requires_compatible_flashoptim(stub_flashoptim, monkeypatchmonkeypatch.setattr(sys.modules["flashoptim"], "__version__", "0.1.1")with pytest.raises(ImportError, match = "flashoptim>=0.1.4, found 0.1.1"):flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args())+++def test_adamw_flash_rejects_fp16_params_with_unquantized_moments(stub_flashoptim, monkeypatch):+ # flashoptim stores unquantized moments in the param dtype: an fp16 variance underflows to 0+ # for |grad| < ~2e-4, and the next step divides the momentum by eps alone.+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ fp16 = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))+ fp32 = torch.nn.Parameter(torch.zeros(4))+ bf16 = torch.nn.Parameter(torch.zeros(4, dtype = torch.bfloat16))+ for optim_args in (None, "quantize=False", "master_weight_bits=none"):+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(+ _flash_kwargs_args(optim_args = optim_args)+ )+ with pytest.raises(ValueError, match = "float16"):+ optimizer_cls([{"params": [fp32, fp16]}], **optimizer_kwargs)+ # bf16 / fp32 keep working with unquantized moments.+ optimizer_cls([{"params": [fp32, bf16]}], **optimizer_kwargs)+ # quantize=True stores the variance as sqrt with its own scales, so fp16 is accepted.+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(+ _flash_kwargs_args(optim_args = "quantize=True")+ )+ optimizer_cls([{"params": [fp32, fp16]}], **optimizer_kwargs)+++def test_adamw_flash_fp16_check_covers_positional_late_and_unfrozen_groups(+ stub_flashoptim, monkeypatch+):+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ optimizer_cls, _ = flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args())+ fp32 = torch.nn.Parameter(torch.zeros(4))+ fp16 = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))+ # quantize passed positionally (lr, betas, eps, weight_decay, decouple_lr, quantize).+ with pytest.raises(ValueError, match = "float16"):+ optimizer_cls([fp32, fp16], 1e-3, (0.9, 0.999), 1e-8, 0.0, False, False)+ # A group added after construction (progressive unfreezing) is checked too.+ optimizer = optimizer_cls([fp32], quantize = False)+ with pytest.raises(ValueError, match = "float16"):+ optimizer.add_param_group({"params": [fp16]})+ # A frozen fp16 base (fp32 LoRA on top) is fine until it is unfrozen and gets a gradient.+ frozen = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16), requires_grad = False)+ optimizer = optimizer_cls([fp32, frozen], quantize = False)+ fp32.grad = torch.ones(4)+ optimizer.step()+ frozen.requires_grad_(True)+ frozen.grad = torch.ones(4, dtype = torch.float16)+ with pytest.raises(ValueError, match = "float16"):+ optimizer.step()+ # Gradient release updates through step_param directly, without step().+ with pytest.raises(ValueError, match = "float16"):+ optimizer.step_param(frozen)+ # Frozen again after backward: .grad stays attached, but flashoptim skips the param.+ frozen.requires_grad_(False)+ optimizer.step()+ optimizer.step_param(frozen)+ # Resuming a checkpoint whose group had quantize=False switches an accepted fp16 group.+ trainable = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))+ optimizer = optimizer_cls([trainable], quantize = True)+ saved = optimizer.state_dict()+ saved["param_groups"][0]["quantize"] = False+ optimizer.load_state_dict(saved)+ trainable.grad = torch.ones(4, dtype = torch.float16)+ with pytest.raises(ValueError, match = "float16"):+ optimizer.step()+ # A vanilla AdamW state dict has no quantize key: the optimizer-wide quantize=False applies.+ trainable = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))+ optimizer = optimizer_cls([trainable], quantize = True)+ saved = optimizer.state_dict()+ saved["param_groups"][0].pop("quantize")+ optimizer = optimizer_cls([trainable], quantize = True)+ optimizer._quantize = False+ optimizer.load_state_dict(saved)+ trainable.grad = torch.ones(4, dtype = torch.float16)+ with pytest.raises(ValueError, match = "float16"):+ optimizer.step()+ # A refused group is not kept, so the caller can add the params again with quantize=True.+ optimizer = optimizer_cls([fp32], quantize = False)+ with pytest.raises(ValueError, match = "float16"):+ optimizer.add_param_group({"params": [fp16]})+ assert len(optimizer.param_groups) == 1+ optimizer.add_param_group({"params": [fp16], "quantize": True})+ assert len(optimizer.param_groups) == 2+ fp16.grad = torch.ones(4, dtype = torch.float16)+ optimizer.step()diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex bc02971a6a8..6f9e194ec16 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -986,17 +986,29 @@ def _wrap_grpo_generate_and_score(trainer_cls):def wrapped(self, *args, **kwargs):was_training = getattr(getattr(self, "model", None), "training", None)parked = None- if os.environ.get(+ park = os.environ.get("UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT", "0"- ) == "1" and _park_optimizer_enabled(self):- parked = _park_optimizer_to_host(self)+ ) == "1" and _park_optimizer_enabled(self)+ # An outer torch.inference_mode() (eval between train calls) would make the pinned buffers and+ # restored state inference tensors, which the next optimizer step cannot update in place.+ inference_mode = getattr(torch, "inference_mode", None) if park else None+ if park:+ if inference_mode is not None:+ with inference_mode(False), torch.no_grad():+ parked = _park_optimizer_to_host(self)+ else:+ parked = _park_optimizer_to_host(self)if parked is not None:self._unsloth_park_count = getattr(self, "_unsloth_park_count", 0) + 1try:return original(self, *args, **kwargs)finally:if parked is not None:- _restore_optimizer_from_host(parked)+ if inference_mode is not None:+ with inference_mode(False), torch.no_grad():+ _restore_optimizer_from_host(parked)+ else:+ _restore_optimizer_from_host(parked)if (was_training is Falseand hasattr(self, "model")diff --git a/unsloth/optimizers/flash_adamw.py b/unsloth/optimizers/flash_adamw.pyindex a215dee4845..9d4fc876803 100644--- a/unsloth/optimizers/flash_adamw.py+++ b/unsloth/optimizers/flash_adamw.py@@ -11,7 +11,8 @@ Qwen2.5-0.5B full finetune. optim_args = "quantize=True" stores the moments in 8(about 8 bytes per parameter; with "master_weight_bits=24", flashoptim's own default,about 7), but its loss curve moved measurably away from AdamW there.fp32 parameters (LoRA adapters) have no correction term; for them only quantize=True-saves memory.+saves memory. Trainable fp16 parameters need quantize=True: unquantized fp16 moments+underflow, so the optimizer refuses them.transformers validates `optim` against OptimizerNames, so the config stores`adamw_torch` and marks itself with `_unsloth_flash_adamw` plus a token in `optim_args`@@ -146,10 +147,60 @@ def _get_flash_adamw_class():)super().__init__(params, *args, master_weight_bits = master_weight_bits, **kwargs)+ # flashoptim stores unquantized moments in the param dtype, so an fp16 variance+ # (grad**2 * (1 - beta2)) underflows to 0 for small grads and the update becomes+ # momentum / eps. quantize=True keeps the variance as sqrt with per-group scales.+ # Checked per group after flashoptim fills its defaults (so positional and per-group+ # quantize both count), for groups added at construction and by add_param_group later.+ def _group_quantizes(self, group):+ # flashoptim resolves a group without the key (a vanilla AdamW state dict) to the+ # optimizer-wide setting.+ return group.get("quantize", getattr(self, "_quantize", True))++ def add_param_group(self, param_group):+ super().add_param_group(param_group)+ group = self.param_groups[-1]+ if not self._group_quantizes(group) and any(+ p.dtype == torch.float16 and p.requires_grad for p in group["params"]+ ):+ # Not kept: a caller that catches this can add the params again with quantize=True.+ self.param_groups.pop()+ self.__dict__.pop("quantized_state_spec", None)+ _refuse_unquantized_fp16()++ # Every update goes through step_param: step() loops over it and gradient release calls+ # it per parameter. Checked there against the group's current quantize (load_state_dict+ # can restore an older group's quantize=False), so a frozen fp16 param unfrozen later is+ # refused before it gets moments. A param frozen after backward keeps its .grad, but+ # flashoptim skips it (requires_grad False), so it is not refused. Only fp16 params pay+ # for the group lookup.+ def step_param(+ self,+ p,+ group = None,+ ):+ if p.dtype == torch.float16 and p.requires_grad and p.grad is not None:+ if group is None:+ group = next(+ (g for g in self.param_groups if any(q is p for q in g["params"])), None+ )+ if group is not None and not self._group_quantizes(group):+ _refuse_unquantized_fp16()+ return super().step_param(p, group)+_FLASH_ADAMW_CLASS = UnslothFlashAdamWreturn _FLASH_ADAMW_CLASS+def _refuse_unquantized_fp16():+ raise ValueError(+ "Unsloth: optim = 'adamw_flash' with quantize=False stores the Adam moments in "+ "the parameter dtype, and float16 moments underflow (wrong updates). Found "+ "trainable float16 parameters: cast them to float32 or bfloat16, or set "+ "optim_args = 'quantize=True'."+ )++def _materialize_params(params):# A one-shot iterable (model.parameters()) would be exhausted by the dtype scan below.return [params] if isinstance(params, torch.Tensor) else list(params)