Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Optimizer parking outside inference mode; refuse fp16 params with unquantized adamw_flash moments (#14201)

Daniel Han143e41f43673
diff --git a/tests/test_grpo_park_optimizer_during_rollout.py b/tests/test_grpo_park_optimizer_during_rollout.py
index ebc09599bef..2b99006db98 100644
--- a/tests/test_grpo_park_optimizer_during_rollout.py
+++ b/tests/test_grpo_park_optimizer_during_rollout.py
@@ -49,7 +49,12 @@ def _load():
return torch.empty_like(tensor, **kwargs)
namespace["torch"] = types.SimpleNamespace(
- Tensor = torch.Tensor, empty_like = empty_like, cuda = cuda, strided = torch.strided
+ Tensor = torch.Tensor,
+ empty_like = empty_like,
+ cuda = cuda,
+ strided = torch.strided,
+ inference_mode = torch.inference_mode,
+ no_grad = torch.no_grad,
)
namespace["_is_parkable_tensor"] = lambda value: isinstance(value, torch.Tensor)
return fns, cuda
@@ -209,3 +214,38 @@ def test_nested_or_layerwise_optimizer_state_is_declined():
layerwise = types.SimpleNamespace(optimizer_dict = {}, param_groups = [], state = {})
trainer = types.SimpleNamespace(model = _Model(), optimizer = layerwise)
assert fns["_park_optimizer_to_host"](trainer) is None
+
+
+def test_park_under_outer_inference_mode_restores_ordinary_tensors(monkeypatch):
+ # An eval loop wrapped in torch.inference_mode() around the rollout must not turn the
+ # optimizer moments / grads into inference tensors, or the next optimizer.step() raises.
+ fns, _ = _load()
+ trainer, model, optimizer = _trainer_after_one_step()
+ state_before, grads_before = _snapshot(optimizer, model)
+
+ class Trainer:
+ def _generate_and_score_completions(self, inputs):
+ return inputs
+
+ fns["_wrap_grpo_generate_and_score"](Trainer)
+ instance = Trainer()
+ instance.__dict__.update(vars(trainer))
+ monkeypatch.setenv("UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT", "1")
+ with torch.inference_mode():
+ assert instance._generate_and_score_completions("x") == "x"
+ assert getattr(instance, "_unsloth_park_count", 0) == 1
+ for p, s in optimizer.state.items():
+ for k, v in s.items():
+ if isinstance(v, torch.Tensor):
+ assert not v.is_inference(), k
+ assert torch.equal(v, state_before[(id(p), k)])
+ for p in model.parameters():
+ assert not p.grad.is_inference()
+ assert torch.equal(p.grad, grads_before[id(p)])
+ for host in instance._unsloth_park_buffers.values():
+ assert not host.is_inference()
+ optimizer.step()
+ # A second rollout reuses the pinned buffers outside inference mode as well.
+ with torch.inference_mode():
+ instance._generate_and_score_completions("x")
+ optimizer.step()
diff --git a/tests/utils/test_flash_adamw_optim.py b/tests/utils/test_flash_adamw_optim.py
index 9fd9272362d..14040f03e9b 100644
--- a/tests/utils/test_flash_adamw_optim.py
+++ b/tests/utils/test_flash_adamw_optim.py
@@ -27,14 +27,40 @@ def stub_flashoptim(monkeypatch):
betas = (0.9, 0.999),
eps = 1e-8,
weight_decay = 1e-2,
+ decouple_lr = False,
+ quantize = True,
+ compress_state_dict = True,
master_weight_bits = 24,
**kwargs,
):
calls.append(
dict(lr = lr, betas = betas, eps = eps, master_weight_bits = master_weight_bits, **kwargs)
)
- # flashoptim's own defaults carry initial_lr = the constructor lr.
- super().__init__(params, dict(lr = lr, initial_lr = lr, weight_decay = weight_decay))
+ if quantize is not True:
+ calls[-1]["quantize"] = quantize
+ if compress_state_dict is not True:
+ calls[-1]["compress_state_dict"] = compress_state_dict
+ # flashoptim 0.1.4's own defaults carry initial_lr = the constructor lr and quantize,
+ # and a group without the key falls back to self._quantize.
+ self._quantize = quantize
+ super().__init__(
+ params, dict(lr = lr, initial_lr = lr, weight_decay = weight_decay, quantize = quantize)
+ )
+
+ # flashoptim: step() calls step_param() for every parameter with a gradient, and
+ # gradient release calls step_param() directly.
+ def step(self, closure = None):
+ for group in self.param_groups:
+ for p in group["params"]:
+ if p.grad is not None:
+ self.step_param(p, group)
+
+ def step_param(
+ self,
+ p,
+ group = None,
+ ):
+ return None
module = types.ModuleType("flashoptim")
module.FlashAdamW = FlashAdamW
@@ -404,3 +430,90 @@ def test_adamw_flash_requires_compatible_flashoptim(stub_flashoptim, monkeypatch
monkeypatch.setattr(sys.modules["flashoptim"], "__version__", "0.1.1")
with pytest.raises(ImportError, match = "flashoptim>=0.1.4, found 0.1.1"):
flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args())
+
+
+def test_adamw_flash_rejects_fp16_params_with_unquantized_moments(stub_flashoptim, monkeypatch):
+ # flashoptim stores unquantized moments in the param dtype: an fp16 variance underflows to 0
+ # for |grad| < ~2e-4, and the next step divides the momentum by eps alone.
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ fp16 = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))
+ fp32 = torch.nn.Parameter(torch.zeros(4))
+ bf16 = torch.nn.Parameter(torch.zeros(4, dtype = torch.bfloat16))
+ for optim_args in (None, "quantize=False", "master_weight_bits=none"):
+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(
+ _flash_kwargs_args(optim_args = optim_args)
+ )
+ with pytest.raises(ValueError, match = "float16"):
+ optimizer_cls([{"params": [fp32, fp16]}], **optimizer_kwargs)
+ # bf16 / fp32 keep working with unquantized moments.
+ optimizer_cls([{"params": [fp32, bf16]}], **optimizer_kwargs)
+ # quantize=True stores the variance as sqrt with its own scales, so fp16 is accepted.
+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(
+ _flash_kwargs_args(optim_args = "quantize=True")
+ )
+ optimizer_cls([{"params": [fp32, fp16]}], **optimizer_kwargs)
+
+
+def test_adamw_flash_fp16_check_covers_positional_late_and_unfrozen_groups(
+ stub_flashoptim, monkeypatch
+):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ optimizer_cls, _ = flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args())
+ fp32 = torch.nn.Parameter(torch.zeros(4))
+ fp16 = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))
+ # quantize passed positionally (lr, betas, eps, weight_decay, decouple_lr, quantize).
+ with pytest.raises(ValueError, match = "float16"):
+ optimizer_cls([fp32, fp16], 1e-3, (0.9, 0.999), 1e-8, 0.0, False, False)
+ # A group added after construction (progressive unfreezing) is checked too.
+ optimizer = optimizer_cls([fp32], quantize = False)
+ with pytest.raises(ValueError, match = "float16"):
+ optimizer.add_param_group({"params": [fp16]})
+ # A frozen fp16 base (fp32 LoRA on top) is fine until it is unfrozen and gets a gradient.
+ frozen = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16), requires_grad = False)
+ optimizer = optimizer_cls([fp32, frozen], quantize = False)
+ fp32.grad = torch.ones(4)
+ optimizer.step()
+ frozen.requires_grad_(True)
+ frozen.grad = torch.ones(4, dtype = torch.float16)
+ with pytest.raises(ValueError, match = "float16"):
+ optimizer.step()
+ # Gradient release updates through step_param directly, without step().
+ with pytest.raises(ValueError, match = "float16"):
+ optimizer.step_param(frozen)
+ # Frozen again after backward: .grad stays attached, but flashoptim skips the param.
+ frozen.requires_grad_(False)
+ optimizer.step()
+ optimizer.step_param(frozen)
+ # Resuming a checkpoint whose group had quantize=False switches an accepted fp16 group.
+ trainable = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))
+ optimizer = optimizer_cls([trainable], quantize = True)
+ saved = optimizer.state_dict()
+ saved["param_groups"][0]["quantize"] = False
+ optimizer.load_state_dict(saved)
+ trainable.grad = torch.ones(4, dtype = torch.float16)
+ with pytest.raises(ValueError, match = "float16"):
+ optimizer.step()
+ # A vanilla AdamW state dict has no quantize key: the optimizer-wide quantize=False applies.
+ trainable = torch.nn.Parameter(torch.zeros(4, dtype = torch.float16))
+ optimizer = optimizer_cls([trainable], quantize = True)
+ saved = optimizer.state_dict()
+ saved["param_groups"][0].pop("quantize")
+ optimizer = optimizer_cls([trainable], quantize = True)
+ optimizer._quantize = False
+ optimizer.load_state_dict(saved)
+ trainable.grad = torch.ones(4, dtype = torch.float16)
+ with pytest.raises(ValueError, match = "float16"):
+ optimizer.step()
+ # A refused group is not kept, so the caller can add the params again with quantize=True.
+ optimizer = optimizer_cls([fp32], quantize = False)
+ with pytest.raises(ValueError, match = "float16"):
+ optimizer.add_param_group({"params": [fp16]})
+ assert len(optimizer.param_groups) == 1
+ optimizer.add_param_group({"params": [fp16], "quantize": True})
+ assert len(optimizer.param_groups) == 2
+ fp16.grad = torch.ones(4, dtype = torch.float16)
+ optimizer.step()
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index bc02971a6a8..6f9e194ec16 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -986,17 +986,29 @@ def _wrap_grpo_generate_and_score(trainer_cls):
def wrapped(self, *args, **kwargs):
was_training = getattr(getattr(self, "model", None), "training", None)
parked = None
- if os.environ.get(
+ park = os.environ.get(
"UNSLOTH_PARK_OPTIMIZER_DURING_ROLLOUT", "0"
- ) == "1" and _park_optimizer_enabled(self):
- parked = _park_optimizer_to_host(self)
+ ) == "1" and _park_optimizer_enabled(self)
+ # An outer torch.inference_mode() (eval between train calls) would make the pinned buffers and
+ # restored state inference tensors, which the next optimizer step cannot update in place.
+ inference_mode = getattr(torch, "inference_mode", None) if park else None
+ if park:
+ if inference_mode is not None:
+ with inference_mode(False), torch.no_grad():
+ parked = _park_optimizer_to_host(self)
+ else:
+ parked = _park_optimizer_to_host(self)
if parked is not None:
self._unsloth_park_count = getattr(self, "_unsloth_park_count", 0) + 1
try:
return original(self, *args, **kwargs)
finally:
if parked is not None:
- _restore_optimizer_from_host(parked)
+ if inference_mode is not None:
+ with inference_mode(False), torch.no_grad():
+ _restore_optimizer_from_host(parked)
+ else:
+ _restore_optimizer_from_host(parked)
if (
was_training is False
and hasattr(self, "model")
diff --git a/unsloth/optimizers/flash_adamw.py b/unsloth/optimizers/flash_adamw.py
index a215dee4845..9d4fc876803 100644
--- a/unsloth/optimizers/flash_adamw.py
+++ b/unsloth/optimizers/flash_adamw.py
@@ -11,7 +11,8 @@ Qwen2.5-0.5B full finetune. optim_args = "quantize=True" stores the moments in 8
(about 8 bytes per parameter; with "master_weight_bits=24", flashoptim's own default,
about 7), but its loss curve moved measurably away from AdamW there.
fp32 parameters (LoRA adapters) have no correction term; for them only quantize=True
-saves memory.
+saves memory. Trainable fp16 parameters need quantize=True: unquantized fp16 moments
+underflow, so the optimizer refuses them.
transformers validates `optim` against OptimizerNames, so the config stores
`adamw_torch` and marks itself with `_unsloth_flash_adamw` plus a token in `optim_args`
@@ -146,10 +147,60 @@ def _get_flash_adamw_class():
)
super().__init__(params, *args, master_weight_bits = master_weight_bits, **kwargs)
+ # flashoptim stores unquantized moments in the param dtype, so an fp16 variance
+ # (grad**2 * (1 - beta2)) underflows to 0 for small grads and the update becomes
+ # momentum / eps. quantize=True keeps the variance as sqrt with per-group scales.
+ # Checked per group after flashoptim fills its defaults (so positional and per-group
+ # quantize both count), for groups added at construction and by add_param_group later.
+ def _group_quantizes(self, group):
+ # flashoptim resolves a group without the key (a vanilla AdamW state dict) to the
+ # optimizer-wide setting.
+ return group.get("quantize", getattr(self, "_quantize", True))
+
+ def add_param_group(self, param_group):
+ super().add_param_group(param_group)
+ group = self.param_groups[-1]
+ if not self._group_quantizes(group) and any(
+ p.dtype == torch.float16 and p.requires_grad for p in group["params"]
+ ):
+ # Not kept: a caller that catches this can add the params again with quantize=True.
+ self.param_groups.pop()
+ self.__dict__.pop("quantized_state_spec", None)
+ _refuse_unquantized_fp16()
+
+ # Every update goes through step_param: step() loops over it and gradient release calls
+ # it per parameter. Checked there against the group's current quantize (load_state_dict
+ # can restore an older group's quantize=False), so a frozen fp16 param unfrozen later is
+ # refused before it gets moments. A param frozen after backward keeps its .grad, but
+ # flashoptim skips it (requires_grad False), so it is not refused. Only fp16 params pay
+ # for the group lookup.
+ def step_param(
+ self,
+ p,
+ group = None,
+ ):
+ if p.dtype == torch.float16 and p.requires_grad and p.grad is not None:
+ if group is None:
+ group = next(
+ (g for g in self.param_groups if any(q is p for q in g["params"])), None
+ )
+ if group is not None and not self._group_quantizes(group):
+ _refuse_unquantized_fp16()
+ return super().step_param(p, group)
+
_FLASH_ADAMW_CLASS = UnslothFlashAdamW
return _FLASH_ADAMW_CLASS
+def _refuse_unquantized_fp16():
+ raise ValueError(
+ "Unsloth: optim = 'adamw_flash' with quantize=False stores the Adam moments in "
+ "the parameter dtype, and float16 moments underflow (wrong updates). Found "
+ "trainable float16 parameters: cast them to float32 or bfloat16, or set "
+ "optim_args = 'quantize=True'."
+ )
+
+
def _materialize_params(params):
# A one-shot iterable (model.parameters()) would be exhausted by the dtype scan below.
return [params] if isinstance(params, torch.Tensor) else list(params)