Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Run each FlashAdamW update on its parameter's CUDA device (#14773)
Daniel Han9d719c0b0202
diff --git a/tests/utils/test_flash_adamw_device.py b/tests/utils/test_flash_adamw_device.pynew file mode 100644index 00000000000..5c7919ffa0e--- /dev/null+++ b/tests/utils/test_flash_adamw_device.py@@ -0,0 +1,111 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Real FlashOptim on two GPUs: an update of a parameter on a non-current device (device_map+split model) must match the same update run with that device current, bit for bit."""++import pytest++import unsloth # noqa: F401 (must precede transformers/trl)+import torch++pytestmark = [+ pytest.mark.gpu,+ pytest.mark.skipif(+ not torch.cuda.is_available() or torch.cuda.device_count() < 2,+ reason = "requires two visible CUDA GPUs",+ ),+]+++ "dtype,quantize",+ [+ (torch.float32, False),+ (torch.bfloat16, False),+ (torch.bfloat16, True),+ (torch.float16, True),+ ],+)[email protected]("entry", ["step", "step_param"])+def test_adamw_flash_noncurrent_cuda_device(dtype, quantize, entry):+ pytest.importorskip("flashoptim")+ from unsloth.optimizers.flash_adamw import _get_flash_adamw_class++ torch.cuda.set_device(0)+ optimizer_cls = _get_flash_adamw_class()+ p0 = torch.nn.Parameter(torch.ones(33, device = "cuda:0", dtype = dtype))+ p1 = torch.nn.Parameter(torch.ones(33, device = "cuda:1", dtype = dtype))+ q0 = torch.nn.Parameter(p0.detach().clone())+ q1 = torch.nn.Parameter(p1.detach().clone())+ optimizer = optimizer_cls([p0, p1], quantize = quantize, lr = 0.001)+ reference = optimizer_cls([q0, q1], quantize = quantize, lr = 0.001)+ for grad in (0.125, -0.25, 0.0, 0.0625):+ for p in (p0, p1, q0, q1):+ p.grad = torch.full_like(p, grad)+ for q in (q0, q1):+ with torch.cuda.device(q.device):+ reference.step_param(q)+ if entry == "step":+ optimizer.step()+ else:+ for p in (p0, p1):+ optimizer.step_param(p)+ assert torch.cuda.current_device() == 0, "the caller's device must be restored"+ for p, q in ((p0, q0), (p1, q1)):+ torch.cuda.synchronize(p.device)+ torch.testing.assert_close(p, q, rtol = 0, atol = 0)+ a, b = optimizer.state[p], reference.state[q]+ assert a["step"].item() == b["step"].item()+ for key in ("exp_avg", "exp_avg_sq"):+ torch.testing.assert_close(+ a[key].kernel_tensor, b[key].kernel_tensor, rtol = 0, atol = 0+ )+ torch.testing.assert_close(+ a[key].kernel_scales_or_self, b[key].kernel_scales_or_self, rtol = 0, atol = 0+ )+ if "error_bits" in a:+ torch.testing.assert_close(a["error_bits"], b["error_bits"], rtol = 0, atol = 0)++[email protected]("dtype", [torch.float32, torch.bfloat16])+def test_adamw_flash_uncompressed_state_dict_round_trip_noncurrent_device(dtype):+ """compress_state_dict=False dequantizes the moments on save and requantizes them on load,+ both with Triton: a parameter on cuda:1 with cuda:0 current must save and resume."""+ pytest.importorskip("flashoptim")+ from unsloth.optimizers.flash_adamw import _get_flash_adamw_class++ torch.cuda.set_device(0)+ optimizer_cls = _get_flash_adamw_class()+ kwargs = dict(quantize = True, compress_state_dict = False, lr = 0.001)+ params = [torch.nn.Parameter(torch.ones(65, device = f"cuda:{i}", dtype = dtype)) for i in (0, 1)]+ optimizer = optimizer_cls(params, **kwargs)+ for grad in (0.125, -0.25):+ for p in params:+ p.grad = torch.full_like(p, grad)+ optimizer.step()+ expected = {}+ for i, p in enumerate(params):+ with torch.cuda.device(p.device):+ expected[i] = {+ k: optimizer.state[p][k].materialize() for k in ("exp_avg", "exp_avg_sq")+ }++ saved = optimizer.state_dict()+ assert torch.cuda.current_device() == 0, "the caller's device must be restored"+ for i, p in enumerate(params):+ for key, value in expected[i].items():+ assert saved["state"][i][key].device == p.device+ torch.testing.assert_close(saved["state"][i][key], value, rtol = 0, atol = 0)++ resumed = optimizer_cls(params, **kwargs)+ resumed.load_state_dict(saved)+ assert torch.cuda.current_device() == 0, "the caller's device must be restored"+ for i, p in enumerate(params):+ for key, value in expected[i].items():+ with torch.cuda.device(p.device):+ loaded = resumed.state[p][key].materialize()+ assert loaded.device == p.device+ torch.testing.assert_close(loaded.float(), value.float(), rtol = 0.02, atol = 1e-6)+ p.grad = torch.full_like(p, 0.0625)+ resumed.step()diff --git a/tests/utils/test_flash_adamw_optim.py b/tests/utils/test_flash_adamw_optim.pyindex 54159aa6e9c..af3c791a588 100644--- a/tests/utils/test_flash_adamw_optim.py+++ b/tests/utils/test_flash_adamw_optim.py@@ -597,3 +597,53 @@ def test_adamw_flash_fused_bf16_variance_follows_fp32(monkeypatch, gradient):assert var.dtype == torch.float32# fp32 rounding differs between the kernel and the torch EMA (about 2e-5 after 2000 steps).torch.testing.assert_close(var, ema, rtol = 1e-4, atol = 1e-12)+++def test_adamw_flash_updates_each_param_on_its_own_device(stub_flashoptim, monkeypatch):+ """A device_map split model: each CUDA update must run with its parameter's device current."""+ import contextlib+ import unsloth.optimizers.flash_adamw as flash_adamw++ FlashAdamW, _ = stub_flashoptim+ current = ["cuda:0"]+ entered = []+ updated = []++ @contextlib.contextmanager+ def fake_device(device):+ entered.append(str(device))+ previous, current[0] = current[0], str(device)+ try:+ yield+ finally:+ current[0] = previous++ def record_step_param(+ self,+ p,+ group = None,+ ):+ updated.append((labels[id(p)], current[0]))++ monkeypatch.setattr(torch.cuda, "device", fake_device)+ monkeypatch.setattr(FlashAdamW, "step_param", record_step_param)+ optimizer_cls = flash_adamw._get_flash_adamw_class()+ cpu_param = torch.nn.Parameter(torch.zeros(4))+ optimizer = optimizer_cls([cpu_param])+ labels = {id(cpu_param): "cpu"}++ # Stand-ins for parameters placed on two GPUs: only device, dtype and is_cuda are read.+ on_gpu0 = types.SimpleNamespace(+ device = torch.device("cuda", 0), is_cuda = True, dtype = torch.float32+ )+ on_gpu1 = types.SimpleNamespace(+ device = torch.device("cuda", 1), is_cuda = True, dtype = torch.float32+ )+ labels.update({id(on_gpu0): "p0", id(on_gpu1): "p1"})+ for p in (on_gpu1, on_gpu0, on_gpu1):+ optimizer.step_param(p, optimizer.param_groups[0])+ optimizer.step_param(cpu_param, optimizer.param_groups[0])++ assert entered == ["cuda:1", "cuda:0", "cuda:1"]+ assert updated == [("p1", "cuda:1"), ("p0", "cuda:0"), ("p1", "cuda:1"), ("cpu", "cuda:0")]+ assert current == ["cuda:0"]diff --git a/unsloth/optimizers/flash_adamw.py b/unsloth/optimizers/flash_adamw.pyindex 6770a63495f..c32ab1c942b 100644--- a/unsloth/optimizers/flash_adamw.py+++ b/unsloth/optimizers/flash_adamw.py@@ -31,6 +31,7 @@ __all__ = ["get_flash_adamw_cls_and_kwargs",]+import contextlibimport osimport sys@@ -177,10 +178,16 @@ def _get_flash_adamw_class():_widen_variance(self.state[p].get("exp_avg_sq"))def _load_state_for_param(self, param, opt_state, hparams):- state = super()._load_state_for_param(param, opt_state, hparams)+ with _on_param_device(param):+ state = super()._load_state_for_param(param, opt_state, hparams)_widen_variance(state.get("exp_avg_sq"))return state+ def _state_dict_for_param(self, param_number, *args, **kwargs):+ idx_to_param = kwargs.get("idx_to_param", args[2] if len(args) > 2 else {})+ with _on_param_device(idx_to_param.get(param_number)):+ return super()._state_dict_for_param(param_number, *args, **kwargs)+def load_state_dict(self, state_dict):# torch.optim casts floating state to the param dtype: restore the fp32 variance.saved_ids = [i for g in state_dict.get("param_groups", []) for i in g["params"]]@@ -232,12 +239,20 @@ def _get_flash_adamw_class():)if group is not None and not self._group_quantizes(group):_refuse_unquantized_fp16()- return super().step_param(p, group)+ with _on_param_device(p):+ return super().step_param(p, group)_FLASH_ADAMW_CLASS = UnslothFlashAdamWreturn _FLASH_ADAMW_CLASS+def _on_param_device(p):+ # Triton launches on the current device, not p.device; device_map splits need it switched.+ if p is not None and p.is_cuda:+ return torch.cuda.device(p.device)+ return contextlib.nullcontext()++def _variance_update(var, grad, beta2):"""fp32 `var * beta2 + grad**2 * (1 - beta2)`, as flashoptim's fused kernel computes it."""global _VARIANCE_UPDATE_KERNEL