Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Run each FlashAdamW update on its parameter's CUDA device (#14773)

Daniel Han9d719c0b0202
diff --git a/tests/utils/test_flash_adamw_device.py b/tests/utils/test_flash_adamw_device.py
new file mode 100644
index 00000000000..5c7919ffa0e
--- /dev/null
+++ b/tests/utils/test_flash_adamw_device.py
@@ -0,0 +1,111 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Real FlashOptim on two GPUs: an update of a parameter on a non-current device (device_map
+split model) must match the same update run with that device current, bit for bit."""
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers/trl)
+import torch
+
+pytestmark = [
+ pytest.mark.gpu,
+ pytest.mark.skipif(
+ not torch.cuda.is_available() or torch.cuda.device_count() < 2,
+ reason = "requires two visible CUDA GPUs",
+ ),
+]
+
+
+ "dtype,quantize",
+ [
+ (torch.float32, False),
+ (torch.bfloat16, False),
+ (torch.bfloat16, True),
+ (torch.float16, True),
+ ],
+)
[email protected]("entry", ["step", "step_param"])
+def test_adamw_flash_noncurrent_cuda_device(dtype, quantize, entry):
+ pytest.importorskip("flashoptim")
+ from unsloth.optimizers.flash_adamw import _get_flash_adamw_class
+
+ torch.cuda.set_device(0)
+ optimizer_cls = _get_flash_adamw_class()
+ p0 = torch.nn.Parameter(torch.ones(33, device = "cuda:0", dtype = dtype))
+ p1 = torch.nn.Parameter(torch.ones(33, device = "cuda:1", dtype = dtype))
+ q0 = torch.nn.Parameter(p0.detach().clone())
+ q1 = torch.nn.Parameter(p1.detach().clone())
+ optimizer = optimizer_cls([p0, p1], quantize = quantize, lr = 0.001)
+ reference = optimizer_cls([q0, q1], quantize = quantize, lr = 0.001)
+ for grad in (0.125, -0.25, 0.0, 0.0625):
+ for p in (p0, p1, q0, q1):
+ p.grad = torch.full_like(p, grad)
+ for q in (q0, q1):
+ with torch.cuda.device(q.device):
+ reference.step_param(q)
+ if entry == "step":
+ optimizer.step()
+ else:
+ for p in (p0, p1):
+ optimizer.step_param(p)
+ assert torch.cuda.current_device() == 0, "the caller's device must be restored"
+ for p, q in ((p0, q0), (p1, q1)):
+ torch.cuda.synchronize(p.device)
+ torch.testing.assert_close(p, q, rtol = 0, atol = 0)
+ a, b = optimizer.state[p], reference.state[q]
+ assert a["step"].item() == b["step"].item()
+ for key in ("exp_avg", "exp_avg_sq"):
+ torch.testing.assert_close(
+ a[key].kernel_tensor, b[key].kernel_tensor, rtol = 0, atol = 0
+ )
+ torch.testing.assert_close(
+ a[key].kernel_scales_or_self, b[key].kernel_scales_or_self, rtol = 0, atol = 0
+ )
+ if "error_bits" in a:
+ torch.testing.assert_close(a["error_bits"], b["error_bits"], rtol = 0, atol = 0)
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16])
+def test_adamw_flash_uncompressed_state_dict_round_trip_noncurrent_device(dtype):
+ """compress_state_dict=False dequantizes the moments on save and requantizes them on load,
+ both with Triton: a parameter on cuda:1 with cuda:0 current must save and resume."""
+ pytest.importorskip("flashoptim")
+ from unsloth.optimizers.flash_adamw import _get_flash_adamw_class
+
+ torch.cuda.set_device(0)
+ optimizer_cls = _get_flash_adamw_class()
+ kwargs = dict(quantize = True, compress_state_dict = False, lr = 0.001)
+ params = [torch.nn.Parameter(torch.ones(65, device = f"cuda:{i}", dtype = dtype)) for i in (0, 1)]
+ optimizer = optimizer_cls(params, **kwargs)
+ for grad in (0.125, -0.25):
+ for p in params:
+ p.grad = torch.full_like(p, grad)
+ optimizer.step()
+ expected = {}
+ for i, p in enumerate(params):
+ with torch.cuda.device(p.device):
+ expected[i] = {
+ k: optimizer.state[p][k].materialize() for k in ("exp_avg", "exp_avg_sq")
+ }
+
+ saved = optimizer.state_dict()
+ assert torch.cuda.current_device() == 0, "the caller's device must be restored"
+ for i, p in enumerate(params):
+ for key, value in expected[i].items():
+ assert saved["state"][i][key].device == p.device
+ torch.testing.assert_close(saved["state"][i][key], value, rtol = 0, atol = 0)
+
+ resumed = optimizer_cls(params, **kwargs)
+ resumed.load_state_dict(saved)
+ assert torch.cuda.current_device() == 0, "the caller's device must be restored"
+ for i, p in enumerate(params):
+ for key, value in expected[i].items():
+ with torch.cuda.device(p.device):
+ loaded = resumed.state[p][key].materialize()
+ assert loaded.device == p.device
+ torch.testing.assert_close(loaded.float(), value.float(), rtol = 0.02, atol = 1e-6)
+ p.grad = torch.full_like(p, 0.0625)
+ resumed.step()
diff --git a/tests/utils/test_flash_adamw_optim.py b/tests/utils/test_flash_adamw_optim.py
index 54159aa6e9c..af3c791a588 100644
--- a/tests/utils/test_flash_adamw_optim.py
+++ b/tests/utils/test_flash_adamw_optim.py
@@ -597,3 +597,53 @@ def test_adamw_flash_fused_bf16_variance_follows_fp32(monkeypatch, gradient):
assert var.dtype == torch.float32
# fp32 rounding differs between the kernel and the torch EMA (about 2e-5 after 2000 steps).
torch.testing.assert_close(var, ema, rtol = 1e-4, atol = 1e-12)
+
+
+def test_adamw_flash_updates_each_param_on_its_own_device(stub_flashoptim, monkeypatch):
+ """A device_map split model: each CUDA update must run with its parameter's device current."""
+ import contextlib
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ FlashAdamW, _ = stub_flashoptim
+ current = ["cuda:0"]
+ entered = []
+ updated = []
+
+ @contextlib.contextmanager
+ def fake_device(device):
+ entered.append(str(device))
+ previous, current[0] = current[0], str(device)
+ try:
+ yield
+ finally:
+ current[0] = previous
+
+ def record_step_param(
+ self,
+ p,
+ group = None,
+ ):
+ updated.append((labels[id(p)], current[0]))
+
+ monkeypatch.setattr(torch.cuda, "device", fake_device)
+ monkeypatch.setattr(FlashAdamW, "step_param", record_step_param)
+ optimizer_cls = flash_adamw._get_flash_adamw_class()
+ cpu_param = torch.nn.Parameter(torch.zeros(4))
+ optimizer = optimizer_cls([cpu_param])
+ labels = {id(cpu_param): "cpu"}
+
+ # Stand-ins for parameters placed on two GPUs: only device, dtype and is_cuda are read.
+ on_gpu0 = types.SimpleNamespace(
+ device = torch.device("cuda", 0), is_cuda = True, dtype = torch.float32
+ )
+ on_gpu1 = types.SimpleNamespace(
+ device = torch.device("cuda", 1), is_cuda = True, dtype = torch.float32
+ )
+ labels.update({id(on_gpu0): "p0", id(on_gpu1): "p1"})
+ for p in (on_gpu1, on_gpu0, on_gpu1):
+ optimizer.step_param(p, optimizer.param_groups[0])
+ optimizer.step_param(cpu_param, optimizer.param_groups[0])
+
+ assert entered == ["cuda:1", "cuda:0", "cuda:1"]
+ assert updated == [("p1", "cuda:1"), ("p0", "cuda:0"), ("p1", "cuda:1"), ("cpu", "cuda:0")]
+ assert current == ["cuda:0"]
diff --git a/unsloth/optimizers/flash_adamw.py b/unsloth/optimizers/flash_adamw.py
index 6770a63495f..c32ab1c942b 100644
--- a/unsloth/optimizers/flash_adamw.py
+++ b/unsloth/optimizers/flash_adamw.py
@@ -31,6 +31,7 @@ __all__ = [
"get_flash_adamw_cls_and_kwargs",
]
+import contextlib
import os
import sys
@@ -177,10 +178,16 @@ def _get_flash_adamw_class():
_widen_variance(self.state[p].get("exp_avg_sq"))
def _load_state_for_param(self, param, opt_state, hparams):
- state = super()._load_state_for_param(param, opt_state, hparams)
+ with _on_param_device(param):
+ state = super()._load_state_for_param(param, opt_state, hparams)
_widen_variance(state.get("exp_avg_sq"))
return state
+ def _state_dict_for_param(self, param_number, *args, **kwargs):
+ idx_to_param = kwargs.get("idx_to_param", args[2] if len(args) > 2 else {})
+ with _on_param_device(idx_to_param.get(param_number)):
+ return super()._state_dict_for_param(param_number, *args, **kwargs)
+
def load_state_dict(self, state_dict):
# torch.optim casts floating state to the param dtype: restore the fp32 variance.
saved_ids = [i for g in state_dict.get("param_groups", []) for i in g["params"]]
@@ -232,12 +239,20 @@ def _get_flash_adamw_class():
)
if group is not None and not self._group_quantizes(group):
_refuse_unquantized_fp16()
- return super().step_param(p, group)
+ with _on_param_device(p):
+ return super().step_param(p, group)
_FLASH_ADAMW_CLASS = UnslothFlashAdamW
return _FLASH_ADAMW_CLASS
+def _on_param_device(p):
+ # Triton launches on the current device, not p.device; device_map splits need it switched.
+ if p is not None and p.is_cuda:
+ return torch.cuda.device(p.device)
+ return contextlib.nullcontext()
+
+
def _variance_update(var, grad, beta2):
"""fp32 `var * beta2 + grad**2 * (1 - beta2)`, as flashoptim's fused kernel computes it."""
global _VARIANCE_UPDATE_KERNEL