Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add optim = "adamw_flash" (FlashAdamW from flashoptim) for lower optimizer memory (#13881)

Daniel Han313bfd9e88f9
diff --git a/tests/utils/test_flash_adamw_optim.py b/tests/utils/test_flash_adamw_optim.py
new file mode 100644
index 00000000000..9fd9272362d
--- /dev/null
+++ b/tests/utils/test_flash_adamw_optim.py
@@ -0,0 +1,406 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import os
+import sys
+import types
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers/trl)
+import torch
+
+OUT = os.path.join(os.environ.get("TMPDIR", "."), "unsloth-flash-adamw-test")
+
+
+def stub_flashoptim(monkeypatch):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ calls = []
+
+ class FlashAdamW(torch.optim.Optimizer):
+ def __init__(
+ self,
+ params,
+ lr = 1e-3,
+ betas = (0.9, 0.999),
+ eps = 1e-8,
+ weight_decay = 1e-2,
+ master_weight_bits = 24,
+ **kwargs,
+ ):
+ calls.append(
+ dict(lr = lr, betas = betas, eps = eps, master_weight_bits = master_weight_bits, **kwargs)
+ )
+ # flashoptim's own defaults carry initial_lr = the constructor lr.
+ super().__init__(params, dict(lr = lr, initial_lr = lr, weight_decay = weight_decay))
+
+ module = types.ModuleType("flashoptim")
+ module.FlashAdamW = FlashAdamW
+ module.__version__ = "0.1.4"
+ monkeypatch.setitem(sys.modules, "flashoptim", module)
+ monkeypatch.setattr(flash_adamw, "_FLASH_ADAMW_CLASS", None)
+ yield FlashAdamW, calls
+ flash_adamw._FLASH_ADAMW_CLASS = None
+
+
+def _sft_config(**kwargs):
+ from trl import SFTConfig
+ return SFTConfig(output_dir = OUT, report_to = [], **kwargs)
+
+
+def _unsloth_args(**kwargs):
+ from unsloth.trainer import UnslothTrainingArguments
+ return UnslothTrainingArguments(output_dir = OUT, report_to = [], **kwargs)
+
+
[email protected]("make_args", [_sft_config, _unsloth_args])
+def test_adamw_flash_selects_flashoptim(stub_flashoptim, make_args, monkeypatch):
+ from transformers import Trainer
+
+ FlashAdamW, calls = stub_flashoptim
+ args = make_args(
+ optim = "adamw_flash",
+ learning_rate = 3e-4,
+ adam_beta2 = 0.95,
+ optim_args = "master_weight_bits=24, quantize=True",
+ )
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(args)
+ assert issubclass(optimizer_cls, FlashAdamW)
+ assert optimizer_kwargs["lr"] == 3e-4
+ assert optimizer_kwargs["betas"] == (0.9, 0.95)
+ assert optimizer_kwargs["master_weight_bits"] == 24
+ assert optimizer_kwargs["quantize"] is True
+
+ bf16 = torch.nn.Parameter(torch.zeros(4, dtype = torch.bfloat16))
+ optimizer_cls([{"params": [bf16], "weight_decay": 0.1}], **optimizer_kwargs)
+ assert calls[-1]["master_weight_bits"] == 24
+ # flashoptim rejects master_weight_bits on all-fp32 params (LoRA adapters).
+ fp32 = torch.nn.Parameter(torch.zeros(4))
+ optimizer_cls([{"params": [fp32], "weight_decay": 0.0}], **optimizer_kwargs)
+ assert calls[-1]["master_weight_bits"] is None
+
+
+def test_adamw_flash_missing_dependency(monkeypatch):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+ from transformers import Trainer
+
+ monkeypatch.setitem(sys.modules, "flashoptim", None)
+ monkeypatch.setattr(flash_adamw, "_FLASH_ADAMW_CLASS", None)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ args = _sft_config(optim = "adamw_flash")
+ with pytest.raises(ImportError, match = "pip install 'flashoptim"):
+ Trainer.get_optimizer_cls_and_kwargs(args)
+
+
+def test_other_optimizers_untouched():
+ from transformers import Trainer
+
+ args = _sft_config(optim = "adamw_torch")
+ assert not getattr(args, "_unsloth_flash_adamw", False)
+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(args)
+ assert optimizer_cls is torch.optim.AdamW
+
+
[email protected]("make_args", [_sft_config, _unsloth_args])
+def test_adamw_flash_survives_dataclasses_replace(stub_flashoptim, make_args, monkeypatch):
+ import dataclasses
+ from transformers import Trainer
+
+ FlashAdamW, _ = stub_flashoptim
+ args = make_args(optim = "adamw_flash")
+ copied = dataclasses.replace(args, learning_rate = 1e-5)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(copied)
+ assert issubclass(optimizer_cls, FlashAdamW)
+ assert optimizer_kwargs["lr"] == 1e-5
+ # Default: full 32 bit master semantics and unquantized moments.
+ assert optimizer_kwargs["master_weight_bits"] == 32
+ assert optimizer_kwargs["quantize"] is False
+
+
[email protected]("make_args", [_sft_config, _unsloth_args])
+def test_adamw_flash_marker_parses_in_transformers(make_args):
+ trainer_module = __import__("transformers.trainer", fromlist = ["_parse_optim_args"])
+ parse = getattr(trainer_module, "_parse_optim_args", None)
+ if parse is None:
+ pytest.skip("transformers has no _parse_optim_args")
+ args = make_args(optim = "adamw_flash", optim_args = "quantize=True")
+ parsed = parse(args.optim_args)
+ assert parsed["quantize"] == "True"
+
+
+def _flash_kwargs_args(**kwargs):
+ defaults = dict(
+ learning_rate = 1e-4,
+ adam_beta1 = 0.9,
+ adam_beta2 = 0.999,
+ adam_epsilon = 1e-8,
+ optim_args = None,
+ fsdp = [],
+ fsdp_config = {},
+ )
+ defaults.update(kwargs)
+ return types.SimpleNamespace(**defaults)
+
+
[email protected]("make_args", [_sft_config, _unsloth_args])
+def test_adamw_flash_restored_from_to_dict(stub_flashoptim, make_args, monkeypatch):
+ import dataclasses
+ import inspect
+ from transformers import Trainer
+
+ FlashAdamW, _ = stub_flashoptim
+ saved = make_args(optim = "adamw_flash").to_dict()
+ assert saved["optim"] == "adamw_torch"
+ # A fresh process has the stock Trainer method until a marked config is built.
+ patched = Trainer.__dict__["get_optimizer_cls_and_kwargs"].__func__
+ original = inspect.getclosurevars(patched).nonlocals["original"]
+ monkeypatch.setattr(Trainer, "get_optimizer_cls_and_kwargs", staticmethod(original))
+ cls = type(make_args(optim = "adamw_torch"))
+ init_fields = {f.name for f in dataclasses.fields(cls) if f.init}
+ rebuilt = cls(
+ **{k: v for k, v in saved.items() if k in init_fields and not k.endswith("_token")}
+ )
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(rebuilt)
+ assert issubclass(optimizer_cls, FlashAdamW)
+
+
+def test_adamw_flash_keeps_per_group_initial_lr(stub_flashoptim, monkeypatch):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(
+ _flash_kwargs_args(optim_args = "decouple_lr=True")
+ )
+ groups = [
+ {"params": [torch.nn.Parameter(torch.zeros(4))], "lr": 1e-4},
+ {"params": [torch.nn.Parameter(torch.zeros(4))], "lr": 5e-6},
+ {"params": [torch.nn.Parameter(torch.zeros(4))], "lr": 1.6e-3},
+ ]
+ optimizer = optimizer_cls(groups, **optimizer_kwargs)
+ assert [g["initial_lr"] for g in optimizer.param_groups] == [1e-4, 5e-6, 1.6e-3]
+ scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda step: 1.0)
+ assert scheduler.base_lrs == [1e-4, 5e-6, 1.6e-3]
+
+
+def test_adamw_flash_rejects_mixed_dtype_uncompressed_quantized(stub_flashoptim, monkeypatch):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ params = [
+ torch.nn.Parameter(torch.zeros(4, dtype = torch.bfloat16)),
+ torch.nn.Parameter(torch.zeros(4)),
+ ]
+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(
+ _flash_kwargs_args(optim_args = "quantize=True,compress_state_dict=False")
+ )
+ with pytest.raises(ValueError, match = "compress_state_dict"):
+ optimizer_cls([{"params": params}], **optimizer_kwargs)
+ for optim_args in (None, "quantize=True"):
+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(
+ _flash_kwargs_args(optim_args = optim_args)
+ )
+ optimizer_cls([{"params": params}], **optimizer_kwargs)
+
+
+ "fsdp_config, env, rejected",
+ [
+ ({"version": 1}, {}, True),
+ ({"version": 2}, {}, False),
+ (None, {"ACCELERATE_USE_FSDP": "true"}, True),
+ (None, {"ACCELERATE_USE_FSDP": "true", "FSDP_VERSION": "2"}, False),
+ (None, {}, False),
+ ],
+)
+def test_adamw_flash_rejects_fsdp1(stub_flashoptim, monkeypatch, fsdp_config, env, rejected):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ for key in ("ACCELERATE_USE_FSDP", "FSDP_VERSION", "FSDP_OFFLOAD_PARAMS"):
+ monkeypatch.delenv(key, raising = False)
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ args = _flash_kwargs_args(
+ fsdp = ["full_shard"] if fsdp_config else [], fsdp_config = fsdp_config or {}
+ )
+ if rejected:
+ with pytest.raises(ValueError, match = "FSDP1"):
+ flash_adamw.get_flash_adamw_cls_and_kwargs(args)
+ else:
+ flash_adamw.get_flash_adamw_cls_and_kwargs(args)
+
+
[email protected]("hip, platform", [("6.2.0", "linux"), (None, "win32")])
+def test_adamw_flash_rejects_unsupported_platform(stub_flashoptim, monkeypatch, hip, platform):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)
+ monkeypatch.setattr(sys, "platform", platform)
+ with pytest.raises(NotImplementedError, match = "NVIDIA CUDA"):
+ flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args())
+
+
[email protected]("make_args", [_sft_config, _unsloth_args])
+def test_adamw_flash_later_optim_change_wins(stub_flashoptim, make_args, monkeypatch):
+ import dataclasses
+ from transformers import Trainer
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ args = make_args(optim = "adamw_flash", optim_args = "quantize=True")
+ replaced = dataclasses.replace(args, optim = "rmsprop")
+ assert "unsloth_adamw_flash" not in str(replaced.optim_args)
+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(replaced)
+ assert optimizer_cls is torch.optim.RMSprop
+ args.optim = "sgd"
+ args.optim_args = None
+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(args)
+ assert optimizer_cls is torch.optim.SGD
+
+
+def test_adamw_flash_keeps_one_shot_param_iterables(stub_flashoptim, monkeypatch):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ model = torch.nn.Linear(4, 4)
+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(
+ _flash_kwargs_args()
+ )
+ optimizer = optimizer_cls([{"params": model.parameters(), "lr": 1e-4}], **optimizer_kwargs)
+ assert len(optimizer.param_groups[0]["params"]) == 2
+ optimizer = optimizer_cls(model.parameters(), **optimizer_kwargs)
+ assert len(optimizer.param_groups[0]["params"]) == 2
+
+
+ "fsdp, fsdp_config, env",
+ [
+ (["full_shard"], {"version": 2, "cpu_offload": True}, {}),
+ ("full_shard offload", {"version": 2}, {}),
+ (
+ [],
+ None,
+ {"ACCELERATE_USE_FSDP": "true", "FSDP_VERSION": "2", "FSDP_OFFLOAD_PARAMS": "true"},
+ ),
+ ],
+)
+def test_adamw_flash_rejects_fsdp2_cpu_offload(
+ stub_flashoptim, monkeypatch, fsdp, fsdp_config, env
+):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ for key in ("ACCELERATE_USE_FSDP", "FSDP_VERSION", "FSDP_OFFLOAD_PARAMS"):
+ monkeypatch.delenv(key, raising = False)
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ args = _flash_kwargs_args(fsdp = fsdp, fsdp_config = fsdp_config or {})
+ with pytest.raises(ValueError, match = "CPU offloaded"):
+ flash_adamw.get_flash_adamw_cls_and_kwargs(args)
+
+
[email protected]("make_args", [_sft_config, _unsloth_args])
+def test_adamw_flash_accepted_by_hf_argument_parser(make_args):
+ from transformers import HfArgumentParser
+
+ cls = type(make_args(optim = "adamw_torch"))
+ (args,) = HfArgumentParser(cls).parse_args_into_dataclasses(
+ ["--output_dir", OUT, "--report_to", "none", "--optim", "adamw_flash"]
+ )
+ assert getattr(args, "_unsloth_flash_adamw", False)
+ with pytest.raises(SystemExit):
+ HfArgumentParser(cls).parse_args_into_dataclasses(
+ ["--output_dir", OUT, "--report_to", "none", "--optim", "not_an_optim"]
+ )
+
+
[email protected]("make_args", [_sft_config, _unsloth_args])
+def test_adamw_flash_switch_back_to_adamw(stub_flashoptim, make_args, monkeypatch):
+ import dataclasses
+ from transformers import Trainer
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ args = make_args(optim = "adamw_flash")
+ replaced = dataclasses.replace(args, optim = "adamw_torch", optim_args = None)
+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(replaced)
+ assert optimizer_cls is torch.optim.AdamW
+
+
[email protected]("make_args", [_sft_config, _unsloth_args])
+def test_adamw_flash_switch_back_in_place(stub_flashoptim, make_args, monkeypatch):
+ from transformers import Trainer
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ args = make_args(optim = "adamw_flash")
+ args.optim = "adamw_torch"
+ args.optim_args = None
+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(args)
+ assert optimizer_cls is torch.optim.AdamW
+
+
+def test_adamw_flash_platform_checked_before_import(monkeypatch):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setitem(sys.modules, "flashoptim", None)
+ monkeypatch.setattr(flash_adamw, "_FLASH_ADAMW_CLASS", None)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ with pytest.raises(NotImplementedError, match = "NVIDIA CUDA"):
+ flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args())
+
+
+ "deepspeed, env",
+ [
+ ({"zero_optimization": {"stage": 2, "offload_optimizer": {"device": "cpu"}}}, {}),
+ ({"zero_optimization": {"stage": 3, "offload_optimizer": {"device": "nvme"}}}, {}),
+ (
+ None,
+ {
+ "ACCELERATE_USE_DEEPSPEED": "true",
+ "ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE": "cpu",
+ },
+ ),
+ ],
+)
+def test_adamw_flash_rejects_deepspeed_optimizer_offload(
+ stub_flashoptim, monkeypatch, deepspeed, env
+):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ for key in ("ACCELERATE_USE_DEEPSPEED", "ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE"):
+ monkeypatch.delenv(key, raising = False)
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ with pytest.raises(ValueError, match = "offload_optimizer"):
+ flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args(deepspeed = deepspeed))
+ no_offload = {"zero_optimization": {"stage": 2, "offload_optimizer": {"device": "none"}}}
+ monkeypatch.delenv("ACCELERATE_USE_DEEPSPEED", raising = False)
+ flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args(deepspeed = no_offload))
+
+
+ "optim_args", ["quantize=ture", "compress_state_dict=tru", "check_numerics=2"]
+)
+def test_adamw_flash_rejects_invalid_booleans(optim_args):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+ with pytest.raises(ValueError, match = "not a boolean"):
+ flash_adamw._parse_optim_args(optim_args)
+ assert flash_adamw._parse_optim_args("quantize=False,decouple_lr=no") == {
+ "quantize": False,
+ "decouple_lr": False,
+ }
+
+
+def test_adamw_flash_requires_compatible_flashoptim(stub_flashoptim, monkeypatch):
+ import unsloth.optimizers.flash_adamw as flash_adamw
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(sys.modules["flashoptim"], "__version__", "0.1.1")
+ with pytest.raises(ImportError, match = "flashoptim>=0.1.4, found 0.1.1"):
+ flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args())
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 3cac0f24602..fcf0bf3a70e 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -537,6 +537,7 @@ class Unsloth{RLConfig_name}({RLConfig_name}):
"""
{__RLConfig_doc__}
"""
+ _unsloth_accepts_adamw_flash = True
vllm_sampling_params: Optional[Any] = field(
default = None,
metadata = {{'help': 'vLLM SamplingParams'}},
@@ -3294,6 +3295,10 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
"if fsdp and isinstance(_unsloth_fsdp_cfg, dict):\n"
f" _unsloth_fsdp2 = _unsloth_fsdp2 or str(_unsloth_fsdp_cfg.get('fsdp_version', _unsloth_fsdp_cfg.get('version', {_FSDP_DEFAULT_VERSION}))) == '2'\n"
)
+ extra_args += (
+ "_unsloth_flash_adamw = str(getattr(optim, 'value', optim)).strip().lower() == 'adamw_flash'\n"
+ "if _unsloth_flash_adamw: optim = 'adamw_torch'\n"
+ )
extra_args += (
"if _unsloth_fsdp2 and str(getattr(optim, 'value', optim)) == 'adamw_8bit':\n"
" print(\"Unsloth: FSDP2 shards parameters as DTensors, which bitsandbytes' 8-bit optimizers cannot step. Switching optim = 'adamw_8bit' to 'adamw_torch_fused'.\")\n"
@@ -3553,6 +3558,15 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
" self.gradient_checkpointing_kwargs = _gc_kwargs\n"
)
+ if "optim" in call_args:
+ RLConfig_post += (
+ " from unsloth.optimizers.flash_adamw import is_flash_adamw_marked, mark_flash_adamw, unmark_flash_adamw\n"
+ " if _unsloth_flash_adamw or is_flash_adamw_marked(self):\n"
+ " mark_flash_adamw(self)\n"
+ " else:\n"
+ " unmark_flash_adamw(self)\n"
+ )
+
RLTrainer_extras = patch_functions(
RLTrainer, trainer_file, RLTrainer_name, all_imports, imports
)
diff --git a/unsloth/optimizers/flash_adamw.py b/unsloth/optimizers/flash_adamw.py
new file mode 100644
index 00000000000..a215dee4845
--- /dev/null
+++ b/unsloth/optimizers/flash_adamw.py
@@ -0,0 +1,366 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""`optim = "adamw_flash"`: FlashAdamW from databricks/flashoptim (arXiv 2602.23349).
+
+bf16 / fp16 parameters keep a 16 bit error correction term in the optimizer state, so
+the effective master weight is fp32, and the Adam moments are stored in the parameter
+dtype: about 10 bytes per parameter (with bf16 grads) instead of 16 for bf16 weights
+plus fp32 master plus fp32 moments. This default tracked fp32 master AdamW on a real
+Qwen2.5-0.5B full finetune. optim_args = "quantize=True" stores the moments in 8 bits
+(about 8 bytes per parameter; with "master_weight_bits=24", flashoptim's own default,
+about 7), but its loss curve moved measurably away from AdamW there.
+fp32 parameters (LoRA adapters) have no correction term; for them only quantize=True
+saves memory.
+
+transformers validates `optim` against OptimizerNames, so the config stores
+`adamw_torch` and marks itself with `_unsloth_flash_adamw` plus a token in `optim_args`
+(a dataclass field, so `dataclasses.replace` keeps it); the patched
+`Trainer.get_optimizer_cls_and_kwargs` reads either mark. Because the placeholder is a real
+optim name, `dataclasses.replace(args, optim = "adamw_torch")` still selects FlashAdamW;
+switch back with `optim_args = None` (or `unmark_flash_adamw(args)`).
+"""
+
+__all__ = [
+ "FLASH_ADAMW_NAME",
+ "is_flash_adamw",
+ "mark_flash_adamw",
+ "unmark_flash_adamw",
+ "get_flash_adamw_cls_and_kwargs",
+]
+
+import os
+import sys
+
+import torch
+
+FLASH_ADAMW_NAME = "adamw_flash"
+_FLASH_ADAMW_PLACEHOLDER = "adamw_torch"
+_FLASH_ADAMW_MARKER = "unsloth_adamw_flash"
+_FLASH_ADAMW_CLASS = None
+_FLASH_ADAMW_MIN_VERSION = "0.1.4"
+_TRUE_VALUES = ("1", "true", "yes")
+_FALSE_VALUES = ("0", "false", "no")
+_announced = False
+
+
+def is_flash_adamw(optim):
+ return str(getattr(optim, "value", optim)).strip().lower() == FLASH_ADAMW_NAME
+
+
+def _parse_optim_args(optim_args):
+ parsed = {}
+ if not optim_args:
+ return parsed
+ for item in str(optim_args).replace(" ", "").split(","):
+ if not item:
+ continue
+ key, _, value = item.partition("=")
+ if key == _FLASH_ADAMW_MARKER:
+ continue
+ if key == "master_weight_bits":
+ parsed[key] = None if value.lower() in ("none", "0", "") else int(value)
+ elif key in ("quantize", "compress_state_dict", "check_numerics", "decouple_lr"):
+ if value.lower() not in _TRUE_VALUES + _FALSE_VALUES:
+ raise ValueError(
+ f"Unsloth: optim = 'adamw_flash' optim_args {key}={value!r} is not a boolean. "
+ "Use true / false, 1 / 0 or yes / no."
+ )
+ parsed[key] = value.lower() in _TRUE_VALUES
+ else:
+ raise ValueError(
+ f"Unsloth: optim = 'adamw_flash' does not understand optim_args key {key!r}. "
+ "Supported: master_weight_bits=24|32|none, quantize, compress_state_dict, check_numerics, decouple_lr."
+ )
+ return parsed
+
+
+def _get_flash_adamw_class():
+ global _FLASH_ADAMW_CLASS
+ if _FLASH_ADAMW_CLASS is not None:
+ return _FLASH_ADAMW_CLASS
+ try:
+ import flashoptim
+ from flashoptim import FlashAdamW
+ except ImportError as error:
+ raise ImportError(
+ "Unsloth: optim = 'adamw_flash' needs the flashoptim package. "
+ f"Install it via `pip install 'flashoptim>={_FLASH_ADAMW_MIN_VERSION}'`."
+ ) from error
+ from packaging.version import Version
+
+ version = getattr(flashoptim, "__version__", None)
+ if version is None:
+ import importlib.metadata
+ version = importlib.metadata.version("flashoptim")
+ # 0.1.4 made compress_state_dict default to True, which the resume guard below relies on.
+ if Version(version) < Version(_FLASH_ADAMW_MIN_VERSION):
+ raise ImportError(
+ f"Unsloth: optim = 'adamw_flash' needs flashoptim>={_FLASH_ADAMW_MIN_VERSION}, found {version}. "
+ f"Upgrade via `pip install -U 'flashoptim>={_FLASH_ADAMW_MIN_VERSION}'`."
+ )
+
+ class UnslothFlashAdamW(FlashAdamW):
+ def __init__(
+ self,
+ params,
+ *args,
+ master_weight_bits = 32,
+ **kwargs,
+ ):
+ params = [
+ {**group, "params": _materialize_params(group["params"])}
+ if isinstance(group, dict) and "params" in group
+ else group
+ for group in params
+ ]
+ # flashoptim seeds every group's initial_lr with the constructor lr, and the LR
+ # scheduler then uses it as the group's base lr (setdefault keeps it), which would
+ # drop embedding_learning_rate and loraplus_lr_ratio, and skew decouple_lr decay.
+ params = [
+ {**group, "initial_lr": group.get("initial_lr", group["lr"])}
+ if isinstance(group, dict) and "lr" in group
+ else group
+ for group in params
+ ]
+ # flashoptim raises when every parameter is fp32 and master_weight_bits is set,
+ # which is the normal LoRA case: there only the 8 bit moments apply.
+ tensors = [
+ p
+ for group in params
+ for p in (group["params"] if isinstance(group, dict) else [group])
+ ]
+ if tensors and all(p.dtype == torch.float32 for p in tensors):
+ master_weight_bits = None
+ # flashoptim's load_state_dict re-quantizes uncompressed moments with one
+ # optimizer wide bf16 check, which corrupts the other dtype's scales on resume.
+ if (
+ kwargs.get("quantize", True)
+ and not kwargs.get("compress_state_dict", True)
+ and len({p.dtype == torch.bfloat16 for p in tensors}) > 1
+ ):
+ raise ValueError(
+ "Unsloth: optim = 'adamw_flash' with quantize=True and compress_state_dict=False "
+ "cannot resume a model mixing bf16 and other trainable dtypes. "
+ "Drop compress_state_dict=False or quantize=True."
+ )
+ super().__init__(params, *args, master_weight_bits = master_weight_bits, **kwargs)
+
+ _FLASH_ADAMW_CLASS = UnslothFlashAdamW
+ return _FLASH_ADAMW_CLASS
+
+
+def _materialize_params(params):
+ # A one-shot iterable (model.parameters()) would be exhausted by the dtype scan below.
+ return [params] if isinstance(params, torch.Tensor) else list(params)
+
+
+def _has_marker_token(args):
+ optim_args = str(getattr(args, "optim_args", None) or "").replace(" ", "")
+ return any(item.partition("=")[0] == _FLASH_ADAMW_MARKER for item in optim_args.split(","))
+
+
+def _has_placeholder_optim(args):
+ optim = getattr(args, "optim", None)
+ return str(getattr(optim, "value", optim)).strip().lower() == _FLASH_ADAMW_PLACEHOLDER
+
+
+def _is_marked_in_optim_args(args):
+ # The mark only counts while optim is still the placeholder, so a later optim change wins.
+ return _has_placeholder_optim(args) and _has_marker_token(args)
+
+
+def _is_marked(args):
+ # The optim_args token alone decides, so an in place `args.optim_args = None` switches back.
+ return _is_marked_in_optim_args(args)
+
+
+def _strip_marker(optim_args):
+ items = str(optim_args or "").split(",")
+ kept = [
+ i
+ for i in items
+ if i.strip() and i.replace(" ", "").partition("=")[0] != _FLASH_ADAMW_MARKER
+ ]
+ return ",".join(kept) or None
+
+
+def _is_truthy(value):
+ return str(value).strip().lower() in ("1", "true", "yes")
+
+
+def _fsdp_cpu_offload(args):
+ fsdp = getattr(args, "fsdp", None)
+ items = (
+ fsdp.split() if isinstance(fsdp, str) else (fsdp if isinstance(fsdp, (list, tuple)) else [])
+ )
+ if any(str(getattr(item, "value", item)).lower() == "offload" for item in items):
+ return True
+ config = getattr(args, "fsdp_config", None)
+ config = config if isinstance(config, dict) else {}
+ if any(
+ _is_truthy(config.get(key, False))
+ for key in ("cpu_offload", "offload_params", "fsdp_offload_params")
+ ):
+ return True
+ return _is_truthy(os.environ.get("FSDP_OFFLOAD_PARAMS", ""))
+
+
+def _fsdp_version(args):
+ if getattr(args, "fsdp", None):
+ config = getattr(args, "fsdp_config", None)
+ config = config if isinstance(config, dict) else {}
+ version = config.get("version", config.get("fsdp_version", None))
+ if version is None:
+ import transformers
+ from packaging.version import Version
+ version = 2 if Version(transformers.__version__) >= Version("5.10.0") else 1
+ return int(version)
+ if os.environ.get("ACCELERATE_USE_FSDP", "").strip().lower() in ("1", "true", "yes"):
+ return int(os.environ.get("FSDP_VERSION", "").strip() or 1)
+ return None
+
+
+def _deepspeed_offloads_optimizer(args):
+ if _is_truthy(os.environ.get("ACCELERATE_USE_DEEPSPEED", "")) and os.environ.get(
+ "ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE", ""
+ ).strip().lower() in ("cpu", "nvme"):
+ return True
+ config = getattr(getattr(args, "hf_deepspeed_config", None), "config", None)
+ if config is None:
+ config = getattr(args, "deepspeed", None)
+ if isinstance(config, str) and config:
+ import json
+ with open(config, "r", encoding = "utf-8") as f:
+ config = json.load(f)
+ if not isinstance(config, dict):
+ return False
+ offload = (config.get("zero_optimization", None) or {}).get("offload_optimizer", None) or {}
+ return str(offload.get("device", "none")).lower() in ("cpu", "nvme")
+
+
+def get_flash_adamw_cls_and_kwargs(args):
+ global _announced
+ # flashoptim supports only Linux with NVIDIA CUDA; ROCm also reports torch.cuda.is_available().
+ if (
+ not torch.cuda.is_available()
+ or getattr(torch.version, "hip", None)
+ or not sys.platform.startswith("linux")
+ ):
+ raise NotImplementedError(
+ "Unsloth: optim = 'adamw_flash' runs flashoptim's Triton kernels, which support only "
+ "Linux with NVIDIA CUDA GPUs. Use optim = 'adamw_8bit' or 'adamw_torch' instead."
+ )
+ if _deepspeed_offloads_optimizer(args):
+ raise ValueError(
+ "Unsloth: optim = 'adamw_flash' cannot be used with DeepSpeed ZeRO offload_optimizer, "
+ "since flashoptim has no CPU implementation. Turn off offload_optimizer or use another optim."
+ )
+ if _fsdp_version(args) == 1:
+ raise ValueError(
+ "Unsloth: optim = 'adamw_flash' does not support FSDP1 (flashoptim supports DDP and FSDP2). "
+ "Set fsdp_config = {'version': 2} or use another optim."
+ )
+ if _fsdp_version(args) is not None and _fsdp_cpu_offload(args):
+ raise ValueError(
+ "Unsloth: optim = 'adamw_flash' cannot step FSDP2 CPU offloaded parameters, since "
+ "flashoptim's Triton kernels need CUDA tensors. Turn off cpu_offload or use another optim."
+ )
+ optimizer_cls = _get_flash_adamw_class()
+ if not _announced:
+ _announced = True
+ print(
+ "Unsloth: using FlashAdamW (optim = 'adamw_flash'). To train this config with plain "
+ "AdamW, set optim = 'adamw_torch' and optim_args = None."
+ )
+ optimizer_kwargs = {
+ "lr": args.learning_rate,
+ "betas": (args.adam_beta1, args.adam_beta2),
+ "eps": args.adam_epsilon,
+ "master_weight_bits": 32,
+ "quantize": False,
+ }
+ optimizer_kwargs.update(_parse_optim_args(getattr(args, "optim_args", None)))
+ return optimizer_cls, optimizer_kwargs
+
+
+def _patch_trainer_get_optimizer_cls_and_kwargs():
+ from transformers import Trainer
+
+ original = Trainer.__dict__["get_optimizer_cls_and_kwargs"]
+ original = getattr(original, "__func__", original)
+ if getattr(original, "_unsloth_flash_adamw", False):
+ return
+
+ def get_optimizer_cls_and_kwargs(args, *other, **kwargs):
+ if _is_marked(args):
+ return get_flash_adamw_cls_and_kwargs(args)
+ if _has_marker_token(args):
+ import copy
+ args = copy.copy(args)
+ args.optim_args = _strip_marker(args.optim_args)
+ return original(args, *other, **kwargs)
+
+ get_optimizer_cls_and_kwargs._unsloth_flash_adamw = True
+ get_optimizer_cls_and_kwargs.__wrapped__ = original
+ get_optimizer_cls_and_kwargs.__doc__ = original.__doc__
+ Trainer.get_optimizer_cls_and_kwargs = staticmethod(get_optimizer_cls_and_kwargs)
+
+
+def is_flash_adamw_marked(training_args):
+ """A config rebuilt from to_dict / to_json_file carries only the optim_args marker."""
+ return _is_marked_in_optim_args(training_args)
+
+
+def unmark_flash_adamw(training_args):
+ """A config that selected adamw_flash and was then rebuilt with another optim."""
+ if getattr(training_args, "_unsloth_flash_adamw", False):
+ training_args._unsloth_flash_adamw = False
+ if _has_marker_token(training_args):
+ training_args.optim_args = _strip_marker(training_args.optim_args)
+ return training_args
+
+
+def _patch_hf_argument_parser():
+ # HfArgumentParser turns `optim: OptimizerNames | str` into an OptimizerNames choice list,
+ # so `--optim adamw_flash` is rejected before an Unsloth config can translate it.
+ try:
+ from transformers import HfArgumentParser
+ except ImportError:
+ return
+ original = getattr(HfArgumentParser, "_add_dataclass_arguments", None)
+ if original is None or getattr(original, "_unsloth_flash_adamw", False):
+ return
+
+ def _add_dataclass_arguments(self, dtype, *args, **kwargs):
+ result = original(self, dtype, *args, **kwargs)
+ if not getattr(dtype, "_unsloth_accepts_adamw_flash", False):
+ return result
+ for action in self._actions:
+ if action.dest != "optim" or not action.choices or FLASH_ADAMW_NAME in action.choices:
+ continue
+ action.choices = list(action.choices) + [FLASH_ADAMW_NAME]
+ convert = action.type
+ if callable(convert):
+ action.type = lambda value, convert = convert: (
+ FLASH_ADAMW_NAME if is_flash_adamw(value) else convert(value)
+ )
+ return result
+
+ _add_dataclass_arguments._unsloth_flash_adamw = True
+ HfArgumentParser._add_dataclass_arguments = _add_dataclass_arguments
+
+
+_patch_hf_argument_parser()
+
+
+def mark_flash_adamw(training_args):
+ """Called after TrainingArguments.__init__ ran with the placeholder optim."""
+ training_args._unsloth_flash_adamw = True
+ if not _is_marked_in_optim_args(training_args):
+ optim_args = getattr(training_args, "optim_args", None)
+ # key=value form: transformers' own optim_args parser splits every item on "=".
+ marker = f"{_FLASH_ADAMW_MARKER}=1"
+ training_args.optim_args = f"{marker},{optim_args}" if optim_args else marker
+ _patch_trainer_get_optimizer_cls_and_kwargs()
+ return training_args
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index 8bcc542b3fa..a238efcdb14 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -27,6 +27,13 @@ from trl import SFTTrainer
# Bypass the partially-initialised unsloth namespace during the _gpu_init load.
from .models._utils import is_bfloat16_supported
+from unsloth.optimizers.flash_adamw import (
+ _FLASH_ADAMW_PLACEHOLDER,
+ is_flash_adamw,
+ is_flash_adamw_marked,
+ mark_flash_adamw,
+ unmark_flash_adamw,
+)
from unsloth.utils import (
configure_padding_free,
configure_sample_packing,
@@ -925,6 +932,8 @@ class QGaloreConfig:
class UnslothTrainingArguments(TrainingArguments):
+ _unsloth_accepts_adamw_flash = True
+
def __init__(
self,
embedding_learning_rate: float = None,
@@ -941,6 +950,18 @@ class UnslothTrainingArguments(TrainingArguments):
self.q_galore_config = q_galore_config
self.embedding_learning_rate = embedding_learning_rate
self.loraplus_lr_ratio = loraplus_lr_ratio
+ # Imported here too: tests exec this class body in a namespace without the module imports.
+ from unsloth.optimizers.flash_adamw import (
+ _FLASH_ADAMW_PLACEHOLDER,
+ is_flash_adamw,
+ is_flash_adamw_marked,
+ mark_flash_adamw,
+ unmark_flash_adamw,
+ )
+
+ flash_adamw = is_flash_adamw(kwargs.get("optim", None))
+ if flash_adamw:
+ kwargs["optim"] = _FLASH_ADAMW_PLACEHOLDER
# Not fields of the pristine SFTConfig captured at import.
context_parallel = {
"context_parallel_size": kwargs.pop("context_parallel_size", 1),
@@ -954,6 +975,10 @@ class UnslothTrainingArguments(TrainingArguments):
super().__init__(*args, **kwargs)
for key, value in context_parallel.items():
setattr(self, key, value)
+ if flash_adamw or is_flash_adamw_marked(self):
+ mark_flash_adamw(self)
+ else:
+ unmark_flash_adamw(self)
self.embedding_learning_rate = embedding_learning_rate
self.loraplus_lr_ratio = loraplus_lr_ratio
if self.eval_steps is not None and self.eval_strategy != "steps":