Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add optim = "adamw_flash" (FlashAdamW from flashoptim) for lower optimizer memory (#13881)
Daniel Han313bfd9e88f9
diff --git a/tests/utils/test_flash_adamw_optim.py b/tests/utils/test_flash_adamw_optim.pynew file mode 100644index 00000000000..9fd9272362d--- /dev/null+++ b/tests/utils/test_flash_adamw_optim.py@@ -0,0 +1,406 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import os+import sys+import types++import pytest++import unsloth # noqa: F401 (must precede transformers/trl)+import torch++OUT = os.path.join(os.environ.get("TMPDIR", "."), "unsloth-flash-adamw-test")+++def stub_flashoptim(monkeypatch):+ import unsloth.optimizers.flash_adamw as flash_adamw++ calls = []++ class FlashAdamW(torch.optim.Optimizer):+ def __init__(+ self,+ params,+ lr = 1e-3,+ betas = (0.9, 0.999),+ eps = 1e-8,+ weight_decay = 1e-2,+ master_weight_bits = 24,+ **kwargs,+ ):+ calls.append(+ dict(lr = lr, betas = betas, eps = eps, master_weight_bits = master_weight_bits, **kwargs)+ )+ # flashoptim's own defaults carry initial_lr = the constructor lr.+ super().__init__(params, dict(lr = lr, initial_lr = lr, weight_decay = weight_decay))++ module = types.ModuleType("flashoptim")+ module.FlashAdamW = FlashAdamW+ module.__version__ = "0.1.4"+ monkeypatch.setitem(sys.modules, "flashoptim", module)+ monkeypatch.setattr(flash_adamw, "_FLASH_ADAMW_CLASS", None)+ yield FlashAdamW, calls+ flash_adamw._FLASH_ADAMW_CLASS = None+++def _sft_config(**kwargs):+ from trl import SFTConfig+ return SFTConfig(output_dir = OUT, report_to = [], **kwargs)+++def _unsloth_args(**kwargs):+ from unsloth.trainer import UnslothTrainingArguments+ return UnslothTrainingArguments(output_dir = OUT, report_to = [], **kwargs)++[email protected]("make_args", [_sft_config, _unsloth_args])+def test_adamw_flash_selects_flashoptim(stub_flashoptim, make_args, monkeypatch):+ from transformers import Trainer++ FlashAdamW, calls = stub_flashoptim+ args = make_args(+ optim = "adamw_flash",+ learning_rate = 3e-4,+ adam_beta2 = 0.95,+ optim_args = "master_weight_bits=24, quantize=True",+ )+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(args)+ assert issubclass(optimizer_cls, FlashAdamW)+ assert optimizer_kwargs["lr"] == 3e-4+ assert optimizer_kwargs["betas"] == (0.9, 0.95)+ assert optimizer_kwargs["master_weight_bits"] == 24+ assert optimizer_kwargs["quantize"] is True++ bf16 = torch.nn.Parameter(torch.zeros(4, dtype = torch.bfloat16))+ optimizer_cls([{"params": [bf16], "weight_decay": 0.1}], **optimizer_kwargs)+ assert calls[-1]["master_weight_bits"] == 24+ # flashoptim rejects master_weight_bits on all-fp32 params (LoRA adapters).+ fp32 = torch.nn.Parameter(torch.zeros(4))+ optimizer_cls([{"params": [fp32], "weight_decay": 0.0}], **optimizer_kwargs)+ assert calls[-1]["master_weight_bits"] is None+++def test_adamw_flash_missing_dependency(monkeypatch):+ import unsloth.optimizers.flash_adamw as flash_adamw+ from transformers import Trainer++ monkeypatch.setitem(sys.modules, "flashoptim", None)+ monkeypatch.setattr(flash_adamw, "_FLASH_ADAMW_CLASS", None)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ args = _sft_config(optim = "adamw_flash")+ with pytest.raises(ImportError, match = "pip install 'flashoptim"):+ Trainer.get_optimizer_cls_and_kwargs(args)+++def test_other_optimizers_untouched():+ from transformers import Trainer++ args = _sft_config(optim = "adamw_torch")+ assert not getattr(args, "_unsloth_flash_adamw", False)+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(args)+ assert optimizer_cls is torch.optim.AdamW++[email protected]("make_args", [_sft_config, _unsloth_args])+def test_adamw_flash_survives_dataclasses_replace(stub_flashoptim, make_args, monkeypatch):+ import dataclasses+ from transformers import Trainer++ FlashAdamW, _ = stub_flashoptim+ args = make_args(optim = "adamw_flash")+ copied = dataclasses.replace(args, learning_rate = 1e-5)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(copied)+ assert issubclass(optimizer_cls, FlashAdamW)+ assert optimizer_kwargs["lr"] == 1e-5+ # Default: full 32 bit master semantics and unquantized moments.+ assert optimizer_kwargs["master_weight_bits"] == 32+ assert optimizer_kwargs["quantize"] is False++[email protected]("make_args", [_sft_config, _unsloth_args])+def test_adamw_flash_marker_parses_in_transformers(make_args):+ trainer_module = __import__("transformers.trainer", fromlist = ["_parse_optim_args"])+ parse = getattr(trainer_module, "_parse_optim_args", None)+ if parse is None:+ pytest.skip("transformers has no _parse_optim_args")+ args = make_args(optim = "adamw_flash", optim_args = "quantize=True")+ parsed = parse(args.optim_args)+ assert parsed["quantize"] == "True"+++def _flash_kwargs_args(**kwargs):+ defaults = dict(+ learning_rate = 1e-4,+ adam_beta1 = 0.9,+ adam_beta2 = 0.999,+ adam_epsilon = 1e-8,+ optim_args = None,+ fsdp = [],+ fsdp_config = {},+ )+ defaults.update(kwargs)+ return types.SimpleNamespace(**defaults)++[email protected]("make_args", [_sft_config, _unsloth_args])+def test_adamw_flash_restored_from_to_dict(stub_flashoptim, make_args, monkeypatch):+ import dataclasses+ import inspect+ from transformers import Trainer++ FlashAdamW, _ = stub_flashoptim+ saved = make_args(optim = "adamw_flash").to_dict()+ assert saved["optim"] == "adamw_torch"+ # A fresh process has the stock Trainer method until a marked config is built.+ patched = Trainer.__dict__["get_optimizer_cls_and_kwargs"].__func__+ original = inspect.getclosurevars(patched).nonlocals["original"]+ monkeypatch.setattr(Trainer, "get_optimizer_cls_and_kwargs", staticmethod(original))+ cls = type(make_args(optim = "adamw_torch"))+ init_fields = {f.name for f in dataclasses.fields(cls) if f.init}+ rebuilt = cls(+ **{k: v for k, v in saved.items() if k in init_fields and not k.endswith("_token")}+ )+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(rebuilt)+ assert issubclass(optimizer_cls, FlashAdamW)+++def test_adamw_flash_keeps_per_group_initial_lr(stub_flashoptim, monkeypatch):+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(+ _flash_kwargs_args(optim_args = "decouple_lr=True")+ )+ groups = [+ {"params": [torch.nn.Parameter(torch.zeros(4))], "lr": 1e-4},+ {"params": [torch.nn.Parameter(torch.zeros(4))], "lr": 5e-6},+ {"params": [torch.nn.Parameter(torch.zeros(4))], "lr": 1.6e-3},+ ]+ optimizer = optimizer_cls(groups, **optimizer_kwargs)+ assert [g["initial_lr"] for g in optimizer.param_groups] == [1e-4, 5e-6, 1.6e-3]+ scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda step: 1.0)+ assert scheduler.base_lrs == [1e-4, 5e-6, 1.6e-3]+++def test_adamw_flash_rejects_mixed_dtype_uncompressed_quantized(stub_flashoptim, monkeypatch):+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ params = [+ torch.nn.Parameter(torch.zeros(4, dtype = torch.bfloat16)),+ torch.nn.Parameter(torch.zeros(4)),+ ]+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(+ _flash_kwargs_args(optim_args = "quantize=True,compress_state_dict=False")+ )+ with pytest.raises(ValueError, match = "compress_state_dict"):+ optimizer_cls([{"params": params}], **optimizer_kwargs)+ for optim_args in (None, "quantize=True"):+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(+ _flash_kwargs_args(optim_args = optim_args)+ )+ optimizer_cls([{"params": params}], **optimizer_kwargs)+++ "fsdp_config, env, rejected",+ [+ ({"version": 1}, {}, True),+ ({"version": 2}, {}, False),+ (None, {"ACCELERATE_USE_FSDP": "true"}, True),+ (None, {"ACCELERATE_USE_FSDP": "true", "FSDP_VERSION": "2"}, False),+ (None, {}, False),+ ],+)+def test_adamw_flash_rejects_fsdp1(stub_flashoptim, monkeypatch, fsdp_config, env, rejected):+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ for key in ("ACCELERATE_USE_FSDP", "FSDP_VERSION", "FSDP_OFFLOAD_PARAMS"):+ monkeypatch.delenv(key, raising = False)+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ args = _flash_kwargs_args(+ fsdp = ["full_shard"] if fsdp_config else [], fsdp_config = fsdp_config or {}+ )+ if rejected:+ with pytest.raises(ValueError, match = "FSDP1"):+ flash_adamw.get_flash_adamw_cls_and_kwargs(args)+ else:+ flash_adamw.get_flash_adamw_cls_and_kwargs(args)++[email protected]("hip, platform", [("6.2.0", "linux"), (None, "win32")])+def test_adamw_flash_rejects_unsupported_platform(stub_flashoptim, monkeypatch, hip, platform):+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)+ monkeypatch.setattr(sys, "platform", platform)+ with pytest.raises(NotImplementedError, match = "NVIDIA CUDA"):+ flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args())++[email protected]("make_args", [_sft_config, _unsloth_args])+def test_adamw_flash_later_optim_change_wins(stub_flashoptim, make_args, monkeypatch):+ import dataclasses+ from transformers import Trainer++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ args = make_args(optim = "adamw_flash", optim_args = "quantize=True")+ replaced = dataclasses.replace(args, optim = "rmsprop")+ assert "unsloth_adamw_flash" not in str(replaced.optim_args)+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(replaced)+ assert optimizer_cls is torch.optim.RMSprop+ args.optim = "sgd"+ args.optim_args = None+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(args)+ assert optimizer_cls is torch.optim.SGD+++def test_adamw_flash_keeps_one_shot_param_iterables(stub_flashoptim, monkeypatch):+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ model = torch.nn.Linear(4, 4)+ optimizer_cls, optimizer_kwargs = flash_adamw.get_flash_adamw_cls_and_kwargs(+ _flash_kwargs_args()+ )+ optimizer = optimizer_cls([{"params": model.parameters(), "lr": 1e-4}], **optimizer_kwargs)+ assert len(optimizer.param_groups[0]["params"]) == 2+ optimizer = optimizer_cls(model.parameters(), **optimizer_kwargs)+ assert len(optimizer.param_groups[0]["params"]) == 2+++ "fsdp, fsdp_config, env",+ [+ (["full_shard"], {"version": 2, "cpu_offload": True}, {}),+ ("full_shard offload", {"version": 2}, {}),+ (+ [],+ None,+ {"ACCELERATE_USE_FSDP": "true", "FSDP_VERSION": "2", "FSDP_OFFLOAD_PARAMS": "true"},+ ),+ ],+)+def test_adamw_flash_rejects_fsdp2_cpu_offload(+ stub_flashoptim, monkeypatch, fsdp, fsdp_config, env+):+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ for key in ("ACCELERATE_USE_FSDP", "FSDP_VERSION", "FSDP_OFFLOAD_PARAMS"):+ monkeypatch.delenv(key, raising = False)+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ args = _flash_kwargs_args(fsdp = fsdp, fsdp_config = fsdp_config or {})+ with pytest.raises(ValueError, match = "CPU offloaded"):+ flash_adamw.get_flash_adamw_cls_and_kwargs(args)++[email protected]("make_args", [_sft_config, _unsloth_args])+def test_adamw_flash_accepted_by_hf_argument_parser(make_args):+ from transformers import HfArgumentParser++ cls = type(make_args(optim = "adamw_torch"))+ (args,) = HfArgumentParser(cls).parse_args_into_dataclasses(+ ["--output_dir", OUT, "--report_to", "none", "--optim", "adamw_flash"]+ )+ assert getattr(args, "_unsloth_flash_adamw", False)+ with pytest.raises(SystemExit):+ HfArgumentParser(cls).parse_args_into_dataclasses(+ ["--output_dir", OUT, "--report_to", "none", "--optim", "not_an_optim"]+ )++[email protected]("make_args", [_sft_config, _unsloth_args])+def test_adamw_flash_switch_back_to_adamw(stub_flashoptim, make_args, monkeypatch):+ import dataclasses+ from transformers import Trainer++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ args = make_args(optim = "adamw_flash")+ replaced = dataclasses.replace(args, optim = "adamw_torch", optim_args = None)+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(replaced)+ assert optimizer_cls is torch.optim.AdamW++[email protected]("make_args", [_sft_config, _unsloth_args])+def test_adamw_flash_switch_back_in_place(stub_flashoptim, make_args, monkeypatch):+ from transformers import Trainer++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ args = make_args(optim = "adamw_flash")+ args.optim = "adamw_torch"+ args.optim_args = None+ optimizer_cls, _ = Trainer.get_optimizer_cls_and_kwargs(args)+ assert optimizer_cls is torch.optim.AdamW+++def test_adamw_flash_platform_checked_before_import(monkeypatch):+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setitem(sys.modules, "flashoptim", None)+ monkeypatch.setattr(flash_adamw, "_FLASH_ADAMW_CLASS", None)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ with pytest.raises(NotImplementedError, match = "NVIDIA CUDA"):+ flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args())+++ "deepspeed, env",+ [+ ({"zero_optimization": {"stage": 2, "offload_optimizer": {"device": "cpu"}}}, {}),+ ({"zero_optimization": {"stage": 3, "offload_optimizer": {"device": "nvme"}}}, {}),+ (+ None,+ {+ "ACCELERATE_USE_DEEPSPEED": "true",+ "ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE": "cpu",+ },+ ),+ ],+)+def test_adamw_flash_rejects_deepspeed_optimizer_offload(+ stub_flashoptim, monkeypatch, deepspeed, env+):+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ for key in ("ACCELERATE_USE_DEEPSPEED", "ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE"):+ monkeypatch.delenv(key, raising = False)+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ with pytest.raises(ValueError, match = "offload_optimizer"):+ flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args(deepspeed = deepspeed))+ no_offload = {"zero_optimization": {"stage": 2, "offload_optimizer": {"device": "none"}}}+ monkeypatch.delenv("ACCELERATE_USE_DEEPSPEED", raising = False)+ flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args(deepspeed = no_offload))+++ "optim_args", ["quantize=ture", "compress_state_dict=tru", "check_numerics=2"]+)+def test_adamw_flash_rejects_invalid_booleans(optim_args):+ import unsloth.optimizers.flash_adamw as flash_adamw+ with pytest.raises(ValueError, match = "not a boolean"):+ flash_adamw._parse_optim_args(optim_args)+ assert flash_adamw._parse_optim_args("quantize=False,decouple_lr=no") == {+ "quantize": False,+ "decouple_lr": False,+ }+++def test_adamw_flash_requires_compatible_flashoptim(stub_flashoptim, monkeypatch):+ import unsloth.optimizers.flash_adamw as flash_adamw++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(sys.modules["flashoptim"], "__version__", "0.1.1")+ with pytest.raises(ImportError, match = "flashoptim>=0.1.4, found 0.1.1"):+ flash_adamw.get_flash_adamw_cls_and_kwargs(_flash_kwargs_args())diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 3cac0f24602..fcf0bf3a70e 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -537,6 +537,7 @@ class Unsloth{RLConfig_name}({RLConfig_name}):"""{__RLConfig_doc__}"""+ _unsloth_accepts_adamw_flash = Truevllm_sampling_params: Optional[Any] = field(default = None,metadata = {{'help': 'vLLM SamplingParams'}},@@ -3294,6 +3295,10 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):"if fsdp and isinstance(_unsloth_fsdp_cfg, dict):\n"f" _unsloth_fsdp2 = _unsloth_fsdp2 or str(_unsloth_fsdp_cfg.get('fsdp_version', _unsloth_fsdp_cfg.get('version', {_FSDP_DEFAULT_VERSION}))) == '2'\n")+ extra_args += (+ "_unsloth_flash_adamw = str(getattr(optim, 'value', optim)).strip().lower() == 'adamw_flash'\n"+ "if _unsloth_flash_adamw: optim = 'adamw_torch'\n"+ )extra_args += ("if _unsloth_fsdp2 and str(getattr(optim, 'value', optim)) == 'adamw_8bit':\n"" print(\"Unsloth: FSDP2 shards parameters as DTensors, which bitsandbytes' 8-bit optimizers cannot step. Switching optim = 'adamw_8bit' to 'adamw_torch_fused'.\")\n"@@ -3553,6 +3558,15 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):" self.gradient_checkpointing_kwargs = _gc_kwargs\n")+ if "optim" in call_args:+ RLConfig_post += (+ " from unsloth.optimizers.flash_adamw import is_flash_adamw_marked, mark_flash_adamw, unmark_flash_adamw\n"+ " if _unsloth_flash_adamw or is_flash_adamw_marked(self):\n"+ " mark_flash_adamw(self)\n"+ " else:\n"+ " unmark_flash_adamw(self)\n"+ )+RLTrainer_extras = patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, imports)diff --git a/unsloth/optimizers/flash_adamw.py b/unsloth/optimizers/flash_adamw.pynew file mode 100644index 00000000000..a215dee4845--- /dev/null+++ b/unsloth/optimizers/flash_adamw.py@@ -0,0 +1,366 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""`optim = "adamw_flash"`: FlashAdamW from databricks/flashoptim (arXiv 2602.23349).++bf16 / fp16 parameters keep a 16 bit error correction term in the optimizer state, so+the effective master weight is fp32, and the Adam moments are stored in the parameter+dtype: about 10 bytes per parameter (with bf16 grads) instead of 16 for bf16 weights+plus fp32 master plus fp32 moments. This default tracked fp32 master AdamW on a real+Qwen2.5-0.5B full finetune. optim_args = "quantize=True" stores the moments in 8 bits+(about 8 bytes per parameter; with "master_weight_bits=24", flashoptim's own default,+about 7), but its loss curve moved measurably away from AdamW there.+fp32 parameters (LoRA adapters) have no correction term; for them only quantize=True+saves memory.++transformers validates `optim` against OptimizerNames, so the config stores+`adamw_torch` and marks itself with `_unsloth_flash_adamw` plus a token in `optim_args`+(a dataclass field, so `dataclasses.replace` keeps it); the patched+`Trainer.get_optimizer_cls_and_kwargs` reads either mark. Because the placeholder is a real+optim name, `dataclasses.replace(args, optim = "adamw_torch")` still selects FlashAdamW;+switch back with `optim_args = None` (or `unmark_flash_adamw(args)`).+"""++__all__ = [+ "FLASH_ADAMW_NAME",+ "is_flash_adamw",+ "mark_flash_adamw",+ "unmark_flash_adamw",+ "get_flash_adamw_cls_and_kwargs",+]++import os+import sys++import torch++FLASH_ADAMW_NAME = "adamw_flash"+_FLASH_ADAMW_PLACEHOLDER = "adamw_torch"+_FLASH_ADAMW_MARKER = "unsloth_adamw_flash"+_FLASH_ADAMW_CLASS = None+_FLASH_ADAMW_MIN_VERSION = "0.1.4"+_TRUE_VALUES = ("1", "true", "yes")+_FALSE_VALUES = ("0", "false", "no")+_announced = False+++def is_flash_adamw(optim):+ return str(getattr(optim, "value", optim)).strip().lower() == FLASH_ADAMW_NAME+++def _parse_optim_args(optim_args):+ parsed = {}+ if not optim_args:+ return parsed+ for item in str(optim_args).replace(" ", "").split(","):+ if not item:+ continue+ key, _, value = item.partition("=")+ if key == _FLASH_ADAMW_MARKER:+ continue+ if key == "master_weight_bits":+ parsed[key] = None if value.lower() in ("none", "0", "") else int(value)+ elif key in ("quantize", "compress_state_dict", "check_numerics", "decouple_lr"):+ if value.lower() not in _TRUE_VALUES + _FALSE_VALUES:+ raise ValueError(+ f"Unsloth: optim = 'adamw_flash' optim_args {key}={value!r} is not a boolean. "+ "Use true / false, 1 / 0 or yes / no."+ )+ parsed[key] = value.lower() in _TRUE_VALUES+ else:+ raise ValueError(+ f"Unsloth: optim = 'adamw_flash' does not understand optim_args key {key!r}. "+ "Supported: master_weight_bits=24|32|none, quantize, compress_state_dict, check_numerics, decouple_lr."+ )+ return parsed+++def _get_flash_adamw_class():+ global _FLASH_ADAMW_CLASS+ if _FLASH_ADAMW_CLASS is not None:+ return _FLASH_ADAMW_CLASS+ try:+ import flashoptim+ from flashoptim import FlashAdamW+ except ImportError as error:+ raise ImportError(+ "Unsloth: optim = 'adamw_flash' needs the flashoptim package. "+ f"Install it via `pip install 'flashoptim>={_FLASH_ADAMW_MIN_VERSION}'`."+ ) from error+ from packaging.version import Version++ version = getattr(flashoptim, "__version__", None)+ if version is None:+ import importlib.metadata+ version = importlib.metadata.version("flashoptim")+ # 0.1.4 made compress_state_dict default to True, which the resume guard below relies on.+ if Version(version) < Version(_FLASH_ADAMW_MIN_VERSION):+ raise ImportError(+ f"Unsloth: optim = 'adamw_flash' needs flashoptim>={_FLASH_ADAMW_MIN_VERSION}, found {version}. "+ f"Upgrade via `pip install -U 'flashoptim>={_FLASH_ADAMW_MIN_VERSION}'`."+ )++ class UnslothFlashAdamW(FlashAdamW):+ def __init__(+ self,+ params,+ *args,+ master_weight_bits = 32,+ **kwargs,+ ):+ params = [+ {**group, "params": _materialize_params(group["params"])}+ if isinstance(group, dict) and "params" in group+ else group+ for group in params+ ]+ # flashoptim seeds every group's initial_lr with the constructor lr, and the LR+ # scheduler then uses it as the group's base lr (setdefault keeps it), which would+ # drop embedding_learning_rate and loraplus_lr_ratio, and skew decouple_lr decay.+ params = [+ {**group, "initial_lr": group.get("initial_lr", group["lr"])}+ if isinstance(group, dict) and "lr" in group+ else group+ for group in params+ ]+ # flashoptim raises when every parameter is fp32 and master_weight_bits is set,+ # which is the normal LoRA case: there only the 8 bit moments apply.+ tensors = [+ p+ for group in params+ for p in (group["params"] if isinstance(group, dict) else [group])+ ]+ if tensors and all(p.dtype == torch.float32 for p in tensors):+ master_weight_bits = None+ # flashoptim's load_state_dict re-quantizes uncompressed moments with one+ # optimizer wide bf16 check, which corrupts the other dtype's scales on resume.+ if (+ kwargs.get("quantize", True)+ and not kwargs.get("compress_state_dict", True)+ and len({p.dtype == torch.bfloat16 for p in tensors}) > 1+ ):+ raise ValueError(+ "Unsloth: optim = 'adamw_flash' with quantize=True and compress_state_dict=False "+ "cannot resume a model mixing bf16 and other trainable dtypes. "+ "Drop compress_state_dict=False or quantize=True."+ )+ super().__init__(params, *args, master_weight_bits = master_weight_bits, **kwargs)++ _FLASH_ADAMW_CLASS = UnslothFlashAdamW+ return _FLASH_ADAMW_CLASS+++def _materialize_params(params):+ # A one-shot iterable (model.parameters()) would be exhausted by the dtype scan below.+ return [params] if isinstance(params, torch.Tensor) else list(params)+++def _has_marker_token(args):+ optim_args = str(getattr(args, "optim_args", None) or "").replace(" ", "")+ return any(item.partition("=")[0] == _FLASH_ADAMW_MARKER for item in optim_args.split(","))+++def _has_placeholder_optim(args):+ optim = getattr(args, "optim", None)+ return str(getattr(optim, "value", optim)).strip().lower() == _FLASH_ADAMW_PLACEHOLDER+++def _is_marked_in_optim_args(args):+ # The mark only counts while optim is still the placeholder, so a later optim change wins.+ return _has_placeholder_optim(args) and _has_marker_token(args)+++def _is_marked(args):+ # The optim_args token alone decides, so an in place `args.optim_args = None` switches back.+ return _is_marked_in_optim_args(args)+++def _strip_marker(optim_args):+ items = str(optim_args or "").split(",")+ kept = [+ i+ for i in items+ if i.strip() and i.replace(" ", "").partition("=")[0] != _FLASH_ADAMW_MARKER+ ]+ return ",".join(kept) or None+++def _is_truthy(value):+ return str(value).strip().lower() in ("1", "true", "yes")+++def _fsdp_cpu_offload(args):+ fsdp = getattr(args, "fsdp", None)+ items = (+ fsdp.split() if isinstance(fsdp, str) else (fsdp if isinstance(fsdp, (list, tuple)) else [])+ )+ if any(str(getattr(item, "value", item)).lower() == "offload" for item in items):+ return True+ config = getattr(args, "fsdp_config", None)+ config = config if isinstance(config, dict) else {}+ if any(+ _is_truthy(config.get(key, False))+ for key in ("cpu_offload", "offload_params", "fsdp_offload_params")+ ):+ return True+ return _is_truthy(os.environ.get("FSDP_OFFLOAD_PARAMS", ""))+++def _fsdp_version(args):+ if getattr(args, "fsdp", None):+ config = getattr(args, "fsdp_config", None)+ config = config if isinstance(config, dict) else {}+ version = config.get("version", config.get("fsdp_version", None))+ if version is None:+ import transformers+ from packaging.version import Version+ version = 2 if Version(transformers.__version__) >= Version("5.10.0") else 1+ return int(version)+ if os.environ.get("ACCELERATE_USE_FSDP", "").strip().lower() in ("1", "true", "yes"):+ return int(os.environ.get("FSDP_VERSION", "").strip() or 1)+ return None+++def _deepspeed_offloads_optimizer(args):+ if _is_truthy(os.environ.get("ACCELERATE_USE_DEEPSPEED", "")) and os.environ.get(+ "ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE", ""+ ).strip().lower() in ("cpu", "nvme"):+ return True+ config = getattr(getattr(args, "hf_deepspeed_config", None), "config", None)+ if config is None:+ config = getattr(args, "deepspeed", None)+ if isinstance(config, str) and config:+ import json+ with open(config, "r", encoding = "utf-8") as f:+ config = json.load(f)+ if not isinstance(config, dict):+ return False+ offload = (config.get("zero_optimization", None) or {}).get("offload_optimizer", None) or {}+ return str(offload.get("device", "none")).lower() in ("cpu", "nvme")+++def get_flash_adamw_cls_and_kwargs(args):+ global _announced+ # flashoptim supports only Linux with NVIDIA CUDA; ROCm also reports torch.cuda.is_available().+ if (+ not torch.cuda.is_available()+ or getattr(torch.version, "hip", None)+ or not sys.platform.startswith("linux")+ ):+ raise NotImplementedError(+ "Unsloth: optim = 'adamw_flash' runs flashoptim's Triton kernels, which support only "+ "Linux with NVIDIA CUDA GPUs. Use optim = 'adamw_8bit' or 'adamw_torch' instead."+ )+ if _deepspeed_offloads_optimizer(args):+ raise ValueError(+ "Unsloth: optim = 'adamw_flash' cannot be used with DeepSpeed ZeRO offload_optimizer, "+ "since flashoptim has no CPU implementation. Turn off offload_optimizer or use another optim."+ )+ if _fsdp_version(args) == 1:+ raise ValueError(+ "Unsloth: optim = 'adamw_flash' does not support FSDP1 (flashoptim supports DDP and FSDP2). "+ "Set fsdp_config = {'version': 2} or use another optim."+ )+ if _fsdp_version(args) is not None and _fsdp_cpu_offload(args):+ raise ValueError(+ "Unsloth: optim = 'adamw_flash' cannot step FSDP2 CPU offloaded parameters, since "+ "flashoptim's Triton kernels need CUDA tensors. Turn off cpu_offload or use another optim."+ )+ optimizer_cls = _get_flash_adamw_class()+ if not _announced:+ _announced = True+ print(+ "Unsloth: using FlashAdamW (optim = 'adamw_flash'). To train this config with plain "+ "AdamW, set optim = 'adamw_torch' and optim_args = None."+ )+ optimizer_kwargs = {+ "lr": args.learning_rate,+ "betas": (args.adam_beta1, args.adam_beta2),+ "eps": args.adam_epsilon,+ "master_weight_bits": 32,+ "quantize": False,+ }+ optimizer_kwargs.update(_parse_optim_args(getattr(args, "optim_args", None)))+ return optimizer_cls, optimizer_kwargs+++def _patch_trainer_get_optimizer_cls_and_kwargs():+ from transformers import Trainer++ original = Trainer.__dict__["get_optimizer_cls_and_kwargs"]+ original = getattr(original, "__func__", original)+ if getattr(original, "_unsloth_flash_adamw", False):+ return++ def get_optimizer_cls_and_kwargs(args, *other, **kwargs):+ if _is_marked(args):+ return get_flash_adamw_cls_and_kwargs(args)+ if _has_marker_token(args):+ import copy+ args = copy.copy(args)+ args.optim_args = _strip_marker(args.optim_args)+ return original(args, *other, **kwargs)++ get_optimizer_cls_and_kwargs._unsloth_flash_adamw = True+ get_optimizer_cls_and_kwargs.__wrapped__ = original+ get_optimizer_cls_and_kwargs.__doc__ = original.__doc__+ Trainer.get_optimizer_cls_and_kwargs = staticmethod(get_optimizer_cls_and_kwargs)+++def is_flash_adamw_marked(training_args):+ """A config rebuilt from to_dict / to_json_file carries only the optim_args marker."""+ return _is_marked_in_optim_args(training_args)+++def unmark_flash_adamw(training_args):+ """A config that selected adamw_flash and was then rebuilt with another optim."""+ if getattr(training_args, "_unsloth_flash_adamw", False):+ training_args._unsloth_flash_adamw = False+ if _has_marker_token(training_args):+ training_args.optim_args = _strip_marker(training_args.optim_args)+ return training_args+++def _patch_hf_argument_parser():+ # HfArgumentParser turns `optim: OptimizerNames | str` into an OptimizerNames choice list,+ # so `--optim adamw_flash` is rejected before an Unsloth config can translate it.+ try:+ from transformers import HfArgumentParser+ except ImportError:+ return+ original = getattr(HfArgumentParser, "_add_dataclass_arguments", None)+ if original is None or getattr(original, "_unsloth_flash_adamw", False):+ return++ def _add_dataclass_arguments(self, dtype, *args, **kwargs):+ result = original(self, dtype, *args, **kwargs)+ if not getattr(dtype, "_unsloth_accepts_adamw_flash", False):+ return result+ for action in self._actions:+ if action.dest != "optim" or not action.choices or FLASH_ADAMW_NAME in action.choices:+ continue+ action.choices = list(action.choices) + [FLASH_ADAMW_NAME]+ convert = action.type+ if callable(convert):+ action.type = lambda value, convert = convert: (+ FLASH_ADAMW_NAME if is_flash_adamw(value) else convert(value)+ )+ return result++ _add_dataclass_arguments._unsloth_flash_adamw = True+ HfArgumentParser._add_dataclass_arguments = _add_dataclass_arguments+++_patch_hf_argument_parser()+++def mark_flash_adamw(training_args):+ """Called after TrainingArguments.__init__ ran with the placeholder optim."""+ training_args._unsloth_flash_adamw = True+ if not _is_marked_in_optim_args(training_args):+ optim_args = getattr(training_args, "optim_args", None)+ # key=value form: transformers' own optim_args parser splits every item on "=".+ marker = f"{_FLASH_ADAMW_MARKER}=1"+ training_args.optim_args = f"{marker},{optim_args}" if optim_args else marker+ _patch_trainer_get_optimizer_cls_and_kwargs()+ return training_argsdiff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 8bcc542b3fa..a238efcdb14 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -27,6 +27,13 @@ from trl import SFTTrainer# Bypass the partially-initialised unsloth namespace during the _gpu_init load.from .models._utils import is_bfloat16_supported+from unsloth.optimizers.flash_adamw import (+ _FLASH_ADAMW_PLACEHOLDER,+ is_flash_adamw,+ is_flash_adamw_marked,+ mark_flash_adamw,+ unmark_flash_adamw,+)from unsloth.utils import (configure_padding_free,configure_sample_packing,@@ -925,6 +932,8 @@ class QGaloreConfig:class UnslothTrainingArguments(TrainingArguments):+ _unsloth_accepts_adamw_flash = True+def __init__(self,embedding_learning_rate: float = None,@@ -941,6 +950,18 @@ class UnslothTrainingArguments(TrainingArguments):self.q_galore_config = q_galore_configself.embedding_learning_rate = embedding_learning_rateself.loraplus_lr_ratio = loraplus_lr_ratio+ # Imported here too: tests exec this class body in a namespace without the module imports.+ from unsloth.optimizers.flash_adamw import (+ _FLASH_ADAMW_PLACEHOLDER,+ is_flash_adamw,+ is_flash_adamw_marked,+ mark_flash_adamw,+ unmark_flash_adamw,+ )++ flash_adamw = is_flash_adamw(kwargs.get("optim", None))+ if flash_adamw:+ kwargs["optim"] = _FLASH_ADAMW_PLACEHOLDER# Not fields of the pristine SFTConfig captured at import.context_parallel = {"context_parallel_size": kwargs.pop("context_parallel_size", 1),@@ -954,6 +975,10 @@ class UnslothTrainingArguments(TrainingArguments):super().__init__(*args, **kwargs)for key, value in context_parallel.items():setattr(self, key, value)+ if flash_adamw or is_flash_adamw_marked(self):+ mark_flash_adamw(self)+ else:+ unmark_flash_adamw(self)self.embedding_learning_rate = embedding_learning_rateself.loraplus_lr_ratio = loraplus_lr_ratioif self.eval_steps is not None and self.eval_strategy != "steps":