Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Optimizer Zoo — v0.41.0 Part A.
Expands optimizer support beyond HF Trainer's built-in set to cover LlamaFactory
+ Axolotl parity: BAdam / APOLLO / Adam-mini / lomo / adalomo / grokadamw /
schedule_free / muon / dion / came_pytorch / ao_adamw_{fp8,4bit,8bit}.
Schema-level allowlist + dependency advisory. Live wiring delegates to HF
Trainer (which accepts arbitrary optimizer strings via TrainingArguments) plus
optional adapter packages installed alongside Soup.
Security:
- Closed allowlist of optimizer names (no arbitrary string accepted at
schema level — prevents YAML-driven typos from silently using the
default).
- Empty / null-byte / non-string optimizer rejected.
- Length cap (64 chars) — defence-in-depth against absurd inputs.
"""
from __future__ import annotations
import types
from typing import Optional
# HF Trainer / transformers built-in optimizers (no extra dep required).
# adamw_apex_fused is NOT here: transformers 5.x rejects it in
# TrainingArguments (OptimizerNames) even though 4.x accepted it, so a config
# that loaded fine used to crash inside the trainer after the model was
# already loaded (#1269). It is refused at config load instead. (adamw_hf is
# allowlisted via MLX_ONLY_OPTIMIZERS — see below.)
_HF_NATIVE: frozenset = frozenset({
"adamw_torch",
"adamw_torch_fused",
"adamw_torch_xla",
"adamw_anyprecision",
"adafactor",
"sgd",
"adagrad",
"rmsprop",
})
# bitsandbytes-backed (require `bitsandbytes` — already a Soup core dep).
_BNB_BACKED: frozenset = frozenset({
"adamw_bnb_8bit",
"adamw_8bit",
"lion_8bit",
"lion_32bit",
"paged_adamw_8bit",
"paged_adamw_32bit",
"paged_lion_8bit",
"paged_lion_32bit",
})
# v0.41.0 — new entries (require optional packages).
_NEW_V0_41_0: frozenset = frozenset({
"apollo_adamw",
"lomo",
"adalomo",
"grokadamw",
"schedule_free_adamw",
"schedule_free_sgd",
})
# Names that used to be in the allowlist but transformers 5.x rejects in
# TrainingArguments (verified on 5.17, #1269). Kept separately so the config
# loader can refuse them with a message that names working alternatives
# instead of letting the run die inside the trainer after the model loads.
# `adamw_hf` and `muon` are NOT here: the MLX backend builds them itself
# (trainer/mlx_optim.py _OPTIMIZER_MAP), so they stay allowlisted and are
# refused per-backend by SoupConfig instead (see MLX_ONLY_OPTIMIZERS).
_TORCH_REJECTED: frozenset = frozenset({
"adam_mini",
"adamw_apex_fused",
"ao_adamw_4bit",
"ao_adamw_8bit",
"ao_adamw_fp8",
"badam",
"came_pytorch",
"dion",
})
# transformers 5.x rejects these in TrainingArguments, but the MLX backend
# never builds TrainingArguments and has its own optimizer map for both
# names (trainer/mlx_optim.py), so only a non-MLX run refuses them (#1283).
MLX_ONLY_OPTIMIZERS: frozenset = frozenset({"adamw_hf", "muon"})
SUPPORTED_OPTIMIZERS: frozenset = (
_HF_NATIVE | _BNB_BACKED | _NEW_V0_41_0 | MLX_ONLY_OPTIMIZERS
)
# Optional package required for each new optimizer. None = HF/bnb native.
# A user picking an entry whose package is missing gets a friendly "pip install"
# advisory at trainer construction time (not at schema-load) — schema-load
# happens before deps can be probed and the trainer can fall back gracefully
# in test environments. Wrapped in MappingProxyType to prevent runtime mutation
# (matches v0.36.0 _REGISTRY policy).
_OPTIMIZER_PACKAGES = types.MappingProxyType({
"apollo_adamw": "apollo-torch",
"lomo": "lomo-optim",
"adalomo": "lomo-optim",
"grokadamw": "grokadamw",
"schedule_free_adamw": "schedulefree",
"schedule_free_sgd": "schedulefree",
})
_MAX_OPTIMIZER_NAME_LEN = 64
def validate_optimizer_name(name: object) -> str:
"""Schema-load-time validator for ``training.optimizer``.
Returns the normalised name (lower-cased) on success, raises ValueError
with an actionable message on failure.
"""
if not isinstance(name, str):
raise ValueError(
f"optimizer must be a string, got {type(name).__name__}"
)
if not name:
raise ValueError("optimizer must be a non-empty string")
if "\x00" in name:
raise ValueError("optimizer must not contain null bytes")
if len(name) > _MAX_OPTIMIZER_NAME_LEN:
raise ValueError(
f"optimizer name exceeds {_MAX_OPTIMIZER_NAME_LEN} chars"
)
normalised = name.lower()
if normalised in _TORCH_REJECTED:
raise ValueError(
f"optimizer={name!r} is no longer supported: transformers 5.x "
"rejects it in TrainingArguments after the model has loaded, so "
"Soup refuses it at config load instead. Use one of: "
"adamw_torch, adamw_torch_fused, adamw_bnb_8bit, adafactor. "
"See soup_cli.utils.optimizer_zoo."
"SUPPORTED_OPTIMIZERS for the complete list."
)
if normalised not in SUPPORTED_OPTIMIZERS:
# Show a tight allowlist preview rather than dumping all 30+ names.
suggestions = sorted(_NEW_V0_41_0)[:6]
raise ValueError(
f"optimizer={name!r} is not in the supported allowlist. "
f"v0.41.0 additions include: {', '.join(suggestions)}. "
"See soup_cli.utils.optimizer_zoo.SUPPORTED_OPTIMIZERS for "
"the complete list."
)
return normalised
def required_package(name: str) -> Optional[str]:
"""Return the pip-installable package for a new-in-v0.41 optimizer.
Returns None for HF-native and bnb-backed optimizers (already in core
deps).
"""
return _OPTIMIZER_PACKAGES.get(name.lower())
def is_new_v0_41_optimizer(name: str) -> bool:
"""True if ``name`` is one of the v0.41.0 additions."""
if not isinstance(name, str):
return False
return name.lower() in _NEW_V0_41_0