Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

5.8 KB
Raw Blame
"""Optimizer Zoo — v0.41.0 Part A.

Expands optimizer support beyond HF Trainer's built-in set to cover LlamaFactory
+ Axolotl parity: BAdam / APOLLO / Adam-mini / lomo / adalomo / grokadamw /
schedule_free / muon / dion / came_pytorch / ao_adamw_{fp8,4bit,8bit}.

Schema-level allowlist + dependency advisory. Live wiring delegates to HF
Trainer (which accepts arbitrary optimizer strings via TrainingArguments) plus
optional adapter packages installed alongside Soup.

Security:
- Closed allowlist of optimizer names (no arbitrary string accepted at
  schema level — prevents YAML-driven typos from silently using the
  default).
- Empty / null-byte / non-string optimizer rejected.
- Length cap (64 chars) — defence-in-depth against absurd inputs.
"""

from __future__ import annotations

import types
from typing import Optional

# HF Trainer / transformers built-in optimizers (no extra dep required).
# adamw_apex_fused is NOT here: transformers 5.x rejects it in
# TrainingArguments (OptimizerNames) even though 4.x accepted it, so a config
# that loaded fine used to crash inside the trainer after the model was
# already loaded (#1269). It is refused at config load instead. (adamw_hf is
# allowlisted via MLX_ONLY_OPTIMIZERS — see below.)
_HF_NATIVE: frozenset = frozenset({
    "adamw_torch",
    "adamw_torch_fused",
    "adamw_torch_xla",
    "adamw_anyprecision",
    "adafactor",
    "sgd",
    "adagrad",
    "rmsprop",
})

# bitsandbytes-backed (require `bitsandbytes` — already a Soup core dep).
_BNB_BACKED: frozenset = frozenset({
    "adamw_bnb_8bit",
    "adamw_8bit",
    "lion_8bit",
    "lion_32bit",
    "paged_adamw_8bit",
    "paged_adamw_32bit",
    "paged_lion_8bit",
    "paged_lion_32bit",
})

# v0.41.0 — new entries (require optional packages).
_NEW_V0_41_0: frozenset = frozenset({
    "apollo_adamw",
    "lomo",
    "adalomo",
    "grokadamw",
    "schedule_free_adamw",
    "schedule_free_sgd",
})

# Names that used to be in the allowlist but transformers 5.x rejects in
# TrainingArguments (verified on 5.17, #1269). Kept separately so the config
# loader can refuse them with a message that names working alternatives
# instead of letting the run die inside the trainer after the model loads.
# `adamw_hf` and `muon` are NOT here: the MLX backend builds them itself
# (trainer/mlx_optim.py _OPTIMIZER_MAP), so they stay allowlisted and are
# refused per-backend by SoupConfig instead (see MLX_ONLY_OPTIMIZERS).
_TORCH_REJECTED: frozenset = frozenset({
    "adam_mini",
    "adamw_apex_fused",
    "ao_adamw_4bit",
    "ao_adamw_8bit",
    "ao_adamw_fp8",
    "badam",
    "came_pytorch",
    "dion",
})

# transformers 5.x rejects these in TrainingArguments, but the MLX backend
# never builds TrainingArguments and has its own optimizer map for both
# names (trainer/mlx_optim.py), so only a non-MLX run refuses them (#1283).
MLX_ONLY_OPTIMIZERS: frozenset = frozenset({"adamw_hf", "muon"})

SUPPORTED_OPTIMIZERS: frozenset = (
    _HF_NATIVE | _BNB_BACKED | _NEW_V0_41_0 | MLX_ONLY_OPTIMIZERS
)

# Optional package required for each new optimizer. None = HF/bnb native.
# A user picking an entry whose package is missing gets a friendly "pip install"
# advisory at trainer construction time (not at schema-load) — schema-load
# happens before deps can be probed and the trainer can fall back gracefully
# in test environments. Wrapped in MappingProxyType to prevent runtime mutation
# (matches v0.36.0 _REGISTRY policy).
_OPTIMIZER_PACKAGES = types.MappingProxyType({
    "apollo_adamw": "apollo-torch",
    "lomo": "lomo-optim",
    "adalomo": "lomo-optim",
    "grokadamw": "grokadamw",
    "schedule_free_adamw": "schedulefree",
    "schedule_free_sgd": "schedulefree",
})

_MAX_OPTIMIZER_NAME_LEN = 64


def validate_optimizer_name(name: object) -> str:
    """Schema-load-time validator for ``training.optimizer``.

    Returns the normalised name (lower-cased) on success, raises ValueError
    with an actionable message on failure.
    """
    if not isinstance(name, str):
        raise ValueError(
            f"optimizer must be a string, got {type(name).__name__}"
        )
    if not name:
        raise ValueError("optimizer must be a non-empty string")
    if "\x00" in name:
        raise ValueError("optimizer must not contain null bytes")
    if len(name) > _MAX_OPTIMIZER_NAME_LEN:
        raise ValueError(
            f"optimizer name exceeds {_MAX_OPTIMIZER_NAME_LEN} chars"
        )
    normalised = name.lower()
    if normalised in _TORCH_REJECTED:
        raise ValueError(
            f"optimizer={name!r} is no longer supported: transformers 5.x "
            "rejects it in TrainingArguments after the model has loaded, so "
            "Soup refuses it at config load instead. Use one of: "
            "adamw_torch, adamw_torch_fused, adamw_bnb_8bit, adafactor. "
            "See soup_cli.utils.optimizer_zoo."
            "SUPPORTED_OPTIMIZERS for the complete list."
        )
    if normalised not in SUPPORTED_OPTIMIZERS:
        # Show a tight allowlist preview rather than dumping all 30+ names.
        suggestions = sorted(_NEW_V0_41_0)[:6]
        raise ValueError(
            f"optimizer={name!r} is not in the supported allowlist. "
            f"v0.41.0 additions include: {', '.join(suggestions)}. "
            "See soup_cli.utils.optimizer_zoo.SUPPORTED_OPTIMIZERS for "
            "the complete list."
        )
    return normalised


def required_package(name: str) -> Optional[str]:
    """Return the pip-installable package for a new-in-v0.41 optimizer.

    Returns None for HF-native and bnb-backed optimizers (already in core
    deps).
    """
    return _OPTIMIZER_PACKAGES.get(name.lower())


def is_new_v0_41_optimizer(name: str) -> bool:
    """True if ``name`` is one of the v0.41.0 additions."""
    if not isinstance(name, str):
        return False
    return name.lower() in _NEW_V0_41_0