Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

11.2 KB
Raw Blame
"""FP8 training — 8-bit floating point training via torchao/transformer_engine.

FP8 training on Hopper (H100, H200) and Blackwell (B100, B200) GPUs uses 8-bit
floating point for matmuls, giving ~2x speedup vs bf16 at comparable quality.

This extends the existing int8-QAT infrastructure (``utils/qat.py``). When the
user sets ``quantization_aware: 'fp8'`` in soup.yaml the FP8 recipe is applied;
``quantization_aware: true`` keeps the legacy int8 QAT path.

Requires:
- NVIDIA Ada or newer GPU (SM 8.9+) — RTX 40/50-series, L4, L40S, H100, H200,
  B100, B200. Rowwise recipes also need a torch that dispatches their kernel on
  the card (Ada >= 2.7, RTX 50 >= 2.8, 11.x >= 2.10) and never run on Windows
  (#835).
- torchao (the floor is TORCHAO_MIN_VERSION in utils/torchao_compat.py, #826)
  OR transformer-engine >= 1.0
- CUDA 12.0+
"""

from __future__ import annotations


def is_torchao_float8_available() -> bool:
    """True when torchao's float8 converter, the one Soup applies, is importable."""
    try:
        from torchao.float8 import convert_to_float8_training  # noqa: F401
    except ImportError:
        return False
    return True


def is_fp8_available() -> bool:
    """Return True if *any* FP8 training backend is importable.

    Checks torchao's FP8 recipe first, then transformer-engine.
    """
    # torchao path (preferred — we already require torchao for int8 QAT)
    try:
        from torchao.float8 import convert_to_float8_training  # noqa: F401

        return True
    except ImportError:
        pass

    try:
        import transformer_engine  # noqa: F401

        return True
    except ImportError:
        pass

    return False


# #835: torch's own floors, read from its source at the release tags.
#
# Every recipe: ``_scaled_mm_allowed_device()`` accepts ``major >= 9 or (8, 9)``
# (identical at v2.4.0, v2.6.0, v2.7.0, v2.13.0).
#
# Rowwise recipes run a CUTLASS kernel in ``RowwiseScaledMM.cu``, which has two
# separate gates:
#
# * a BUILD gate, ``BUILD_ROWWISE_FP8_KERNEL``, defined only when
#   ``!USE_ROCM && !_WIN32`` (every release through v2.14.0) and, before v2.11.0,
#   ``CUDA_VERSION >= 12000``. Without it, ``f8f8bf16_rowwise`` raises "Rowwise
#   scaling is not currenlty supported on your device" on EVERY device -- the
#   message names the device but the refusal is the build's. So rowwise never
#   runs on Windows, and never on a CUDA 11 build of torch before 2.11.
# * a DEVICE dispatch, first added in v2.7.0: ``sm89 || sm9x || sm10x``, then
#   ``sm12x`` in v2.8.0 and ``sm11x`` in v2.10.0 (still the full list at
#   v2.14.0). Any other major raises "Rowwise scaling is not currently supported
#   on your device" at the first forward.
#: TRANSCRIBED from torch's source, not queried from torch: torch 2.14 exposes no
#: runtime FP8 capability API (no `scaled`/`float8` symbol on `torch._C` or
#: `torch.backends.cuda`), so a table is the only option short of a trial matmul.
#: Checked against torch source through v2.14.0; a major that is not listed below
#: is REFUSED for rowwise until someone updates this table, which is the
#: conservative direction (#1044 review).
_FP8_MIN_CAPABILITY = (8, 9)

#: The first torch whose rowwise dispatch accepts each compute-capability major.
#: A major missing here is accepted by no release this was checked against.
#: 9.x and 10.x are Soup's own torch floor (2.6): v2.6.0 has no device dispatch
#: at all, and v2.7.0 already lists both.
_ROWWISE_MIN_TORCH_BY_MAJOR = {
    8: (2, 7),
    9: (2, 6),
    10: (2, 6),
    11: (2, 10),
    12: (2, 8),
}

#: Before this torch, the rowwise kernel is only built against CUDA >= 12.
_ROWWISE_ANY_CUDA_TORCH = (2, 11)

_FP8_GPU_REFUSAL = (
    "FP8 training requires an Ada or newer GPU (compute capability >= 8.9): "
    "RTX 40/50-series, L4, L40S, RTX 6000 Ada, Hopper (H100/H200) or "
    "Blackwell (B100/B200)."
)


class FP8DependencyMissingError(RuntimeError):
    """An explicitly requested FP8 setting whose converter, torchao, is not installed.

    The sibling of :class:`FP8HardwareUnsupportedError`, and stopped the same way
    (#835 ruling, 2026-09-19): whether the reason is the card or a missing
    package does not change what the user asked for, and a run that trains in
    bf16 under a config saying FP8 records settings that did not happen.
    """


#: One message for both FP8 paths, naming the fix rather than the internals.
FP8_TORCHAO_MISSING = (
    "FP8 was requested (quantization_aware: fp8 or fp8_attention: true), but "
    "torchao's float8 training is not installed, so nothing would be converted. "
    'Install it with: pip install "soup-cli[qat]"'
)


class FP8HardwareUnsupportedError(RuntimeError):
    """An explicitly requested FP8 setting this card, OS or torch build cannot run.

    Raised before anything is converted, and never caught as an optional
    feature: a setting that is accepted and then silently not applied is the
    defect class this exists to stop (#835 review). Every trainer lets it end
    the run at setup.
    """


def _cuda_capability() -> "tuple[int, int] | None":
    """Return the first CUDA device's capability, or None without CUDA."""
    try:
        import torch

        if not torch.cuda.is_available():
            return None
        major, minor = torch.cuda.get_device_capability(0)
        return int(major), int(minor)
    except (ImportError, RuntimeError, AssertionError):
        return None


def _version_at_least(version: object, floor: "tuple[int, int]") -> bool:
    """True when ``version``'s major.minor is at least ``floor``; False if unreadable."""
    import re

    match = re.match(r"(\d+)\.(\d+)", str(version))
    if match is None:
        return False
    return (int(match.group(1)), int(match.group(2))) >= floor


def _torch_at_least(floor: "tuple[int, int]") -> bool:
    """True when the installed torch's major.minor is at least ``floor``."""
    import torch

    return _version_at_least(torch.__version__, floor)


def is_fp8_gpu_supported() -> bool:
    """Return True if the GPU meets torch's FP8 floor (SM 8.9+, Ada or newer).

    This is the recipe-independent part of :func:`fp8_training_supported`.
    """
    capability = _cuda_capability()
    return capability is not None and capability >= _FP8_MIN_CAPABILITY


def _rowwise_refusal(recipe: str, capability: "tuple[int, int] | None") -> "str | None":
    """Why ``recipe`` (a rowwise one) cannot run here, or None if it can."""
    import sys

    import torch

    if sys.platform.startswith("win"):
        return (
            f"fp8_recipe '{recipe}' cannot run on Windows: torch never builds its "
            "rowwise FP8 kernel there (BUILD_ROWWISE_FP8_KERNEL requires !_WIN32). "
            "Use fp8_recipe: tensorwise."
        )
    cuda_build = getattr(torch.version, "cuda", None)
    if not _torch_at_least(_ROWWISE_ANY_CUDA_TORCH) and not _version_at_least(
        cuda_build, (12, 0)
    ):
        return (
            f"fp8_recipe '{recipe}' needs a torch built against CUDA >= 12 before "
            f"torch 2.11 (installed: torch {torch.__version__}, CUDA {cuda_build}); "
            "use fp8_recipe: tensorwise."
        )
    if capability is None:
        return None
    floor = _ROWWISE_MIN_TORCH_BY_MAJOR.get(capability[0])
    if floor is None:
        return (
            f"fp8_recipe '{recipe}' is not supported on compute capability "
            f"{capability[0]}.{capability[1]}: no torch release checked (through 2.14) "
            "dispatches its rowwise FP8 kernel there. Use fp8_recipe: tensorwise."
        )
    if not _torch_at_least(floor):
        return (
            f"fp8_recipe '{recipe}' on compute capability {capability[0]}."
            f"{capability[1]} needs torch >= {floor[0]}.{floor[1]} (installed: "
            f"{torch.__version__}); upgrade torch or use fp8_recipe: tensorwise."
        )
    return None


def fp8_training_supported(recipe: str = "tensorwise") -> "tuple[bool, str]":
    """The one FP8 hardware gate (#835): ``(ok, reason)`` for ``recipe``.

    Both ``quantization_aware: fp8`` (:func:`apply_fp8_training`) and
    ``fp8_attention`` (``advanced_precision.apply_fp8_attention``) ask this.
    """
    if not is_fp8_gpu_supported():
        return False, _FP8_GPU_REFUSAL
    if recipe != "tensorwise":
        reason = _rowwise_refusal(recipe, _cuda_capability())
        if reason is not None:
            return False, reason
    return True, ""


def apply_fp8_training(
    model,
    recipe: str = "tensorwise",
) -> bool:
    """Convert eligible linear layers to FP8 for training.

    Uses torchao's ``convert_to_float8_training`` with a scaling recipe
    selected via :pydata:`Float8LinearConfig.from_recipe_name`.

    Supported recipes (from ``torchao.float8.config.Float8LinearRecipeName``):

    - ``"tensorwise"`` — single scale per tensor, cuBLAS kernel (fastest,
      default, v0.28.0 behavior).
    - ``"rowwise"`` — per-row scale, CUTLASS kernel, e4m3 everywhere,
      power-of-2 scales (more accurate).
    - ``"rowwise_with_gw_hp"`` — rowwise but grad_weight stays in high
      precision (most accurate).

    Args:
        model: PyTorch model to convert (typically after LoRA has been applied).
        recipe: Scaling recipe name. Default ``"tensorwise"``.

    Returns:
        True: every failure raises instead (#1152).

    Raises:
        FP8HardwareUnsupportedError: this card, OS or torch build cannot run
            ``recipe`` (#835). Nothing is converted, and the run must stop.
        FP8DependencyMissingError: torchao's float8 training is not installed
            (#835 ruling). Checked after the card, so an unsupported card is
            named as the reason rather than a package that would not help.
        RuntimeError: the conversion itself failed. It converts in place, so
            the model may be partly converted; the run stops rather than train
            it in bf16 under a config that asked for FP8 (#1152).
    """
    # The hardware gate runs FIRST, before the dependency probe (#1044 review).
    # torchao is not a default dependency, so "absent" is the common case: asking
    # availability first meant an Ampere user who wrote quantization_aware: fp8
    # got a yellow line and a bf16 run -- exactly what the ruling removes. What
    # the user asked for cannot run here whether or not torchao is installed.
    ok, reason = fp8_training_supported(recipe)
    if not ok:
        raise FP8HardwareUnsupportedError(reason)

    if not is_fp8_available():
        raise FP8DependencyMissingError(FP8_TORCHAO_MISSING)

    try:
        from torchao.float8 import convert_to_float8_training
        from torchao.float8.config import Float8LinearConfig
    except ImportError as exc:
        # transformer-engine alone satisfies is_fp8_available(), but this
        # converter is torchao's: that box is missing torchao just the same.
        raise FP8DependencyMissingError(FP8_TORCHAO_MISSING) from exc

    try:
        config = Float8LinearConfig.from_recipe_name(recipe)
        convert_to_float8_training(model, config=config)
    except Exception as exc:  # noqa: BLE001 — in-place mutation honesty
        raise RuntimeError(
            "quantization_aware=fp8: the float8 conversion failed and the model may "
            "be partly converted, so the run stops instead of training without FP8 "
            f"({type(exc).__name__}: {exc})"
        ) from exc
    return True