Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

13.0 KB
Raw Blame
"""Fast-LoRA fused SwiGLU MLP autograd path for issue #837.

Targets gate_proj/up_proj/down_proj blocks with SiLU/swish. This is the
correctness slice from tracker #792; it makes no throughput claim.
"""

from __future__ import annotations

import logging
import types
from typing import Any

from soup_cli.utils.fast_lora import (
    _GROUP_PATCH_OWNER_MARKER,
    _as_dtype,
    _dense_weight,
    _flatten,
    _is_supported_lora_projection,
    _projection_state,
    _release_single_projection,
    _scaled_lora_add,
)

logger = logging.getLogger(__name__)
_PATCH_MARKER = "_soup_fast_lora_mlp"
_ORIGINAL_FORWARD_MARKER = "_soup_fast_lora_mlp_original_forward"
_HAD_INSTANCE_FORWARD_MARKER = "_soup_fast_lora_mlp_had_instance_forward"
_OWNER = "mlp"
_FUNCTION: Any = None

__all__ = ["patch_fast_lora_mlp", "unpatch_fast_lora_mlp"]


def _mlp_function() -> Any:
    global _FUNCTION
    if _FUNCTION is not None:
        return _FUNCTION

    import torch
    import torch.nn.functional as functional

    class _FastLoraSwiGLU(torch.autograd.Function):
        @staticmethod
        def forward(
            ctx, x, wg, bg, wu, bu, wd, bd,
            ag, bgl, au, bul, ad, bdl,
            sg, su, sd, qg_meta, qu_meta, qd_meta, *qparts,
        ):
            metas = (qg_meta, qu_meta, qd_meta)
            counts = [0 if meta is None else int(meta["_count"]) for meta in metas]
            starts = (0, counts[0], counts[0] + counts[1])
            qlists = [
                list(qparts[starts[i] : starts[i] + counts[i]]) for i in range(3)
            ]

            dense_g = _dense_weight(wg, qg_meta, qlists[0], x.dtype)
            dense_u = _dense_weight(wu, qu_meta, qlists[1], x.dtype)
            g = functional.linear(x, dense_g, _as_dtype(bg, x.dtype) if bg is not None else None)
            u = functional.linear(x, dense_u, _as_dtype(bu, x.dtype) if bu is not None else None)
            del dense_g, dense_u

            has_g, has_u, has_d = ag.numel() != 0, au.numel() != 0, ad.numel() != 0
            gu_parts = []
            if has_g:
                gu_parts.append(ag)
            if has_u:
                gu_parts.append(au)
            if gu_parts:
                # #837's advertised fusion: gate/up LoRA A projections share X,
                # so concatenate A and perform one GEMM, then split by rank.
                a_gu = torch.cat(gu_parts, dim=0)
                h_gu = functional.linear(_as_dtype(x, a_gu.dtype), a_gu)
                cursor = 0
                if has_g:
                    rank = ag.shape[0]
                    hg = h_gu[..., cursor : cursor + rank]
                    g = _scaled_lora_add(g, hg, bgl, sg)
                    cursor += rank
                if has_u:
                    rank = au.shape[0]
                    hu = h_gu[..., cursor : cursor + rank]
                    u = _scaled_lora_add(u, hu, bul, su)
            else:
                h_gu = x.new_empty((*x.shape[:-1], 0))

            m = functional.silu(g) * u
            dense_d = _dense_weight(wd, qd_meta, qlists[2], m.dtype)
            y = functional.linear(m, dense_d, _as_dtype(bd, m.dtype) if bd is not None else None)
            del dense_d
            if has_d:
                hd = functional.linear(_as_dtype(m, ad.dtype), ad)
                y = _scaled_lora_add(y, hd, bdl, sd)

            ctx.has_adapters = (has_g, has_u, has_d)
            ctx.gu_ranks = (ag.shape[0] if has_g else 0, au.shape[0] if has_u else 0)
            ctx.scalings = (float(sg), float(su), float(sd))
            ctx.qmetas = metas
            ctx.qcounts = counts
            ctx.qparts_len = len(qparts)
            ctx.save_for_backward(
                x, wg, wu, wd, g, u, ag, bgl, au, bul, ad, bdl, h_gu, *qparts
            )
            return y

        @staticmethod
        @torch.autograd.function.once_differentiable
        def backward(ctx, grad_y):
            saved = ctx.saved_tensors
            x, wg, wu, wd, g, u, ag, bgl, au, bul, ad, bdl, h_gu = saved[:13]
            qparts = list(saved[13:])
            qg_meta, qu_meta, qd_meta = ctx.qmetas
            cg, cu, cd = ctx.qcounts
            qg = qparts[:cg]
            qu = qparts[cg : cg + cu]
            qd = qparts[cg + cu : cg + cu + cd]
            has_g, has_u, has_d = ctx.has_adapters
            sg, su, sd = ctx.scalings

            silu_g = torch.nn.functional.silu(g)
            m = silu_g * u
            grad_ad = grad_bdl = None

            dense_d = _dense_weight(wd, qd_meta, qd, grad_y.dtype)
            grad_m = torch.matmul(grad_y, dense_d)
            del dense_d
            if has_d:
                hd = torch.matmul(_as_dtype(m, ad.dtype), ad.t())
                grad_bdl = _flatten(_as_dtype(grad_y, hd.dtype)).t() @ _flatten(hd) * sd
                grad_hd = torch.matmul(_as_dtype(grad_y, bdl.dtype), bdl) * sd
                grad_ad = _flatten(grad_hd).t() @ _as_dtype(_flatten(m), grad_hd.dtype)
                grad_m = torch.add(
                    grad_m,
                    torch.matmul(_as_dtype(grad_hd, grad_m.dtype), _as_dtype(ad, grad_m.dtype)),
                )

            grad_u = grad_m * silu_g
            # Match autograd's opmath rule for SiLU backward: bf16/fp16
            # elementwise arithmetic runs in fp32 and rounds once.  Evaluating
            # ``1 + g * (1 - sigmoid(g))`` in bf16 magnifies cancellation near
            # the derivative's zero and can dominate the NF4 error budget.
            opmath = torch.promote_types(g.dtype, torch.float32)
            g_hi = g.to(opmath)
            sig = torch.sigmoid(g_hi)
            grad_g = (
                grad_m.to(opmath)
                * u.to(opmath)
                * sig
                * (1 + g_hi * (1 - sig))
            ).to(g.dtype)

            grad_x = None
            if ctx.needs_input_grad[0]:
                dense_g = _dense_weight(wg, qg_meta, qg, grad_g.dtype)
                grad_x = torch.matmul(grad_g, dense_g)
                del dense_g
                dense_u = _dense_weight(wu, qu_meta, qu, grad_u.dtype)
                grad_x = torch.add(grad_x, torch.matmul(grad_u, dense_u))
                del dense_u

            grad_ag = grad_bgl = grad_au = grad_bul = None
            dh_parts = []
            a_parts = []
            cursor = 0
            if has_g:
                rank = ctx.gu_ranks[0]
                hg = h_gu[..., cursor : cursor + rank]
                grad_bgl = _flatten(_as_dtype(grad_g, hg.dtype)).t() @ _flatten(hg) * sg
                grad_hg = torch.matmul(_as_dtype(grad_g, bgl.dtype), bgl) * sg
                dh_parts.append(grad_hg)
                a_parts.append(ag)
                cursor += rank
            if has_u:
                rank = ctx.gu_ranks[1]
                hu = h_gu[..., cursor : cursor + rank]
                grad_bul = _flatten(_as_dtype(grad_u, hu.dtype)).t() @ _flatten(hu) * su
                grad_hu = torch.matmul(_as_dtype(grad_u, bul.dtype), bul) * su
                dh_parts.append(grad_hu)
                a_parts.append(au)

            if dh_parts:
                # One dA GEMM for the gate/up pair, mirroring the one forward
                # X @ A_gu^T GEMM above; split the concatenated result by rank.
                dh_gu = dh_parts[0] if len(dh_parts) == 1 else torch.cat(dh_parts, dim=-1)
                a_gu = a_parts[0] if len(a_parts) == 1 else torch.cat(a_parts, dim=0)
                grad_a_gu = _flatten(dh_gu).t() @ _as_dtype(_flatten(x), dh_gu.dtype)
                if grad_x is not None:
                    grad_x = torch.add(
                        grad_x,
                        torch.matmul(
                            _as_dtype(dh_gu, grad_x.dtype),
                            _as_dtype(a_gu, grad_x.dtype),
                        ),
                    )
                cursor = 0
                if has_g:
                    rank = ctx.gu_ranks[0]
                    grad_ag = grad_a_gu[cursor : cursor + rank]
                    cursor += rank
                if has_u:
                    rank = ctx.gu_ranks[1]
                    grad_au = grad_a_gu[cursor : cursor + rank]

            result = [
                grad_x,
                None, None, None, None, None, None,
                grad_ag, grad_bgl, grad_au, grad_bul, grad_ad, grad_bdl,
                None, None, None, None, None, None,
            ]
            result.extend([None] * ctx.qparts_len)
            return tuple(result)

    _FUNCTION = _FastLoraSwiGLU
    return _FUNCTION


def _make_mlp_forward(original_forward: Any) -> Any:
    fast = _mlp_function()

    def _forward(self, x, *args, **kwargs):
        if args or kwargs:
            return original_forward(x, *args, **kwargs)
        states = [
            _projection_state(getattr(self, name), x, allow_unadapted=True)
            for name in ("gate_proj", "up_proj", "down_proj")
        ]
        if any(state is None for state in states):
            return original_forward(x)
        gate, up, down = states
        compute_dtypes = {
            state.compute_dtype for state in states if state.compute_dtype is not None
        }
        if len(compute_dtypes) > 1:
            return original_forward(x)
        input_dtype = x.dtype
        work_x = x
        if compute_dtypes:
            compute_dtype = next(iter(compute_dtypes))
            if work_x.dtype != compute_dtype:
                work_x = work_x.to(compute_dtype)

        qparts = [*gate.qparts, *up.qparts, *down.qparts]
        out = fast.apply(
            work_x,
            gate.weight, gate.bias, up.weight, up.bias, down.weight, down.bias,
            gate.lora_a, gate.lora_b, up.lora_a, up.lora_b, down.lora_a, down.lora_b,
            gate.scaling, up.scaling, down.scaling,
            gate.qmeta, up.qmeta, down.qmeta,
            *qparts,
        )
        return out if work_x is x else out.to(input_dtype)

    return _forward


def _module_uses_silu(module: Any) -> bool:
    act_fn = getattr(module, "act_fn", None)
    if act_fn is None:
        return False
    name = (getattr(act_fn, "__name__", "") or type(act_fn).__name__).lower()
    return "silu" in name or "swish" in name


def _looks_like_moe_expert(path: str, module: Any) -> bool:
    parts = path.lower().split(".")
    cls = type(module).__name__.lower()
    return (
        any(part == "experts" or part.startswith("expert") for part in parts)
        or "moe" in cls
        or "expert" in cls
        or "blocksparse" in cls
    )


def patch_fast_lora_mlp(model: Any) -> int:
    """Patch supported dense SiLU MLP blocks and return the number patched."""
    hidden_act = str(getattr(getattr(model, "config", None), "hidden_act", "")).lower()
    if hidden_act not in {"silu", "swish"}:
        logger.info(
            "Fast-LoRA MLP skipped: hidden_act=%r is not SiLU/swish", hidden_act or None
        )
        return 0

    patched = 0
    for path, module in model.named_modules():
        if getattr(module, _PATCH_MARKER, False):
            continue
        if _looks_like_moe_expert(path, module):
            continue
        if not all(
            hasattr(module, name) for name in ("gate_proj", "up_proj", "down_proj")
        ):
            continue
        if not _module_uses_silu(module):
            continue
        if not any(
            hasattr(getattr(module, name), "lora_A")
            for name in ("gate_proj", "up_proj", "down_proj")
        ):
            continue
        projections = [
            getattr(module, name) for name in ("gate_proj", "up_proj", "down_proj")
        ]
        if any(hasattr(proj, "modules_to_save") for proj in projections):
            continue
        if any(
            hasattr(proj, "lora_A") and not _is_supported_lora_projection(proj)
            for proj in projections
        ):
            continue
        if any(getattr(proj, _GROUP_PATCH_OWNER_MARKER, None) is not None for proj in projections):
            continue
        for proj in projections:
            _release_single_projection(proj)
            setattr(proj, _GROUP_PATCH_OWNER_MARKER, _OWNER)
        setattr(module, _ORIGINAL_FORWARD_MARKER, module.forward)
        setattr(module, _HAD_INSTANCE_FORWARD_MARKER, "forward" in vars(module))
        setattr(module, _PATCH_MARKER, True)
        module.forward = types.MethodType(_make_mlp_forward(module.forward), module)
        patched += 1
    return patched


def unpatch_fast_lora_mlp(model: Any) -> int:
    """Restore original MLP forwards. Returns the number restored."""
    restored = 0
    for module in model.modules():
        original = getattr(module, _ORIGINAL_FORWARD_MARKER, None)
        if original is None or not getattr(module, _PATCH_MARKER, False):
            continue
        if getattr(module, _HAD_INSTANCE_FORWARD_MARKER, False):
            module.forward = original
        else:
            del module.forward
        delattr(module, _ORIGINAL_FORWARD_MARKER)
        delattr(module, _HAD_INSTANCE_FORWARD_MARKER)
        delattr(module, _PATCH_MARKER)
        for name in ("gate_proj", "up_proj", "down_proj"):
            proj = getattr(module, name)
            if getattr(proj, _GROUP_PATCH_OWNER_MARKER, None) == _OWNER:
                delattr(proj, _GROUP_PATCH_OWNER_MARKER)
        restored += 1
    return restored