Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Liger Kernel — fused operations for faster, memory-efficient training.
Liger Kernel provides fused CUDA kernels (RMSNorm, SwiGLU, CrossEntropy, RoPE, etc.)
that replace standard HuggingFace operations with optimized fused versions.
Measured on an H100 (sm90) with Llama-3.1-8B + LoRA, batch 4, seq 1024,
5 interleaved repeats: **12.9% memory saving and 5.1% throughput**, verified by
confirming the module classes were actually swapped for Liger's. The saving would
shrink further with gradient checkpointing on.
Numbers: benchmarks/gate-h100-validation.md.
Requires: liger-kernel >= 0.3.0
"""
from __future__ import annotations
from soup_cli.utils.run_provenance import base_revision_kwargs
def check_liger_available() -> bool:
"""Check if liger-kernel is installed."""
try:
import liger_kernel # noqa: F401
return True
except ImportError:
return False
def get_liger_version() -> str | None:
"""Return liger-kernel version string, or None if not installed."""
try:
import liger_kernel
return getattr(liger_kernel, "__version__", "unknown")
except ImportError:
return None
def apply_liger_kernel(model_name: str) -> bool:
"""Apply Liger Kernel fused operations for the given model architecture.
Patches the model class in-place so that all subsequent model instantiations
use fused kernels (RMSNorm, SwiGLU, CrossEntropy, RoPE, FusedLinearCrossEntropy).
This must be called BEFORE loading the model.
Args:
model_name: HuggingFace model name/path (used to detect architecture).
Returns:
True if Liger Kernel was applied, False otherwise.
"""
if not check_liger_available():
return False
clean_name = str(model_name or "").strip().replace("\\", "/").rstrip("/")
last_component = clean_name.rsplit("/", 1)[-1].lower() if clean_name else ""
# Detect the architecture from the CONFIG, not from the path.
#
# This used to call `AutoLigerKernelForCausalLM._apply_liger_kernel(model_name)`
# and fall back to matching "llama" / "mistral" / "qwen2" as a SUBSTRING of the
# model name. That private classmethod does not exist in liger-kernel 0.8.1
# (`AttributeError: type object 'AutoLigerKernelForCausalLM' has no attribute
# '_apply_liger_kernel'`, verified), so the fallback was the only live path —
# and a model loaded from a local directory carries no architecture in its
# name. Every such run printed "no matching architecture found" and trained
# WITHOUT Liger, on a flag the user had explicitly set.
#
# `AutoConfig.model_type` is exactly the string liger's module-level
# `_apply_liger_kernel(model_type)` expects, and a directory cannot rename it.
model_type = _detect_model_type(model_name)
if model_type:
# #1811: liger's `_apply_liger_kernel` logs and returns, without raising,
# for a type it has no kernels for, so calling it proved nothing. Ask its
# own table first. A known type that is not in it is unsupported: the
# name-based fallback below is for a config that cannot be read, and on
# `phi` (phi-2) it would "succeed" by patching Phi-3's classes.
supported = _liger_model_types()
if supported is not None and model_type not in supported:
return False
try:
from liger_kernel.transformers import _apply_liger_kernel
_apply_liger_kernel(model_type)
return True
except (ImportError, AttributeError, NotImplementedError, KeyError, ValueError):
# liger does not support this architecture, or moved the entry point
# again — fall through to the name-based attempt rather than claiming
# success we cannot back.
pass
return _apply_liger_manual(last_component)
def _liger_model_types() -> frozenset[str] | None:
"""The model types liger has kernels for, from its own table, or None.
``None`` means the table could not be read (an older or reorganised liger);
the caller then keeps the previous behaviour instead of guessing.
"""
try:
from liger_kernel.transformers.monkey_patch import MODEL_TYPE_TO_APPLY_LIGER_FN
except (ImportError, AttributeError):
return None
try:
return frozenset(str(key) for key in MODEL_TYPE_TO_APPLY_LIGER_FN)
except TypeError:
return None
def _detect_model_type(model_name: str) -> str:
"""``config.model_type`` for a local path or hub id, or "" when unavailable.
Deliberately quiet: a missing config is not an error here, it just means the
caller falls back to the older name-based match and the run still trains.
"""
try:
from transformers import AutoConfig
config = AutoConfig.from_pretrained(
model_name, trust_remote_code=False,
**base_revision_kwargs(model_name),
)
except Exception: # noqa: BLE001 — detection is best-effort by design
return ""
return str(getattr(config, "model_type", "") or "")
def _apply_liger_manual(last_component: str) -> bool:
"""Manually apply Liger Kernel patches matching the last path component (#456)."""
try:
if "llama" in last_component or "codellama" in last_component:
from liger_kernel.transformers import apply_liger_kernel_to_llama
apply_liger_kernel_to_llama()
return True
elif "mistral" in last_component or "mixtral" in last_component:
from liger_kernel.transformers import apply_liger_kernel_to_mistral
apply_liger_kernel_to_mistral()
return True
elif "gemma" in last_component:
from liger_kernel.transformers import apply_liger_kernel_to_gemma2
apply_liger_kernel_to_gemma2()
return True
elif "qwen" in last_component:
from liger_kernel.transformers import apply_liger_kernel_to_qwen2
apply_liger_kernel_to_qwen2()
return True
elif "phi" in last_component:
from liger_kernel.transformers import apply_liger_kernel_to_phi3
apply_liger_kernel_to_phi3()
return True
except (ImportError, AttributeError):
pass
return False
def validate_liger_config(use_liger: bool, backend: str, device: str) -> list[str]:
"""Validate Liger Kernel configuration and return error messages.
Args:
use_liger: Whether Liger Kernel is requested.
backend: Training backend (transformers/unsloth).
device: Training device (cuda/cpu/mps).
Returns:
List of error messages. Empty list means valid.
"""
errors: list[str] = []
if not use_liger:
return errors
if not check_liger_available():
errors.append(
"liger-kernel is not installed. "
"Install it with: pip install \"soup-cli[liger]\""
)
if backend == "unsloth":
errors.append(
"Liger Kernel is not compatible with the unsloth backend. "
"Unsloth has its own fused kernels. Use backend: transformers."
)
if device != "cuda":
errors.append(
"Liger Kernel requires CUDA. "
f"Current device: {device}. Use a GPU for Liger Kernel."
)
return errors