Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#1466: `quantization: mxfp4` must load, not die after the model load.
MXFP4 is transformers' own format (`Mxfp4Config`), not a bitsandbytes
`quant_type`: bitsandbytes accepts only `nf4` and `fp4`, so the
`BitsAndBytesConfig(bnb_4bit_quant_type="mxfp4")` Soup used to build was
rejected for every linear layer, and clashed with the `Mxfp4Config` a
pre-quantized checkpoint (the one Autopilot passes through) carries.
transformers also refuses to train an MXFP4 model as loaded and points at
`Mxfp4Config(dequantize=True)`, which is what the loader now returns.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from soup_cli.config.loader import load_config_from_string
_MXFP4_YAML = """
base: {base}
task: sft
data: {{train: d.jsonl}}
training: {{quantization: mxfp4}}
"""
def _tiny_llama(path: Path, *, mxfp4: bool) -> str:
"""A two-layer Llama on disk; `mxfp4` marks its config as the checkpoint
of a pre-quantized base does."""
import torch
import transformers
config = transformers.LlamaConfig(
vocab_size=128,
hidden_size=64,
intermediate_size=128,
num_hidden_layers=2,
num_attention_heads=4,
num_key_value_heads=4,
)
transformers.LlamaForCausalLM(config).to(torch.bfloat16).save_pretrained(path)
if mxfp4:
config_path = path / "config.json"
raw = json.loads(config_path.read_text(encoding="utf-8"))
raw["quantization_config"] = {"quant_method": "mxfp4"}
config_path.write_text(json.dumps(raw), encoding="utf-8")
return str(path)
def _loader_config(base: str):
from soup_cli.utils.quant_menu import build_quantization_config_for_loader
cfg = load_config_from_string(_MXFP4_YAML.format(base=base))
return build_quantization_config_for_loader(tcfg=cfg.training, base=cfg.base)
class TestLoaderConfig:
def test_mxfp4_is_transformers_own_config_dequantized(self, tmp_path):
from transformers import BitsAndBytesConfig, Mxfp4Config
quant = _loader_config(_tiny_llama(tmp_path / "base", mxfp4=True))
assert isinstance(quant, Mxfp4Config)
assert not isinstance(quant, BitsAndBytesConfig)
# transformers cannot train an MXFP4 model as loaded.
assert quant.dequantize is True
def test_a_base_marked_mxfp4_loads_with_the_loader_config(self, tmp_path):
"""The Autopilot pass-through case: the checkpoint says Mxfp4Config."""
import torch
import transformers
base = _tiny_llama(tmp_path / "base", mxfp4=True)
model = transformers.AutoModelForCausalLM.from_pretrained(
base, quantization_config=_loader_config(base), dtype=torch.bfloat16
)
assert model.model.layers[0].self_attn.q_proj.weight.dtype == torch.bfloat16
def test_autopilot_pass_through_round_trips_to_the_same_config(self, tmp_path):
from transformers import Mxfp4Config
from soup_cli.autopilot.decisions import (
decide_quantization,
detect_prequantized_format,
)
fmt = decide_quantization(
20.0,
80.0,
prequantized=detect_prequantized_format(
"openai/gpt-oss-20b", {"quantization_config": {"quant_method": "mxfp4"}}
),
)
assert fmt == "mxfp4"
base = _tiny_llama(tmp_path / "base", mxfp4=True)
assert isinstance(_loader_config(base), Mxfp4Config)
def test_a_hub_id_is_taken_on_faith(self):
from transformers import Mxfp4Config
assert isinstance(_loader_config("openai/gpt-oss-20b"), Mxfp4Config)
class TestLocalBaseThatIsNotMxfp4:
def test_a_plain_local_base_is_refused(self, tmp_path):
"""Dequantizing a base that was never MXFP4 loads it as it is, so the
run would train unquantized while the config says otherwise."""
base = _tiny_llama(tmp_path / "plain", mxfp4=False)
with pytest.raises(ValueError, match="mxfp4") as raised:
_loader_config(base)
# The directory name, not the absolute path.
# The message shows the name with repr(), which doubles a Windows
# path's backslashes: undo that so the check can fail there too.
assert str(tmp_path) not in str(raised.value).replace("\\\\", "\\")
def test_a_local_base_of_another_format_is_refused(self, tmp_path):
base = Path(_tiny_llama(tmp_path / "other", mxfp4=False))
raw = json.loads((base / "config.json").read_text(encoding="utf-8"))
raw["quantization_config"] = {"quant_method": "gptq"}
(base / "config.json").write_text(json.dumps(raw), encoding="utf-8")
with pytest.raises(ValueError, match="mxfp4"):
_loader_config(str(base))
class TestSchema:
def test_quant_storage_is_bitsandbytes_only(self):
"""`bnb_4bit_quant_storage` is a BitsAndBytesConfig argument; MXFP4
has no such field, so it would be a silent no-op."""
with pytest.raises(ValueError, match="bnb_4bit_quant_storage"):
load_config_from_string(
"""
base: m
task: sft
data: {train: d.jsonl}
training: {quantization: mxfp4, bnb_4bit_quant_storage: bfloat16}
"""
)
def test_the_field_no_longer_calls_mxfp4_a_bnb_type(self):
from soup_cli.config.schema import TrainingConfig
description = TrainingConfig.model_fields["quantization"].description or ""
assert "BNB 4-bit MXFP4" not in description
class TestKbitPrep:
def test_kbit_prep_would_upcast_a_dequantized_model(self, tmp_path):
"""Why `mxfp4` left the trainers' k-bit tuple (pinned per trainer in
test_v0405_part_a.py): the dequantized model is plain bf16, and peft
casts every bf16 parameter of a model that is not k-bit to fp32."""
import torch
import transformers
from peft import prepare_model_for_kbit_training
base = _tiny_llama(tmp_path / "base", mxfp4=True)
model = transformers.AutoModelForCausalLM.from_pretrained(
base, quantization_config=_loader_config(base), dtype=torch.bfloat16
)
prepared = prepare_model_for_kbit_training(
model, use_gradient_checkpointing=False
)
weight = prepared.model.layers[0].self_attn.q_proj.weight
assert weight.dtype == torch.float32
_TRAINER_DIR = Path(__file__).resolve().parent.parent / "src" / "soup_cli" / "trainer"
@pytest.mark.parametrize("filename", ["sft.py", "online_dpo.py"])
def test_sft_and_online_dpo_keep_mxfp4_out_of_kbit_prep(filename):
"""The two trainers `_TRAINER_FILES` in test_v0405_part_a.py does not list."""
import re
src = (_TRAINER_DIR / filename).read_text(encoding="utf-8")
assert re.search(r'\(\s*"4bit"\s*,\s*"8bit"\s*\)', src)
assert not re.search(r'\(\s*"4bit"\s*,\s*"8bit"\s*,\s*"mxfp4"\s*\)', src)
class TestHardwareFit:
"""The hardware-fit calculator prices what is loaded: a dequantized model is bf16."""
@staticmethod
def _weights_gb(quant: str) -> float:
from soup_cli.utils.hardware_fit import HardwareFitInput, estimate_peak_vram_gb
return estimate_peak_vram_gb(
HardwareFitInput(
params_b=20.9,
seq_len=2048,
batch_size=1,
optimizer="adamw_torch",
quant=quant,
peft="lora",
gradient_checkpointing=True,
)
).weights_gb
def test_mxfp4_weights_are_priced_as_bf16(self):
assert self._weights_gb("mxfp4") == pytest.approx(self._weights_gb("none"))
def test_a_gpt_oss_20b_sized_base_does_not_pass_as_a_12_gb_load(self):
# 20.9B parameters at 2 bytes each; 0.55 bytes each said about 11.5 GB.
assert self._weights_gb("mxfp4") == pytest.approx(41.8, rel=0.08)