Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#835 — both FP8 training paths ask one hardware gate, with the floor torch uses.
``training.fp8_attention`` refused Ada (sm_89), which ``torch._scaled_mm`` accepts,
while ``training.quantization_aware: fp8`` converted the model on any card,
Ampere included. Both now ask ``fp8_training_supported(recipe)``.
The floors are torch's own (source, not hardware):
- every recipe: ``_scaled_mm_allowed_device()`` is ``major >= 9 or (8, 9)``
(identical at v2.4.0, v2.6.0, v2.7.0, v2.13.0);
- rowwise at (8, 9): torch >= 2.7, the first release with a CUTLASS sm89 rowwise
kernel (v2.6.0 ``RowwiseScaledMM.cu`` builds only ``cutlass::arch::Sm90``).
torchao is replaced by a stub that records conversions; the capability is patched.
Nothing here needs a GPU.
"""
from __future__ import annotations
import importlib.machinery
import sys
from types import ModuleType
import pytest
torch = pytest.importorskip("torch")
nn = torch.nn
class _Attn(nn.Module):
def __init__(self) -> None:
super().__init__()
self.q_proj = nn.Linear(4, 4)
self.k_proj = nn.Linear(4, 4)
self.v_proj = nn.Linear(4, 4)
self.o_proj = nn.Linear(4, 4)
@pytest.fixture
def converts(monkeypatch):
"""Stub torchao.float8; returns the list of recorded conversions."""
calls: list[str] = []
def _convert(model, config=None, module_filter_fn=None):
calls.append(config)
package_names = {
"torchao",
"torchao.float8",
"torchao.prototype",
"torchao.prototype.safetensors",
}
module_names = package_names | {
"torchao.float8.config",
"torchao.prototype.safetensors.safetensors_support",
}
modules = {name: ModuleType(name) for name in module_names}
for name, module in modules.items():
is_package = name in package_names
if is_package:
module.__path__ = []
module.__spec__ = importlib.machinery.ModuleSpec(name, None, is_package=is_package)
float8 = modules["torchao.float8"]
float8.convert_to_float8_training = _convert
float8.config = modules["torchao.float8.config"]
class Float8LinearConfig:
@staticmethod
def from_recipe_name(name):
return name
float8.config.Float8LinearConfig = Float8LinearConfig
prototype = modules["torchao.prototype"]
safetensors = modules["torchao.prototype.safetensors"]
safetensors_support = modules["torchao.prototype.safetensors.safetensors_support"]
safetensors_support.flatten_tensor_state_dict = lambda *args, **kwargs: None
safetensors.safetensors_support = safetensors_support
prototype.safetensors = safetensors
modules["torchao"].prototype = prototype
modules["torchao"].float8 = float8
for name, module in modules.items():
monkeypatch.setitem(sys.modules, name, module)
return calls
def _card(monkeypatch, capability, version="2.13.0", *, platform="linux", cuda="12.4"):
"""A patched card. The OS and torch's CUDA build are pinned too: rowwise
depends on both, and the suite also runs on Windows and macOS."""
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *_a, **_k: capability)
monkeypatch.setattr(torch, "__version__", version)
monkeypatch.setattr(sys, "platform", platform)
monkeypatch.setattr(torch.version, "cuda", cuda)
RECIPES = ("tensorwise", "rowwise", "rowwise_with_gw_hp")
class TestTheFloor:
@pytest.mark.parametrize("recipe", RECIPES)
@pytest.mark.parametrize("capability", [(9, 0), (10, 0)])
def test_hopper_and_blackwell_datacenter_pass_every_recipe_on_old_torch(
self, monkeypatch, capability, recipe
):
"""v2.6.0 has no rowwise device dispatch; v2.7.0 lists sm9x and sm10x."""
from soup_cli.utils.fp8 import fp8_training_supported
_card(monkeypatch, capability, version="2.6.0")
assert fp8_training_supported(recipe) == (True, "")
@pytest.mark.parametrize("capability", [(11, 0), (12, 0), (13, 0)])
def test_every_major_from_ada_passes_tensorwise_on_old_torch(self, monkeypatch, capability):
"""``_scaled_mm_allowed_device()`` is ``major >= 9``: tensorwise has no
per-major dispatch."""
from soup_cli.utils.fp8 import fp8_training_supported
_card(monkeypatch, capability, version="2.6.0")
assert fp8_training_supported("tensorwise") == (True, "")
@pytest.mark.parametrize("recipe", ["rowwise", "rowwise_with_gw_hp"])
@pytest.mark.parametrize(
"capability, refused_on, accepted_on",
[
# sm12x joins the dispatch in v2.8.0 (RowwiseScaledMM.cu:950).
((12, 0), ["2.6.0", "2.7.0", "2.7.1+cu128"], ["2.8.0", "2.14.0"]),
# sm11x joins it in v2.10.0 (:961).
((11, 0), ["2.6.0", "2.8.0", "2.9.1"], ["2.10.0", "2.14.0"]),
],
ids=["rtx50-sm120", "sm110"],
)
def test_newer_majors_need_the_torch_that_added_them(
self, monkeypatch, recipe, capability, refused_on, accepted_on
):
from soup_cli.utils.fp8 import fp8_training_supported
for version in refused_on:
_card(monkeypatch, capability, version=version)
ok, reason = fp8_training_supported(recipe)
assert ok is False, (capability, version)
assert recipe in reason and "tensorwise" in reason, reason
assert f"{capability[0]}.{capability[1]}" in reason, reason
for version in accepted_on:
_card(monkeypatch, capability, version=version)
assert fp8_training_supported(recipe) == (True, ""), (capability, version)
@pytest.mark.parametrize("version", ["2.8.0", "2.14.0", "3.0.0"])
def test_a_major_no_release_dispatches_refuses_rowwise(self, monkeypatch, version):
"""No release through v2.14.0 lists sm13x: refused, not assumed to work."""
from soup_cli.utils.fp8 import fp8_training_supported
_card(monkeypatch, (13, 0), version=version)
ok, reason = fp8_training_supported("rowwise")
assert ok is False
assert "13.0" in reason
@pytest.mark.parametrize("capability", [(8, 9), (9, 0), (12, 0)])
@pytest.mark.parametrize("version", ["2.8.0", "2.14.0"])
def test_windows_refuses_rowwise_at_every_torch(self, monkeypatch, capability, version):
"""``BUILD_ROWWISE_FP8_KERNEL`` requires ``!_WIN32`` through v2.14.0, so the
kernel does not exist there whatever the card; tensorwise is unaffected."""
from soup_cli.utils.fp8 import fp8_training_supported
_card(monkeypatch, capability, version=version, platform="win32")
ok, reason = fp8_training_supported("rowwise")
assert ok is False
assert "Windows" in reason
assert fp8_training_supported("tensorwise") == (True, "")
@pytest.mark.parametrize(
"version, cuda, ok",
[
("2.8.0", "11.8", False),
("2.10.0", "11.8", False),
("2.10.0", "12.1", True),
("2.11.0", "11.8", True),
("2.10.0", None, False),
],
)
def test_a_cuda_11_build_has_no_rowwise_kernel_before_torch_2_11(
self, monkeypatch, version, cuda, ok
):
"""Before v2.11.0 the build gate also requires ``CUDA_VERSION >= 12000``."""
from soup_cli.utils.fp8 import fp8_training_supported
_card(monkeypatch, (9, 0), version=version, cuda=cuda)
assert fp8_training_supported("rowwise")[0] is ok
@pytest.mark.parametrize("version", ["2.6.0", "2.6.0+cu124", "2.13.0"])
def test_ada_passes_tensorwise_on_every_supported_torch(self, monkeypatch, version):
from soup_cli.utils.fp8 import fp8_training_supported
_card(monkeypatch, (8, 9), version=version)
assert fp8_training_supported("tensorwise") == (True, "")
@pytest.mark.parametrize("recipe", ["rowwise", "rowwise_with_gw_hp"])
@pytest.mark.parametrize("version", ["2.7.0", "2.7.1+cu126", "2.13.0", "3.0.0"])
def test_ada_passes_rowwise_from_torch_2_7(self, monkeypatch, recipe, version):
from soup_cli.utils.fp8 import fp8_training_supported
_card(monkeypatch, (8, 9), version=version)
assert fp8_training_supported(recipe) == (True, "")
@pytest.mark.parametrize("recipe", ["rowwise", "rowwise_with_gw_hp"])
@pytest.mark.parametrize("version", ["2.6.0", "2.6.0+cu124", "2.6.1"])
def test_ada_refuses_rowwise_before_torch_2_7(self, monkeypatch, recipe, version):
from soup_cli.utils.fp8 import fp8_training_supported
_card(monkeypatch, (8, 9), version=version)
ok, reason = fp8_training_supported(recipe)
assert ok is False
assert recipe in reason
assert "2.7" in reason
assert "tensorwise" in reason
def test_ada_refuses_rowwise_when_the_torch_version_is_unreadable(self, monkeypatch):
"""Fail closed: a version the gate cannot parse is not assumed to be >= 2.7."""
from soup_cli.utils.fp8 import fp8_training_supported
_card(monkeypatch, (8, 9), version="unknown")
assert fp8_training_supported("rowwise")[0] is False
assert fp8_training_supported("tensorwise") == (True, "")
@pytest.mark.parametrize("recipe", RECIPES)
@pytest.mark.parametrize("capability", [(7, 5), (8, 0), (8, 6), (8, 7)])
def test_below_ada_refuses_every_recipe(self, monkeypatch, capability, recipe):
from soup_cli.utils.fp8 import fp8_training_supported
_card(monkeypatch, capability)
ok, reason = fp8_training_supported(recipe)
assert ok is False
assert "8.9" in reason
assert "Ada" in reason
def test_no_cuda_refuses(self, monkeypatch):
from soup_cli.utils.fp8 import fp8_training_supported
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
ok, reason = fp8_training_supported("tensorwise")
assert ok is False
assert reason
def test_is_fp8_gpu_supported_admits_ada(self, monkeypatch):
from soup_cli.utils.fp8 import is_fp8_gpu_supported
_card(monkeypatch, (8, 9))
assert is_fp8_gpu_supported() is True
_card(monkeypatch, (8, 6))
assert is_fp8_gpu_supported() is False
class TestFp8Attention:
"""Acceptance 1: (8, 9) does not raise the Hopper error."""
def test_ada_converts_attention(self, monkeypatch, converts):
from soup_cli.utils.advanced_precision import apply_fp8_attention
_card(monkeypatch, (8, 9))
assert apply_fp8_attention(_Attn()) == 4
assert converts == ["tensorwise"]
def test_ampere_is_refused_without_converting(self, monkeypatch, converts):
from soup_cli.utils.advanced_precision import apply_fp8_attention
_card(monkeypatch, (8, 6))
with pytest.raises(RuntimeError, match="8.9"):
apply_fp8_attention(_Attn())
assert converts == []
def test_rowwise_on_ada_with_old_torch_is_refused(self, monkeypatch, converts):
from soup_cli.utils.advanced_precision import apply_fp8_attention
_card(monkeypatch, (8, 9), version="2.6.0")
with pytest.raises(RuntimeError, match="2.7"):
apply_fp8_attention(_Attn(), recipe="rowwise")
assert converts == []
class TestQuantizationAwareFp8:
"""Acceptance 2: (8, 6) is refused before convert_to_float8_training runs."""
def test_ampere_is_refused_before_conversion(self, monkeypatch, converts):
from soup_cli.utils.fp8 import apply_fp8_training
_card(monkeypatch, (8, 6))
with pytest.raises(RuntimeError, match="8.9"):
apply_fp8_training(_Attn())
assert converts == []
def test_ada_converts(self, monkeypatch, converts):
from soup_cli.utils.fp8 import apply_fp8_training
_card(monkeypatch, (8, 9))
assert apply_fp8_training(_Attn()) is True
assert converts == ["tensorwise"]
def test_rowwise_on_ada_with_old_torch_is_refused(self, monkeypatch, converts):
from soup_cli.utils.fp8 import apply_fp8_training
_card(monkeypatch, (8, 9), version="2.6.0")
with pytest.raises(RuntimeError, match="2.7"):
apply_fp8_training(_Attn(), recipe="rowwise")
assert converts == []
def test_the_card_is_refused_even_when_torchao_is_missing(self, monkeypatch):
"""#1044 review: the gate must run BEFORE the dependency probe.
torchao is not a default dependency, so absent is the common case. With
the checks in the other order an Ampere user who asked for FP8 got
"torchao.float8 unavailable", a yellow line and a bf16 run -- the silent
no-op this issue exists to remove. What they asked for cannot run on this
card either way."""
from soup_cli.utils.fp8 import FP8HardwareUnsupportedError, apply_fp8_training
_card(monkeypatch, (8, 6))
monkeypatch.setattr("soup_cli.utils.fp8.is_fp8_available", lambda: False)
with pytest.raises(FP8HardwareUnsupportedError, match="8.9"):
apply_fp8_training(_Attn())
def test_fp8_attention_refuses_the_card_before_naming_torchao(self, monkeypatch):
"""Same ordering in the attention path: the message must name the card,
not send the user to install a package that would not help."""
from soup_cli.utils.advanced_precision import apply_fp8_attention
from soup_cli.utils.fp8 import FP8HardwareUnsupportedError
_card(monkeypatch, (8, 6))
monkeypatch.setitem(sys.modules, "torchao", None)
with pytest.raises(FP8HardwareUnsupportedError, match="8.9"):
apply_fp8_attention(_Attn())
def test_missing_torchao_now_stops_the_run(self, monkeypatch):
"""INVERTED by the #835 ruling (2026-09-19); this test used to pin
``False``. On a card that CAN run FP8, a missing torchao no longer lets
the run go on in bf16: FP8 was asked for, and whether the reason is the
card or a package does not change what the user asked for. The message
names the fix."""
from soup_cli.utils.fp8 import FP8DependencyMissingError, apply_fp8_training
_card(monkeypatch, (9, 0))
monkeypatch.setattr("soup_cli.utils.fp8.is_fp8_available", lambda: False)
with pytest.raises(FP8DependencyMissingError, match=r"soup-cli\[qat\]"):
apply_fp8_training(_Attn())
def test_transformer_engine_alone_is_still_missing_torchao(self, monkeypatch):
"""``is_fp8_available()`` also accepts transformer-engine, but this
converter is torchao's, so a box with TE and no torchao stops too."""
from soup_cli.utils.fp8 import FP8DependencyMissingError, apply_fp8_training
_card(monkeypatch, (9, 0))
monkeypatch.setattr("soup_cli.utils.fp8.is_fp8_available", lambda: True)
for name in [n for n in sys.modules if n == "torchao" or n.startswith("torchao.")]:
monkeypatch.delitem(sys.modules, name, raising=False)
monkeypatch.setitem(sys.modules, "torchao", None)
with pytest.raises(FP8DependencyMissingError):
apply_fp8_training(_Attn())
class TestOneGate:
"""Acceptance 3: both paths call the same gate, with their recipe."""
@pytest.fixture
def gate(self, monkeypatch):
seen: list[str] = []
verdict = {"value": (False, "SENTINEL-REASON")}
def _gate(recipe="tensorwise"):
seen.append(recipe)
return verdict["value"]
monkeypatch.setattr("soup_cli.utils.fp8.fp8_training_supported", _gate)
return seen, verdict
def test_both_paths_refuse_with_the_gates_reason(self, monkeypatch, converts, gate):
from soup_cli.utils.advanced_precision import apply_fp8_attention
from soup_cli.utils.fp8 import apply_fp8_training
seen, _ = gate
# A card every old check would admit: only the gate can refuse here.
_card(monkeypatch, (12, 0))
with pytest.raises(RuntimeError, match="SENTINEL-REASON"):
apply_fp8_attention(_Attn(), recipe="rowwise")
with pytest.raises(RuntimeError, match="SENTINEL-REASON"):
apply_fp8_training(_Attn(), recipe="rowwise_with_gw_hp")
assert seen == ["rowwise", "rowwise_with_gw_hp"]
assert converts == []
def test_both_paths_convert_when_the_gate_admits(self, monkeypatch, converts, gate):
"""Control: a card every old check would refuse, admitted by the gate."""
from soup_cli.utils.advanced_precision import apply_fp8_attention
from soup_cli.utils.fp8 import apply_fp8_training
seen, verdict = gate
verdict["value"] = (True, "")
_card(monkeypatch, (7, 5))
assert apply_fp8_attention(_Attn()) == 4
assert apply_fp8_training(_Attn()) is True
assert seen == ["tensorwise", "tensorwise"]
assert converts == ["tensorwise", "tensorwise"]
class TestTheRunStops:
"""#835 ruling: an explicitly requested FP8 that the card cannot run stops the
run, on every trainer. Warning and training on without it is the defect class
v0.75.0 removed: a setting accepted, then silently not applied."""
def _console(self):
from io import StringIO
from rich.console import Console
out = StringIO()
return out, Console(file=out, width=400, force_terminal=False, color_system=None)
def test_the_shared_helper_raises_for_quantization_aware(self, monkeypatch, converts):
"""The path 11 trainers take, and SFT's once #1022 routes it here."""
from soup_cli.config.schema import TrainingConfig
from soup_cli.utils.fp8 import FP8HardwareUnsupportedError
from soup_cli.utils.v028_features import apply_v028_speed_memory
_card(monkeypatch, (8, 6))
_out, console = self._console()
with pytest.raises(FP8HardwareUnsupportedError, match="8.9"):
apply_v028_speed_memory(
model=_Attn(),
tcfg=TrainingConfig(quantization_aware="fp8"),
base_model="m",
console=console,
device="cuda",
)
assert converts == []
def test_the_shared_helper_raises_for_fp8_attention(self, monkeypatch, converts):
from soup_cli.config.schema import TrainingConfig
from soup_cli.utils.fp8 import FP8HardwareUnsupportedError
from soup_cli.utils.v028_features import apply_v028_speed_memory
_card(monkeypatch, (12, 0), version="2.7.0")
# quantization_aware passes (tensorwise); fp8_attention asks for rowwise,
# which sm120 cannot run before torch 2.8. Neither may convert.
tcfg = TrainingConfig(quantization_aware="fp8", fp8_attention=True)
monkeypatch.setattr(tcfg, "fp8_recipe", "rowwise")
monkeypatch.setattr(
"soup_cli.utils.fp8.apply_fp8_training", lambda *_a, **_k: True
)
_out, console = self._console()
with pytest.raises(FP8HardwareUnsupportedError, match="fp8_attention.*2.8"):
apply_v028_speed_memory(
model=_Attn(), tcfg=tcfg, base_model="m", console=console, device="cuda"
)
assert converts == []
def test_missing_torchao_now_stops_instead_of_warning(self, monkeypatch):
"""INVERTED by the #835 ruling (2026-09-19); this test used to pin the
yellow advisory. On a Hopper card with no torchao, both explicitly
requested FP8 settings now end the run at setup, and neither is reported
as applied."""
from soup_cli.config.schema import TrainingConfig
from soup_cli.utils.fp8 import FP8DependencyMissingError
from soup_cli.utils.v028_features import apply_v028_speed_memory
_card(monkeypatch, (9, 0))
monkeypatch.setattr("soup_cli.utils.fp8.is_fp8_available", lambda: False)
monkeypatch.setitem(sys.modules, "torchao", None)
for settings in ({"quantization_aware": "fp8"}, {"fp8_attention": True}):
out, console = self._console()
with pytest.raises(FP8DependencyMissingError, match=r"soup-cli\[qat\]"):
apply_v028_speed_memory(
model=_Attn(),
tcfg=TrainingConfig(**settings),
base_model="m",
console=console,
device="cuda",
)
assert "unavailable" not in out.getvalue(), settings
def test_a_config_that_asked_for_no_fp8_is_untouched(self, monkeypatch):
"""Control: the stop is scoped to an explicit request. The same box, no
torchao, a config without FP8 -- nothing raises, nothing is printed."""
from soup_cli.config.schema import TrainingConfig
from soup_cli.utils.v028_features import apply_v028_speed_memory
_card(monkeypatch, (9, 0))
monkeypatch.setattr("soup_cli.utils.fp8.is_fp8_available", lambda: False)
monkeypatch.setitem(sys.modules, "torchao", None)
out, console = self._console()
applied = apply_v028_speed_memory(
model=_Attn(), tcfg=TrainingConfig(), base_model="m",
console=console, device="cuda",
)
assert applied["fp8"] is False and "fp8_attention" not in applied
assert out.getvalue() == ""
def test_sft_stops_at_setup(self, monkeypatch, converts):
"""``SFTTrainerWrapper._apply_quantization_aware`` on an 8.6 card. Wrapping
its call in ``try/except RuntimeError`` used to leave every test green."""
from types import SimpleNamespace
from soup_cli.config.schema import TrainingConfig
from soup_cli.trainer.sft import SFTTrainerWrapper
from soup_cli.utils.fp8 import FP8HardwareUnsupportedError
_card(monkeypatch, (8, 6))
wrapper = object.__new__(SFTTrainerWrapper)
wrapper.model = _Attn()
wrapper.config = SimpleNamespace(base="org/model", backend="transformers")
wrapper.device = "cuda:0"
with pytest.raises(FP8HardwareUnsupportedError, match="8.9"):
wrapper._apply_quantization_aware(TrainingConfig(quantization_aware="fp8"))
assert converts == []
@pytest.mark.parametrize(
"capability, stops", [((8, 6), True), ((12, 0), False)], ids=["sm86", "sm120"]
)
def test_a_non_sft_trainer_stops_at_setup(self, monkeypatch, converts, capability, stops):
"""Through KTO's real ``_setup_transformers``: the loaders and PEFT wrapping
are stubbed, the v028 call and the FP8 gate are not. The sm120 control
completes setup and converts, so the stop is the card, not the stubs."""
from types import SimpleNamespace
import peft
import transformers
from soup_cli.config.schema import SoupConfig
from soup_cli.trainer.kto import KTOTrainerWrapper
from soup_cli.utils import peft_wiring
from soup_cli.utils.fp8 import FP8HardwareUnsupportedError
_card(monkeypatch, capability)
cfg = SoupConfig(
base="org/model",
task="kto",
data={"train": "x.jsonl", "format": "kto"},
training={"quantization": "none", "quantization_aware": "fp8"},
)
tokenizer = SimpleNamespace(pad_token=None, eos_token="</s>")
monkeypatch.setattr(
transformers.AutoTokenizer, "from_pretrained", lambda *_a, **_k: tokenizer
)
monkeypatch.setattr(
transformers.AutoModelForCausalLM, "from_pretrained", lambda *_a, **_k: _Attn()
)
monkeypatch.setattr(peft, "get_peft_model", lambda model, _config: model)
monkeypatch.setattr(peft_wiring, "resolve_lora_target_modules", lambda *_a: ["q_proj"])
monkeypatch.setattr(peft_wiring, "build_lora_config", lambda *_a, **_k: object())
monkeypatch.setattr(peft_wiring, "apply_pre_lora_patches", lambda *_a: None)
monkeypatch.setattr(peft_wiring, "apply_post_lora_patches", lambda *_a: None)
wrapper = object.__new__(KTOTrainerWrapper)
wrapper.config = cfg
wrapper.device = "cuda"
wrapper._trust_remote_code = False
if stops:
with pytest.raises(FP8HardwareUnsupportedError, match="8.9"):
wrapper._setup_transformers(cfg, cfg.training)
assert converts == []
else:
wrapper._setup_transformers(cfg, cfg.training)
assert converts == ["tensorwise"]
class TestSoupTrainReachesTheStop:
"""#1154 review: ``soup train`` ran the int8 QAT validator for ``quantization_aware:
fp8`` too, so the run stopped on "pip install torchao" before the FP8 path could
name ``soup-cli[qat]``. These drive the real command; the run stops in pre-flight,
before any model is fetched, so the base need not exist."""
def _train(
self, tmp_path, monkeypatch, *, quantization_aware, card_ok, torchao,
gate=None, recipe=None, backend=None, dry_run=False, te=False,
):
import yaml
from typer.testing import CliRunner
import soup_cli.utils.fp8 as fp8
import soup_cli.utils.qat as qat
from soup_cli.cli import app
from tests.conftest import strip_ansi
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("HF_HUB_OFFLINE", "1")
(tmp_path / "train.jsonl").write_text(
'{"messages": [{"role": "user", "content": "hi"}, '
'{"role": "assistant", "content": "hello"}]}\n',
encoding="utf-8",
)
(tmp_path / "soup.yaml").write_text(
yaml.safe_dump({
"base": "nobody/not-a-real-model",
"task": "sft",
# val_split 0: at the default 0.1 the one row goes to validation
# and `soup train` stops on an empty train split (#1217).
"data": {
"train": "train.jsonl", "format": "chatml", "max_length": 64,
"val_split": 0.0,
},
**({"backend": backend} if backend else {}),
"training": {
"epochs": 1,
"quantization": "none",
"quantization_aware": quantization_aware,
**({"fp8_recipe": recipe} if recipe else {}),
},
"output": "./out",
}),
encoding="utf-8",
)
reason = "FP8 training requires an Ada or newer GPU (compute capability >= 8.9)"
monkeypatch.setattr(
fp8, "fp8_training_supported",
gate or (lambda recipe="tensorwise": (True, "") if card_ok else (False, reason)),
)
# transformer-engine alone satisfies is_fp8_available(), not the torchao probe.
monkeypatch.setattr(fp8, "is_fp8_available", lambda: torchao or te)
monkeypatch.setattr(fp8, "is_torchao_float8_available", lambda: torchao)
monkeypatch.setattr(qat, "is_qat_available", lambda: torchao)
args = ["train", "--config", "soup.yaml", "--allow-oom-attempt", "--yes"]
result = CliRunner().invoke(app, args + (["--dry-run"] if dry_run else []))
# Rich wraps long lines; compare on single-spaced plain text.
return result, " ".join(strip_ansi(result.output).split())
def test_a_supported_card_without_torchao_names_the_extra(self, tmp_path, monkeypatch):
result, out = self._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=True, torchao=False
)
assert result.exit_code == 1
# The brackets survive: Rich would otherwise read [qat] as a markup tag.
assert 'pip install "soup-cli[qat]"' in out
assert "pip install torchao" not in out
def test_an_unsupported_card_is_named_before_torchao(self, tmp_path, monkeypatch):
result, out = self._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=False, torchao=False
)
assert result.exit_code == 1
assert "compute capability >= 8.9" in out
assert "soup-cli[qat]" not in out
assert "pip install torchao" not in out
def test_int8_qat_is_refused_before_either_torchao_hint(self, tmp_path, monkeypatch):
"""Control: ``quantization_aware: true`` used to be the int8 path, keeping its
own "pip install torchao" hint beside FP8's ``soup-cli[qat]`` one. Since #1222
it is refused when the config loads, so it reaches neither pre-flight hint.
FP8's own hint stays pinned by the two tests above."""
result, out = self._train(
tmp_path, monkeypatch, quantization_aware=True, card_ok=True, torchao=False
)
assert result.exit_code == 1
assert "training.quantization_aware: true is refused (#1222)" in out
assert "pip install" not in out
assert "QAT error" not in out
def test_fp8_with_card_and_torchao_passes_the_preflight(self, tmp_path, monkeypatch):
"""Control: the pre-flight is not refusing every FP8 config. A dry run, so
the command ends right after validation and must end there successfully --
an absent "QAT error" alone would also pass on a crash elsewhere."""
result, out = self._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=True, torchao=True,
dry_run=True,
)
assert result.exit_code == 0, out
assert "Config valid" in out
assert "QAT error" not in out and "Note:" not in out
class TestARealRunGetsThroughThePreflight:
"""#1154 review, round 4: 320e8c20 turned the only real-run pass-through control
into a dry run (a CodeRabbit suggestion), so a pre-flight that refused FP8 on a
real run with a working card and torchao -- or exited 1 silently -- stayed
green. The dry-run control stays; this real-run one sits next to it. The base
does not exist, so reaching model setup proves the pre-flight let the run go."""
def test_a_real_fp8_run_with_card_and_torchao_gets_past_the_preflight(
self, tmp_path, monkeypatch
):
_result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=True, torchao=True
)
assert "QAT error" not in out
assert "Setting up model + trainer" in out
class TestTheRecipeReachesTheGate:
"""#1154 review, round 2: every stub above ignored its ``recipe`` argument, so
dropping ``fp8_recipe=`` at the call site left the suite green -- and then
``fp8_recipe: rowwise`` on Windows told the user to install soup-cli[qat]
instead of printing the Windows refusal. This gate refuses only non-tensorwise
recipes, so it can see which recipe it was asked about."""
@staticmethod
def _gate(recipe="tensorwise"):
return (recipe == "tensorwise", "ROWWISE REFUSED")
def test_a_rowwise_refusal_is_printed_not_the_install_hint(self, tmp_path, monkeypatch):
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=True, torchao=False,
gate=self._gate, recipe="rowwise",
)
assert result.exit_code == 1
assert "ROWWISE REFUSED" in out
assert "soup-cli[qat]" not in out
def test_tensorwise_passes_the_same_gate(self, tmp_path, monkeypatch):
"""Control: the gate is not refusing everything."""
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=True, torchao=False,
gate=self._gate, recipe="tensorwise",
)
assert "ROWWISE REFUSED" not in out
assert 'pip install "soup-cli[qat]"' in out
class TestADryRunDoesNotFailOnTheLocalCard:
"""#1154 review ruling: a dry run validates a config, and FP8 configs are
written on a laptop for a remote card. The card is a note there; the
dependency and schema checks stay errors; the real run still stops."""
def test_an_unsupported_card_is_a_note_and_exits_0(self, tmp_path, monkeypatch):
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=False, torchao=True,
dry_run=True,
)
assert result.exit_code == 0, out
assert "Note: this machine could not run it:" in out
assert "compute capability >= 8.9" in out
assert "Config valid" in out
def test_a_missing_torchao_still_fails_the_dry_run(self, tmp_path, monkeypatch):
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=False, torchao=False,
dry_run=True,
)
assert result.exit_code == 1
assert 'pip install "soup-cli[qat]"' in out
def test_the_real_run_still_stops_on_the_card(self, tmp_path, monkeypatch):
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=False, torchao=True,
)
assert result.exit_code == 1
assert "compute capability >= 8.9" in out
assert "Note:" not in out
class TestThePreflightAsksForTorchaoItself:
def test_transformer_engine_alone_stops_before_the_model_loads(self, tmp_path, monkeypatch):
"""``is_fp8_available()`` accepts transformer-engine, but the converter is
torchao's: that box used to pass pre-flight, load the model, then stop."""
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=True, torchao=False,
te=True,
)
assert result.exit_code == 1
assert 'pip install "soup-cli[qat]"' in out
def test_unsloth_gets_its_refusal_without_an_install_hint(self, tmp_path, monkeypatch):
"""The unsloth refusal already stops the run; a torchao hint under it would
point at a package that cannot help."""
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=True, torchao=False,
backend="unsloth",
)
assert result.exit_code == 1
# Either wording of the unsloth refusal: soup train's "not compatible with
# the unsloth backend", or #1155's parse-time "not supported on the unsloth
# backend". What matters is that it is the unsloth refusal and nothing else.
assert "unsloth backend" in out
# Broad on purpose (round 3): the narrow check passed while the int8
# "pip install torchao" was printed instead.
assert "pip install" not in out
assert "torchao" not in out.lower().replace("not compatible", "")
class TestTheSecondAttentionRaiseSite:
def test_torchao_present_without_float8_raises_the_dependency_error(self, monkeypatch):
"""``advanced_precision``'s second raise site: torchao importable, its
float8 module not. Unreachable through the trainers (the fp8 block stops
first), so it is pinned here directly."""
import sys
import types
import torch.nn as nn
import soup_cli.utils.advanced_precision as ap
import soup_cli.utils.fp8 as fp8
monkeypatch.setitem(sys.modules, "torchao", types.ModuleType("torchao"))
monkeypatch.setitem(sys.modules, "torchao.float8", None)
monkeypatch.setattr(fp8, "fp8_training_supported", lambda recipe="tensorwise": (True, ""))
class _Attn(nn.Module):
def __init__(self):
super().__init__()
self.q_proj = nn.Linear(4, 4)
with pytest.raises(fp8.FP8DependencyMissingError, match="soup-cli"):
ap.apply_fp8_attention(_Attn())
class TestTheDryRunNoteRound3:
"""#1154 review, round 3: the note's own call site dropped the recipe, its FP8
condition was unpinned, and a dry run lacking torchao hid the card verdict."""
def test_a_rowwise_dry_run_notes_the_recipe_refusal(self, tmp_path, monkeypatch):
"""The note must ask about the configured recipe, not the default."""
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=True, torchao=True,
gate=TestTheRecipeReachesTheGate._gate, recipe="rowwise", dry_run=True,
)
assert result.exit_code == 0, out
assert "Note: this machine could not run it: ROWWISE REFUSED" in out
def test_an_int8_dry_run_is_refused_at_load_with_no_fp8_note(self, tmp_path, monkeypatch):
"""``quantization_aware: true`` no longer reaches the note: the config is
refused when it loads (#1222)."""
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware=True, card_ok=False, torchao=True,
dry_run=True,
)
assert result.exit_code == 1, out
assert "training.quantization_aware: true is refused (#1222)" in out
assert "Note:" not in out
def test_a_dry_run_without_fp8_prints_no_fp8_note(self, tmp_path, monkeypatch):
"""What the int8 case above used to pin, on a config that still loads: the
card note is FP8's, so a dry run that asks for no FP8 prints none, even on
a card that could not run FP8."""
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware=False, card_ok=False, torchao=True,
dry_run=True,
)
assert result.exit_code == 0, out
assert "Config valid" in out
assert "Note:" not in out
def test_an_unsloth_dry_run_prints_only_the_refusal(self, tmp_path, monkeypatch):
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=False, torchao=False,
backend="unsloth", dry_run=True,
)
assert result.exit_code == 1
# Either wording of the unsloth refusal: soup train's "not compatible with
# the unsloth backend", or #1155's parse-time "not supported on the unsloth
# backend". What matters is that it is the unsloth refusal and nothing else.
assert "unsloth backend" in out
assert "Note:" not in out and "pip install" not in out
def test_no_torchao_and_no_card_shows_both(self, tmp_path, monkeypatch):
"""Nit taken: the card verdict is printed before the dependency error exits."""
result, out = TestSoupTrainReachesTheStop()._train(
tmp_path, monkeypatch, quantization_aware="fp8", card_ok=False, torchao=False,
dry_run=True,
)
assert result.exit_code == 1
assert "Note: this machine could not run it:" in out
assert 'pip install "soup-cli[qat]"' in out
class TestTheTorchaoProbeItself:
"""Every other test stubs ``is_torchao_float8_available``; these run the real one
against import-machinery stubs, so a probe that answered True on ImportError
(which would let a torchao-less run load the model, then stop) fails here."""
def test_torchao_without_float8_is_not_available(self, monkeypatch):
import types
from soup_cli.utils.fp8 import is_torchao_float8_available
monkeypatch.setitem(sys.modules, "torchao", types.ModuleType("torchao"))
monkeypatch.setitem(sys.modules, "torchao.float8", None)
assert is_torchao_float8_available() is False
def test_no_torchao_at_all_is_not_available(self, monkeypatch):
from soup_cli.utils.fp8 import is_torchao_float8_available
for name in [n for n in sys.modules if n == "torchao" or n.startswith("torchao.")]:
monkeypatch.delitem(sys.modules, name, raising=False)
monkeypatch.setitem(sys.modules, "torchao", None)
assert is_torchao_float8_available() is False
def test_a_float8_module_with_the_converter_is_available(self, monkeypatch):
import types
from soup_cli.utils.fp8 import is_torchao_float8_available
pkg = types.ModuleType("torchao")
pkg.__path__ = []
float8 = types.ModuleType("torchao.float8")
float8.convert_to_float8_training = lambda *a, **k: None
pkg.float8 = float8
monkeypatch.setitem(sys.modules, "torchao", pkg)
monkeypatch.setitem(sys.modules, "torchao.float8", float8)
assert is_torchao_float8_available() is True
class TestTheConversionFailedWording:
def test_a_failed_conversion_says_so(self, monkeypatch, converts):
"""Round 1 renamed the yellow line; nothing held it (non-blocking, taken).
INVERTED by #1152 (ruled 2026-09-22): the failure now stops the run instead
of printing that line, and its message still names the conversion, not a
missing torchao."""
from soup_cli.config.schema import TrainingConfig
from soup_cli.utils.v028_features import apply_v028_speed_memory
def _fail(*_a, **_k):
raise RuntimeError("conversion blew up")
_card(monkeypatch, (9, 0))
monkeypatch.setattr(sys.modules["torchao.float8"], "convert_to_float8_training", _fail)
out, console = TestTheRunStops()._console()
with pytest.raises(RuntimeError) as info:
apply_v028_speed_memory(
model=_Attn(), tcfg=TrainingConfig(quantization_aware="fp8"), base_model="m",
console=console, device="cuda",
)
assert "the float8 conversion failed" in str(info.value)
assert "torchao.float8 missing" not in str(info.value)
assert out.getvalue() == ""