Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Tests for Apple Silicon MLX backend — Part E of v0.25.0.
These tests mock MLX entirely so they run on CI (Linux / Windows / macOS).
"""
import pytest
from tests.conftest import strip_ansi
# ---------------------------------------------------------------------------
# MLX detection
# ---------------------------------------------------------------------------
class TestMLXDetection:
def test_detect_mlx_not_installed(self, monkeypatch):
"""detect_mlx returns False if mlx import fails."""
import builtins
real_import = builtins.__import__
def fake_import(name, *args, **kwargs):
if name.startswith("mlx"):
raise ImportError("no mlx")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", fake_import)
from soup_cli.utils import mlx as mlx_utils
# Force re-check via direct call
assert mlx_utils.detect_mlx() is False
def test_detect_mlx_installed_mock(self, monkeypatch):
"""detect_mlx returns True when mlx modules are importable (mocked)."""
import sys
import types
fake_mlx = types.ModuleType("mlx")
fake_mlx.__version__ = "0.20.0"
fake_core = types.ModuleType("mlx.core")
# #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
fake_core.metal = types.SimpleNamespace(is_available=lambda: True)
fake_mlx.core = fake_core
monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
monkeypatch.setitem(sys.modules, "mlx.core", fake_core)
from soup_cli.utils import mlx as mlx_utils
assert mlx_utils.detect_mlx() is True
def test_get_mlx_info_not_installed(self, monkeypatch):
import builtins
real_import = builtins.__import__
def fake_import(name, *args, **kwargs):
if name.startswith("mlx"):
raise ImportError("no mlx")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", fake_import)
from soup_cli.utils import mlx as mlx_utils
info = mlx_utils.get_mlx_info()
assert info["available"] is False
def test_get_mlx_version_reads_core_version(self, monkeypatch):
"""get_mlx_version prefers mlx.core over stale distribution metadata (#659)."""
import sys
import types
fake_mlx = types.ModuleType("mlx") # no __version__ on the top-level pkg
fake_core = types.ModuleType("mlx.core")
# #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
fake_core.__version__ = "0.32.2"
fake_mlx.core = fake_core
monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
monkeypatch.setitem(sys.modules, "mlx.core", fake_core)
from soup_cli.utils import mlx as mlx_utils
# The module version is authoritative after an in-place upgrade, while
# distribution metadata can still report the prior installed release.
monkeypatch.setattr(
"importlib.metadata.version", lambda name: "0.31.0" if name == "mlx" else None
)
assert mlx_utils.get_mlx_version() == "0.32.2"
def test_get_mlx_version_metadata_fallback(self, monkeypatch):
"""get_mlx_version falls back to importlib.metadata when core lacks it."""
import sys
import types
fake_mlx = types.ModuleType("mlx")
fake_core = types.ModuleType("mlx.core")
# #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
fake_core.metal = types.SimpleNamespace(is_available=lambda: True)
fake_mlx.core = fake_core
monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
monkeypatch.setitem(sys.modules, "mlx.core", fake_core)
from soup_cli.utils import mlx as mlx_utils
monkeypatch.setattr(
"importlib.metadata.version", lambda name: "0.32.2" if name == "mlx" else None
)
assert mlx_utils.get_mlx_version() == "0.32.2"
def test_estimate_mlx_batch_size_small_model(self):
from soup_cli.utils.mlx import estimate_mlx_batch_size
# 7B model on 16GB unified memory
batch = estimate_mlx_batch_size(
model_params_b=7.0,
unified_memory_bytes=16 * 1024**3,
max_length=2048,
quantization="4bit",
)
assert batch >= 1
def test_estimate_mlx_batch_size_large_model_tiny_mem(self):
from soup_cli.utils.mlx import estimate_mlx_batch_size
# 70B on 16GB is not going to fit — should return 1 minimum
batch = estimate_mlx_batch_size(
model_params_b=70.0,
unified_memory_bytes=16 * 1024**3,
max_length=2048,
quantization="4bit",
)
assert batch >= 1
# ---------------------------------------------------------------------------
# Backend enum
# ---------------------------------------------------------------------------
class TestMLXBackendConfig:
def test_backend_mlx_accepted(self):
from soup_cli.config.loader import load_config_from_string
yaml_str = """
base: mlx-community/Llama-3.1-8B-Instruct-4bit
task: sft
backend: mlx
data:
train: ./data/train.jsonl
format: chatml
training:
epochs: 1
lr: 1e-4
output: ./output
"""
cfg = load_config_from_string(yaml_str)
assert cfg.backend == "mlx"
# ---------------------------------------------------------------------------
# MLX SFT trainer wrapper (mocked)
# ---------------------------------------------------------------------------
class TestMLXSFTTrainer:
def test_trainer_import(self):
"""Import the MLX SFT trainer."""
from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper
assert MLXSFTTrainerWrapper is not None
def test_trainer_setup_mocked(self, tmp_path):
from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig
from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper
cfg = SoupConfig(
base="mlx-community/Llama-3.1-8B-Instruct-4bit",
task="sft",
backend="mlx",
data=DataConfig(train="./data/train.jsonl", format="chatml"),
training=TrainingConfig(epochs=1),
output=str(tmp_path),
)
wrapper = MLXSFTTrainerWrapper(cfg)
assert wrapper.config is cfg
assert wrapper.model is None
def test_trainer_raises_when_mlx_missing(self, tmp_path, monkeypatch):
import builtins
real_import = builtins.__import__
def fake_import(name, *args, **kwargs):
if name.startswith("mlx"):
raise ImportError("no mlx")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", fake_import)
from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig
from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper
cfg = SoupConfig(
base="mlx-community/Llama-3.1-8B-Instruct-4bit",
task="sft",
backend="mlx",
data=DataConfig(train="./data/train.jsonl", format="chatml"),
training=TrainingConfig(),
output=str(tmp_path),
)
wrapper = MLXSFTTrainerWrapper(cfg)
with pytest.raises((ImportError, RuntimeError)):
wrapper.setup({"train": [], "val": []})
# ---------------------------------------------------------------------------
# MLX DPO + GRPO trainers — smoke import
# ---------------------------------------------------------------------------
class TestMLXOtherTrainers:
def test_mlx_dpo_import(self):
from soup_cli.trainer.mlx_dpo import MLXDPOTrainerWrapper
assert MLXDPOTrainerWrapper is not None
def test_mlx_grpo_import(self):
from soup_cli.trainer.mlx_grpo import MLXGRPOTrainerWrapper
assert MLXGRPOTrainerWrapper is not None
# ---------------------------------------------------------------------------
# train command routing
# ---------------------------------------------------------------------------
class TestMLXRouting:
def test_mlx_routing_map(self):
"""Routing dict should map backend=mlx tasks to MLX trainers."""
from soup_cli.trainer import mlx_routing
assert mlx_routing.MLX_TRAINER_REGISTRY["sft"].__name__ == "MLXSFTTrainerWrapper"
assert mlx_routing.MLX_TRAINER_REGISTRY["dpo"].__name__ == "MLXDPOTrainerWrapper"
assert mlx_routing.MLX_TRAINER_REGISTRY["grpo"].__name__ == "MLXGRPOTrainerWrapper"
def test_mlx_unsupported_task_rejected(self):
from soup_cli.trainer import mlx_routing
assert "ppo" not in mlx_routing.MLX_TRAINER_REGISTRY
assert "pretrain" not in mlx_routing.MLX_TRAINER_REGISTRY
assert "embedding" not in mlx_routing.MLX_TRAINER_REGISTRY
# ---------------------------------------------------------------------------
# Recipes
# ---------------------------------------------------------------------------
class TestMLXRecipes:
def test_llama3_1_8b_sft_mlx(self):
from soup_cli.config.loader import load_config_from_string
from soup_cli.recipes.catalog import get_recipe
recipe = get_recipe("llama3.1-8b-sft-mlx")
assert recipe is not None
cfg = load_config_from_string(recipe.yaml_str)
assert cfg.backend == "mlx"
assert cfg.task == "sft"
def test_qwen3_8b_sft_mlx(self):
from soup_cli.config.loader import load_config_from_string
from soup_cli.recipes.catalog import get_recipe
recipe = get_recipe("qwen3-8b-sft-mlx")
assert recipe is not None
assert recipe.model == "mlx-community/Qwen3-8B-4bit"
cfg = load_config_from_string(recipe.yaml_str)
assert cfg.base == "mlx-community/Qwen3-8B-4bit"
assert cfg.backend == "mlx"
def test_gemma3_4b_sft_mlx(self):
from soup_cli.config.loader import load_config_from_string
from soup_cli.recipes.catalog import get_recipe
recipe = get_recipe("gemma3-4b-sft-mlx")
assert recipe is not None
assert recipe.model == "mlx-community/gemma-3-4b-it-4bit"
assert recipe.size == "4B"
cfg = load_config_from_string(recipe.yaml_str)
assert cfg.base == "mlx-community/gemma-3-4b-it-4bit"
assert cfg.backend == "mlx"
def test_mlx_dpo_config_loads(self):
"""backend=mlx + task=dpo has a trainer (trainer/mlx_dpo.py) and loads."""
from soup_cli.config.loader import load_config_from_string
yaml_str = """
base: mlx-community/Llama-3.1-8B-Instruct-4bit
task: dpo
backend: mlx
data:
train: ./x.jsonl
format: dpo
training:
epochs: 1
lr: 1e-6
output: ./output
"""
cfg = load_config_from_string(yaml_str)
assert (cfg.task, cfg.backend) == ("dpo", "mlx")
def test_mlx_grpo_config_rejected_at_load(self):
from soup_cli.config.loader import load_config_from_string
yaml_str = """
base: mlx-community/Llama-3.1-8B-Instruct-4bit
task: grpo
backend: mlx
data:
train: ./x.jsonl
format: chatml
training:
epochs: 1
lr: 1e-6
output: ./output
"""
with pytest.raises(ValueError, match="MLX backend only ships SFT"):
load_config_from_string(yaml_str)
class TestMlxRecipeRepoIds:
"""Pin the repo id every shipped MLX recipe declares, on RecipeMeta.model
and YAML base: independently (#661). This does not check that the repo resolves.
"""
MLX_RECIPES = [
("llama3.1-8b-sft-mlx", "mlx-community/Llama-3.1-8B-Instruct-4bit"),
("qwen3-8b-sft-mlx", "mlx-community/Qwen3-8B-4bit"),
("gemma3-4b-sft-mlx", "mlx-community/gemma-3-4b-it-4bit"),
]
@pytest.mark.parametrize("name,expected_repo", MLX_RECIPES)
def test_meta_model_matches_expected_repo(self, name: str, expected_repo: str):
from soup_cli.recipes.catalog import get_recipe
recipe = get_recipe(name)
assert recipe is not None, f"Recipe {name} missing from catalog"
assert recipe.model == expected_repo
@pytest.mark.parametrize("name,expected_repo", MLX_RECIPES)
def test_yaml_base_matches_expected_repo(self, name: str, expected_repo: str):
import yaml
from soup_cli.recipes.catalog import get_recipe
recipe = get_recipe(name)
assert recipe is not None
parsed = yaml.safe_load(recipe.yaml_str)
assert parsed["base"] == expected_repo
# ---------------------------------------------------------------------------
# doctor command reports MLX
# ---------------------------------------------------------------------------
class TestMLXDoctor:
def test_doctor_command_renders_mlx_panel(self, monkeypatch):
"""`soup doctor` renders the MLX panel when MLX is available."""
import sys
import types
from importlib.machinery import ModuleSpec
from typer.testing import CliRunner
from soup_cli.cli import app
fake_mlx = types.ModuleType("mlx")
fake_mlx.__spec__ = ModuleSpec("mlx", loader=None)
fake_core = types.ModuleType("mlx.core")
# #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
fake_core.__spec__ = ModuleSpec("mlx.core", loader=None)
fake_core.__version__ = "0.32.2"
fake_core.metal = types.SimpleNamespace(is_available=lambda: True)
fake_mlx.core = fake_core
monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
monkeypatch.setitem(sys.modules, "mlx.core", fake_core)
result = CliRunner().invoke(app, ["doctor"])
assert result.exit_code == 0, result.output
assert "MLX" in result.output
assert "0.32.2" in result.output
def test_doctor_command_mlx_absent_control(self, monkeypatch):
"""`soup doctor` renders the Apple-only MLX install guidance when absent."""
import builtins
from typer.testing import CliRunner
from soup_cli.cli import app
real_import = builtins.__import__
def fake_import(name, *args, **kwargs):
if name == "mlx" or name == "mlx.core":
raise ImportError("no mlx")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", fake_import)
# Force the Apple-specific branch; otherwise optional dependencies also
# render "not installed" and the assertion would be vacuous off-Mac.
monkeypatch.setattr("soup_cli.utils.mlx.is_apple_silicon", lambda: True)
result = CliRunner().invoke(app, ["doctor"])
assert result.exit_code == 0
assert "MLX" in result.output
assert 'pip install "soup-cli[mlx]"' in strip_ansi(result.output)
def test_doctor_command_omits_mlx_panel_off_apple_silicon(self, monkeypatch):
"""`soup doctor` stays quiet about MLX on non-Apple platforms."""
from typer.testing import CliRunner
from soup_cli.cli import app
# The doctor report should not dedicate a panel to an unsupported backend.
monkeypatch.setattr(
"soup_cli.commands.doctor._get_mlx_info",
lambda: {"available": False, "apple_silicon": False},
)
result = CliRunner().invoke(app, ["doctor"])
assert result.exit_code == 0, result.output
# "Apple Silicon only" was asserted here originally and is vacuous: the
# string exists nowhere in src/, so it could never fail. Assert on a
# string the MLX branch really does emit -- the sibling test above
# asserts this exact hint IS present when apple_silicon is True -- so
# this now fails if `doctor` starts advertising MLX off Apple Silicon.
assert 'pip install "soup-cli[mlx]"' not in result.output
assert "MLX" not in result.output
if __name__ == "__main__":
pytest.main([__file__, "-v"])