Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

15.9 KB
Raw Blame
"""Tests for Apple Silicon MLX backend — Part E of v0.25.0.

These tests mock MLX entirely so they run on CI (Linux / Windows / macOS).
"""


import pytest

from tests.conftest import strip_ansi

# ---------------------------------------------------------------------------
# MLX detection
# ---------------------------------------------------------------------------

class TestMLXDetection:
    def test_detect_mlx_not_installed(self, monkeypatch):
        """detect_mlx returns False if mlx import fails."""
        import builtins

        real_import = builtins.__import__

        def fake_import(name, *args, **kwargs):
            if name.startswith("mlx"):
                raise ImportError("no mlx")
            return real_import(name, *args, **kwargs)

        monkeypatch.setattr(builtins, "__import__", fake_import)
        from soup_cli.utils import mlx as mlx_utils

        # Force re-check via direct call
        assert mlx_utils.detect_mlx() is False

    def test_detect_mlx_installed_mock(self, monkeypatch):
        """detect_mlx returns True when mlx modules are importable (mocked)."""
        import sys
        import types

        fake_mlx = types.ModuleType("mlx")
        fake_mlx.__version__ = "0.20.0"
        fake_core = types.ModuleType("mlx.core")
        # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
        fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
        fake_core.metal = types.SimpleNamespace(is_available=lambda: True)
        fake_mlx.core = fake_core
        monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
        monkeypatch.setitem(sys.modules, "mlx.core", fake_core)

        from soup_cli.utils import mlx as mlx_utils

        assert mlx_utils.detect_mlx() is True

    def test_get_mlx_info_not_installed(self, monkeypatch):
        import builtins

        real_import = builtins.__import__

        def fake_import(name, *args, **kwargs):
            if name.startswith("mlx"):
                raise ImportError("no mlx")
            return real_import(name, *args, **kwargs)

        monkeypatch.setattr(builtins, "__import__", fake_import)
        from soup_cli.utils import mlx as mlx_utils

        info = mlx_utils.get_mlx_info()
        assert info["available"] is False

    def test_get_mlx_version_reads_core_version(self, monkeypatch):
        """get_mlx_version prefers mlx.core over stale distribution metadata (#659)."""
        import sys
        import types

        fake_mlx = types.ModuleType("mlx")  # no __version__ on the top-level pkg
        fake_core = types.ModuleType("mlx.core")
        # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
        fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
        fake_core.__version__ = "0.32.2"
        fake_mlx.core = fake_core
        monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
        monkeypatch.setitem(sys.modules, "mlx.core", fake_core)

        from soup_cli.utils import mlx as mlx_utils

        # The module version is authoritative after an in-place upgrade, while
        # distribution metadata can still report the prior installed release.
        monkeypatch.setattr(
            "importlib.metadata.version", lambda name: "0.31.0" if name == "mlx" else None
        )
        assert mlx_utils.get_mlx_version() == "0.32.2"

    def test_get_mlx_version_metadata_fallback(self, monkeypatch):
        """get_mlx_version falls back to importlib.metadata when core lacks it."""
        import sys
        import types

        fake_mlx = types.ModuleType("mlx")
        fake_core = types.ModuleType("mlx.core")
        # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
        fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
        fake_core.metal = types.SimpleNamespace(is_available=lambda: True)
        fake_mlx.core = fake_core
        monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
        monkeypatch.setitem(sys.modules, "mlx.core", fake_core)

        from soup_cli.utils import mlx as mlx_utils

        monkeypatch.setattr(
            "importlib.metadata.version", lambda name: "0.32.2" if name == "mlx" else None
        )
        assert mlx_utils.get_mlx_version() == "0.32.2"

    def test_estimate_mlx_batch_size_small_model(self):
        from soup_cli.utils.mlx import estimate_mlx_batch_size

        # 7B model on 16GB unified memory
        batch = estimate_mlx_batch_size(
            model_params_b=7.0,
            unified_memory_bytes=16 * 1024**3,
            max_length=2048,
            quantization="4bit",
        )
        assert batch >= 1

    def test_estimate_mlx_batch_size_large_model_tiny_mem(self):
        from soup_cli.utils.mlx import estimate_mlx_batch_size

        # 70B on 16GB is not going to fit — should return 1 minimum
        batch = estimate_mlx_batch_size(
            model_params_b=70.0,
            unified_memory_bytes=16 * 1024**3,
            max_length=2048,
            quantization="4bit",
        )
        assert batch >= 1


# ---------------------------------------------------------------------------
# Backend enum
# ---------------------------------------------------------------------------

class TestMLXBackendConfig:
    def test_backend_mlx_accepted(self):
        from soup_cli.config.loader import load_config_from_string

        yaml_str = """
base: mlx-community/Llama-3.1-8B-Instruct-4bit
task: sft
backend: mlx
data:
  train: ./data/train.jsonl
  format: chatml
training:
  epochs: 1
  lr: 1e-4
output: ./output
"""
        cfg = load_config_from_string(yaml_str)
        assert cfg.backend == "mlx"


# ---------------------------------------------------------------------------
# MLX SFT trainer wrapper (mocked)
# ---------------------------------------------------------------------------

class TestMLXSFTTrainer:
    def test_trainer_import(self):
        """Import the MLX SFT trainer."""
        from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper

        assert MLXSFTTrainerWrapper is not None

    def test_trainer_setup_mocked(self, tmp_path):
        from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig
        from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper

        cfg = SoupConfig(
            base="mlx-community/Llama-3.1-8B-Instruct-4bit",
            task="sft",
            backend="mlx",
            data=DataConfig(train="./data/train.jsonl", format="chatml"),
            training=TrainingConfig(epochs=1),
            output=str(tmp_path),
        )
        wrapper = MLXSFTTrainerWrapper(cfg)
        assert wrapper.config is cfg
        assert wrapper.model is None

    def test_trainer_raises_when_mlx_missing(self, tmp_path, monkeypatch):
        import builtins

        real_import = builtins.__import__

        def fake_import(name, *args, **kwargs):
            if name.startswith("mlx"):
                raise ImportError("no mlx")
            return real_import(name, *args, **kwargs)

        monkeypatch.setattr(builtins, "__import__", fake_import)

        from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig
        from soup_cli.trainer.mlx_sft import MLXSFTTrainerWrapper

        cfg = SoupConfig(
            base="mlx-community/Llama-3.1-8B-Instruct-4bit",
            task="sft",
            backend="mlx",
            data=DataConfig(train="./data/train.jsonl", format="chatml"),
            training=TrainingConfig(),
            output=str(tmp_path),
        )
        wrapper = MLXSFTTrainerWrapper(cfg)
        with pytest.raises((ImportError, RuntimeError)):
            wrapper.setup({"train": [], "val": []})


# ---------------------------------------------------------------------------
# MLX DPO + GRPO trainers — smoke import
# ---------------------------------------------------------------------------

class TestMLXOtherTrainers:
    def test_mlx_dpo_import(self):
        from soup_cli.trainer.mlx_dpo import MLXDPOTrainerWrapper

        assert MLXDPOTrainerWrapper is not None

    def test_mlx_grpo_import(self):
        from soup_cli.trainer.mlx_grpo import MLXGRPOTrainerWrapper

        assert MLXGRPOTrainerWrapper is not None


# ---------------------------------------------------------------------------
# train command routing
# ---------------------------------------------------------------------------

class TestMLXRouting:
    def test_mlx_routing_map(self):
        """Routing dict should map backend=mlx tasks to MLX trainers."""
        from soup_cli.trainer import mlx_routing

        assert mlx_routing.MLX_TRAINER_REGISTRY["sft"].__name__ == "MLXSFTTrainerWrapper"
        assert mlx_routing.MLX_TRAINER_REGISTRY["dpo"].__name__ == "MLXDPOTrainerWrapper"
        assert mlx_routing.MLX_TRAINER_REGISTRY["grpo"].__name__ == "MLXGRPOTrainerWrapper"

    def test_mlx_unsupported_task_rejected(self):
        from soup_cli.trainer import mlx_routing

        assert "ppo" not in mlx_routing.MLX_TRAINER_REGISTRY
        assert "pretrain" not in mlx_routing.MLX_TRAINER_REGISTRY
        assert "embedding" not in mlx_routing.MLX_TRAINER_REGISTRY


# ---------------------------------------------------------------------------
# Recipes
# ---------------------------------------------------------------------------

class TestMLXRecipes:
    def test_llama3_1_8b_sft_mlx(self):
        from soup_cli.config.loader import load_config_from_string
        from soup_cli.recipes.catalog import get_recipe

        recipe = get_recipe("llama3.1-8b-sft-mlx")
        assert recipe is not None
        cfg = load_config_from_string(recipe.yaml_str)
        assert cfg.backend == "mlx"
        assert cfg.task == "sft"

    def test_qwen3_8b_sft_mlx(self):
        from soup_cli.config.loader import load_config_from_string
        from soup_cli.recipes.catalog import get_recipe

        recipe = get_recipe("qwen3-8b-sft-mlx")
        assert recipe is not None
        assert recipe.model == "mlx-community/Qwen3-8B-4bit"
        cfg = load_config_from_string(recipe.yaml_str)
        assert cfg.base == "mlx-community/Qwen3-8B-4bit"
        assert cfg.backend == "mlx"

    def test_gemma3_4b_sft_mlx(self):
        from soup_cli.config.loader import load_config_from_string
        from soup_cli.recipes.catalog import get_recipe

        recipe = get_recipe("gemma3-4b-sft-mlx")
        assert recipe is not None
        assert recipe.model == "mlx-community/gemma-3-4b-it-4bit"
        assert recipe.size == "4B"
        cfg = load_config_from_string(recipe.yaml_str)
        assert cfg.base == "mlx-community/gemma-3-4b-it-4bit"
        assert cfg.backend == "mlx"

    def test_mlx_dpo_config_loads(self):
        """backend=mlx + task=dpo has a trainer (trainer/mlx_dpo.py) and loads."""
        from soup_cli.config.loader import load_config_from_string

        yaml_str = """
base: mlx-community/Llama-3.1-8B-Instruct-4bit
task: dpo
backend: mlx
data:
  train: ./x.jsonl
  format: dpo
training:
  epochs: 1
  lr: 1e-6
output: ./output
"""
        cfg = load_config_from_string(yaml_str)
        assert (cfg.task, cfg.backend) == ("dpo", "mlx")

    def test_mlx_grpo_config_rejected_at_load(self):
        from soup_cli.config.loader import load_config_from_string

        yaml_str = """
base: mlx-community/Llama-3.1-8B-Instruct-4bit
task: grpo
backend: mlx
data:
  train: ./x.jsonl
  format: chatml
training:
  epochs: 1
  lr: 1e-6
output: ./output
"""
        with pytest.raises(ValueError, match="MLX backend only ships SFT"):
            load_config_from_string(yaml_str)


class TestMlxRecipeRepoIds:
    """Pin the repo id every shipped MLX recipe declares, on RecipeMeta.model
    and YAML base: independently (#661). This does not check that the repo resolves.
    """

    MLX_RECIPES = [
        ("llama3.1-8b-sft-mlx", "mlx-community/Llama-3.1-8B-Instruct-4bit"),
        ("qwen3-8b-sft-mlx", "mlx-community/Qwen3-8B-4bit"),
        ("gemma3-4b-sft-mlx", "mlx-community/gemma-3-4b-it-4bit"),
    ]

    @pytest.mark.parametrize("name,expected_repo", MLX_RECIPES)
    def test_meta_model_matches_expected_repo(self, name: str, expected_repo: str):
        from soup_cli.recipes.catalog import get_recipe

        recipe = get_recipe(name)
        assert recipe is not None, f"Recipe {name} missing from catalog"
        assert recipe.model == expected_repo

    @pytest.mark.parametrize("name,expected_repo", MLX_RECIPES)
    def test_yaml_base_matches_expected_repo(self, name: str, expected_repo: str):
        import yaml

        from soup_cli.recipes.catalog import get_recipe

        recipe = get_recipe(name)
        assert recipe is not None
        parsed = yaml.safe_load(recipe.yaml_str)
        assert parsed["base"] == expected_repo


# ---------------------------------------------------------------------------
# doctor command reports MLX
# ---------------------------------------------------------------------------


class TestMLXDoctor:
    def test_doctor_command_renders_mlx_panel(self, monkeypatch):
        """`soup doctor` renders the MLX panel when MLX is available."""
        import sys
        import types
        from importlib.machinery import ModuleSpec

        from typer.testing import CliRunner

        from soup_cli.cli import app

        fake_mlx = types.ModuleType("mlx")
        fake_mlx.__spec__ = ModuleSpec("mlx", loader=None)
        fake_core = types.ModuleType("mlx.core")
        # #1850: MLX SFT's train() seeds mx.random before attaching LoRA.
        fake_core.random = types.SimpleNamespace(seed=lambda _seed: None)
        fake_core.__spec__ = ModuleSpec("mlx.core", loader=None)
        fake_core.__version__ = "0.32.2"
        fake_core.metal = types.SimpleNamespace(is_available=lambda: True)
        fake_mlx.core = fake_core
        monkeypatch.setitem(sys.modules, "mlx", fake_mlx)
        monkeypatch.setitem(sys.modules, "mlx.core", fake_core)

        result = CliRunner().invoke(app, ["doctor"])

        assert result.exit_code == 0, result.output
        assert "MLX" in result.output
        assert "0.32.2" in result.output

    def test_doctor_command_mlx_absent_control(self, monkeypatch):
        """`soup doctor` renders the Apple-only MLX install guidance when absent."""
        import builtins

        from typer.testing import CliRunner

        from soup_cli.cli import app

        real_import = builtins.__import__

        def fake_import(name, *args, **kwargs):
            if name == "mlx" or name == "mlx.core":
                raise ImportError("no mlx")
            return real_import(name, *args, **kwargs)

        monkeypatch.setattr(builtins, "__import__", fake_import)
        # Force the Apple-specific branch; otherwise optional dependencies also
        # render "not installed" and the assertion would be vacuous off-Mac.
        monkeypatch.setattr("soup_cli.utils.mlx.is_apple_silicon", lambda: True)
        result = CliRunner().invoke(app, ["doctor"])

        assert result.exit_code == 0
        assert "MLX" in result.output
        assert 'pip install "soup-cli[mlx]"' in strip_ansi(result.output)

    def test_doctor_command_omits_mlx_panel_off_apple_silicon(self, monkeypatch):
        """`soup doctor` stays quiet about MLX on non-Apple platforms."""
        from typer.testing import CliRunner

        from soup_cli.cli import app

        # The doctor report should not dedicate a panel to an unsupported backend.
        monkeypatch.setattr(
            "soup_cli.commands.doctor._get_mlx_info",
            lambda: {"available": False, "apple_silicon": False},
        )
        result = CliRunner().invoke(app, ["doctor"])

        assert result.exit_code == 0, result.output
        # "Apple Silicon only" was asserted here originally and is vacuous: the
        # string exists nowhere in src/, so it could never fail. Assert on a
        # string the MLX branch really does emit -- the sibling test above
        # asserts this exact hint IS present when apple_silicon is True -- so
        # this now fails if `doctor` starts advertising MLX off Apple Silicon.
        assert 'pip install "soup-cli[mlx]"' not in result.output
        assert "MLX" not in result.output


if __name__ == "__main__":
    pytest.main([__file__, "-v"])