Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

11.1 KB
Raw Blame
"""v0.40.1 Part C tests — autopilot fallback / dependency cap / quickstart
GPU-aware model pick / lr_finder import regression.
"""

from __future__ import annotations

from pathlib import Path
from unittest.mock import patch

import pytest

# --- C3: autopilot fallback to 1B (not 7B) ---------------------------------


def test_guess_params_unknown_model_falls_back_to_1b():
    from soup_cli.autopilot.analyzer import _guess_params_from_name

    assert _guess_params_from_name("tiny-gpt2") == 1.0


def test_guess_params_extracts_billions_first():
    from soup_cli.autopilot.analyzer import _guess_params_from_name

    assert _guess_params_from_name("Qwen/Qwen2.5-7B-Instruct") == 7.0


def test_guess_params_handles_millions_suffix():
    from soup_cli.autopilot.analyzer import _guess_params_from_name

    # SmolLM2-135M → 0.135 B
    assert _guess_params_from_name("HuggingFaceTB/SmolLM2-135M-Instruct") == pytest.approx(
        0.135, abs=1e-3
    )


def test_probe_cache_returns_none_for_missing_repo():
    from soup_cli.autopilot.analyzer import _probe_cache_param_count

    # Random name guaranteed not in cache.
    assert _probe_cache_param_count("nonexistent-org/never-cached-XYZ") is None


# --- C5: doctor flags unsupported breaking majors -------------------------


def test_version_ge_handles_dev_suffix():
    from soup_cli.commands.doctor import _version_ge

    assert _version_ge("5.0.0.dev0", "5.0.0") is True
    assert _version_ge("4.36.2", "5.0.0") is False


def test_version_ge_short_version_string():
    from soup_cli.commands.doctor import _version_ge

    assert _version_ge("5", "5.0.0") is True
    assert _version_ge("4.99", "5.0.0") is False


def test_max_exclusive_table_allows_transformers5_and_caps_transformers6():
    from soup_cli.commands.doctor import _MAX_EXCLUSIVE

    assert _MAX_EXCLUSIVE.get("transformers") == "6.0.0"


# --- G12: lr_finder real loop uses load_raw_data ---------------------------


def test_live_lr_sweep_uses_load_raw_data_not_load_local():
    import inspect
    import re

    from soup_cli.commands.train import _live_lr_sweep_from_config

    src = inspect.getsource(_live_lr_sweep_from_config)
    assert "load_raw_data" in src
    # The broken import was `from soup_cli.data.loader import load_local`;
    # match only that import line to avoid false positives on the comment
    # describing the fix.
    assert not re.search(r"from\s+soup_cli\.data\.loader\s+import\s+load_local", src), (
        "load_local was removed; live sweep must use load_raw_data"
    )


# --- G1: quickstart picks SmolLM2 on ≤6 GB VRAM ----------------------------


def test_pick_quickstart_model_no_cuda_uses_tinyllama():
    from soup_cli.commands import quickstart as qs

    with patch("torch.cuda.is_available", return_value=False):
        model, advisory = qs._pick_quickstart_model()
    assert model == qs._DEFAULT_MODEL
    assert advisory is None


def test_pick_quickstart_model_low_vram_switches_to_smollm():
    from soup_cli.commands import quickstart as qs

    fake_props = type("P", (), {"total_memory": int(4 * 1024**3)})()  # 4 GB
    with (
        patch("torch.cuda.is_available", return_value=True),
        patch("torch.cuda.get_device_properties", return_value=fake_props),
    ):
        model, advisory = qs._pick_quickstart_model()
    assert model == qs._LOW_VRAM_MODEL
    assert advisory is not None
    assert "VRAM" in advisory


def test_pick_quickstart_model_high_vram_keeps_default():
    from soup_cli.commands import quickstart as qs

    fake_props = type("P", (), {"total_memory": int(24 * 1024**3)})()  # 24 GB
    with (
        patch("torch.cuda.is_available", return_value=True),
        patch("torch.cuda.get_device_properties", return_value=fake_props),
    ):
        model, advisory = qs._pick_quickstart_model()
    assert model == qs._DEFAULT_MODEL
    assert advisory is None


# --- N3: GPU diagnostic distinguishes CPU build from no GPU ----------------


def test_detect_gpu_hw_without_torch_cuda_no_nvidia_smi():
    import subprocess

    from soup_cli.commands.doctor import _detect_gpu_hw_without_torch_cuda

    with patch("shutil.which", return_value=None):
        assert _detect_gpu_hw_without_torch_cuda() == ""

    # nvidia-smi present but failing → empty advisory.
    completed = subprocess.CompletedProcess(args=[], returncode=1, stdout="", stderr="")
    with (
        patch("shutil.which", return_value="/usr/bin/nvidia-smi"),
        patch("subprocess.run", return_value=completed),
    ):
        assert _detect_gpu_hw_without_torch_cuda() == ""


def test_detect_gpu_hw_returns_advisory_when_smi_succeeds():
    import subprocess

    from soup_cli.commands.doctor import _detect_gpu_hw_without_torch_cuda

    completed = subprocess.CompletedProcess(
        args=[], returncode=0, stdout="NVIDIA GeForce RTX 3050\n", stderr=""
    )
    with (
        patch("shutil.which", return_value="/usr/bin/nvidia-smi"),
        patch("subprocess.run", return_value=completed),
        patch(
            "soup_cli.commands.doctor._nvidia_smi_cuda_version",
            return_value=None,
        ),
    ):
        advisory = _detect_gpu_hw_without_torch_cuda()
    assert "RTX 3050" in advisory
    # Unknown driver: do not guess a CUDA wheel index.
    assert "download.pytorch.org/whl/" not in advisory
    assert "whl/None" not in advisory
    assert "could not be confirmed" in advisory


def test_detect_gpu_hw_uses_driver_cuda_wheel():
    import subprocess

    from soup_cli.commands.doctor import _detect_gpu_hw_without_torch_cuda

    completed = subprocess.CompletedProcess(
        args=[], returncode=0, stdout="NVIDIA GeForce RTX 5060 Ti\n", stderr=""
    )
    with (
        patch("shutil.which", return_value="/usr/bin/nvidia-smi"),
        patch("subprocess.run", return_value=completed),
        patch(
            "soup_cli.commands.doctor._nvidia_smi_cuda_version",
            return_value=(13, 2),
        ),
        # Installed CUDA torch version string, as on a real GPU box.
        # A naive `assert "cu121" not in advisory` matches this and goes
        # red locally / green on CPU-only CI.
        patch("importlib.metadata.version", return_value="2.5.1+cu121"),
    ):
        advisory = _detect_gpu_hw_without_torch_cuda()
    assert "RTX 5060 Ti" in advisory
    assert "2.5.1+cu121" in advisory
    assert "whl/cu132" in advisory
    assert "whl/cu121" not in advisory


def test_nvidia_smi_invoked_by_absolute_path():
    import subprocess

    from soup_cli.commands.doctor import (
        _detect_gpu_hw_without_torch_cuda,
        _nvidia_smi_cuda_version,
    )

    smi = "/usr/bin/nvidia-smi"
    failed = subprocess.CompletedProcess(args=[], returncode=1, stdout="", stderr="")
    with (
        patch("shutil.which", return_value=smi),
        patch("subprocess.run", return_value=failed) as run,
    ):
        _nvidia_smi_cuda_version()
    argv = run.call_args.args[0]
    assert argv[0] == smi
    assert argv[0] != "nvidia-smi"

    with (
        patch("shutil.which", return_value=smi),
        patch("subprocess.run", return_value=failed) as run,
    ):
        _detect_gpu_hw_without_torch_cuda()
    argv = run.call_args.args[0]
    assert argv[0] == smi
    assert "--query-gpu=name" in argv


def test_parse_cuda_version_from_nvidia_smi_header():
    from soup_cli.commands.doctor import _parse_cuda_version

    header = (
        "NVIDIA-SMI 596.49                 Driver Version: 596.49"
        "         CUDA Version: 13.2"
    )
    assert _parse_cuda_version(header) == (13, 2)
    assert _parse_cuda_version("CUDA Version:13.2") == (13, 2)
    assert _parse_cuda_version("CUDA UMD Version: 13.4") == (13, 4)
    assert _parse_cuda_version("CUDA  UMD  Version: 13.4") == (13, 4)
    assert _parse_cuda_version("CUDA Version: 12.10") == (12, 10)
    assert _parse_cuda_version("CUDA Version: N/A") is None
    assert _parse_cuda_version("no cuda here") is None
    assert _parse_cuda_version("") is None


def test_detect_gpu_hw_distinguishes_cuda_build_from_cpu_build(monkeypatch):
    import sys
    import types

    from soup_cli.commands.doctor import _detect_gpu_hw_without_torch_cuda

    fake_torch = types.SimpleNamespace(
        version=types.SimpleNamespace(cuda="13.0"),
    )

    monkeypatch.setitem(sys.modules, "torch", fake_torch)
    monkeypatch.setattr(
        "soup_cli.commands.doctor._nvidia_smi_cuda_version",
        lambda: (13, 0),
    )
    monkeypatch.setattr(
        "soup_cli.commands.doctor._nvidia_smi_executable",
        lambda: "/usr/bin/nvidia-smi",
    )
    monkeypatch.setattr(
        "shutil.which",
        lambda name: "/usr/bin/nvidia-smi",
    )

    completed = types.SimpleNamespace(
        returncode=0,
        stdout="NVIDIA GeForce RTX 5070 Laptop GPU\\n",
        stderr="",
    )
    monkeypatch.setattr("subprocess.run", lambda *args, **kwargs: completed)
    monkeypatch.setattr(
        "importlib.metadata.version",
        lambda name: "2.14.0+cu130",
    )

    advisory = _detect_gpu_hw_without_torch_cuda()

    assert "CUDA build (13.0) could not initialise" in advisory
    assert "the CPU build" not in advisory
    assert "--force-reinstall" in advisory
    assert "torch>=2.6.0" in advisory
    assert "whl/cu130" in advisory


def test_torch_cuda_wheel_tag_picks_supported_index():
    from soup_cli.commands.doctor import _TORCH_CUDA_WHEELS, _torch_cuda_wheel_tag

    assert (11, 8, "cu118") in _TORCH_CUDA_WHEELS

    # Fixed published-index mapping; do not derive arbitrary CUDA tags.
    assert _torch_cuda_wheel_tag((13, 4)) == "cu132"
    assert _torch_cuda_wheel_tag((13, 2)) == "cu132"
    assert _torch_cuda_wheel_tag((13, 1)) == "cu130"
    assert _torch_cuda_wheel_tag((13, 0)) == "cu130"
    assert _torch_cuda_wheel_tag((12, 8)) == "cu128"
    assert _torch_cuda_wheel_tag((12, 7)) == "cu126"
    assert _torch_cuda_wheel_tag((12, 6)) == "cu126"
    assert _torch_cuda_wheel_tag((12, 4)) == "cu124"
    assert _torch_cuda_wheel_tag((12, 1)) == "cu118"
    assert _torch_cuda_wheel_tag((11, 8)) == "cu118"
    assert _torch_cuda_wheel_tag((11, 0)) is None

    # An unreadable driver must never guess an arbitrary CUDA index.
    assert _torch_cuda_wheel_tag(None) is None


def test_readme_does_not_hardcode_a_cuda_wheel_index():
    readme = Path(__file__).resolve().parents[1] / "README.md"
    text = readme.read_text(encoding="utf-8")
    assert "download.pytorch.org/whl/cu" not in text


# --- N4: dual-Python interpreter detector ---------------------------------


def test_detect_dual_python_no_path_python():
    from soup_cli.commands.doctor import _detect_dual_python_interpreters

    with patch("shutil.which", return_value=None):
        assert _detect_dual_python_interpreters() == ""


# --- M1: rich version probe falls back to importlib.metadata --------------


def test_doctor_rich_version_probe_uses_metadata_when_module_lacks_attr():
    """Sanity: importing rich and probing version doesn't return '?'."""
    from importlib.metadata import version

    import rich

    # Rich does export __version__, so this just guards the importlib.metadata
    # fallback path is reachable and returns a real string for rich.
    assert version("rich")  # non-empty
    assert hasattr(rich, "__version__") or version("rich")