Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

22.4 KB
Raw Blame
"""Tests for vLLM backend in soup serve."""

from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest

# ============================================================
# vLLM utility tests
# ============================================================


class TestVllmDetection:
    """Test vLLM availability detection."""

    def test_is_vllm_available_when_installed(self):
        """Should return True when vllm is importable."""
        from soup_cli.utils.vllm import is_vllm_available

        mock_vllm = MagicMock()
        with patch.dict("sys.modules", {"vllm": mock_vllm}):
            assert is_vllm_available() is True

    def test_is_vllm_available_when_not_installed(self):
        """Should return False when vllm is not importable."""
        from soup_cli.utils.vllm import is_vllm_available

        with patch.dict("sys.modules", {"vllm": None}):
            assert is_vllm_available() is False

    def test_get_vllm_version_installed(self):
        """Should return version string when installed."""
        from soup_cli.utils.vllm import get_vllm_version

        mock_vllm = MagicMock()
        mock_vllm.__version__ = "0.5.0"
        with patch.dict("sys.modules", {"vllm": mock_vllm}):
            assert get_vllm_version() == "0.5.0"

    def test_get_vllm_version_not_installed(self):
        """Should return 'not installed' when vllm is missing."""
        from soup_cli.utils.vllm import get_vllm_version

        with patch.dict("sys.modules", {"vllm": None}):
            assert get_vllm_version() == "not installed"

    def test_get_vllm_version_no_attr(self):
        """Should return 'unknown' if __version__ not set."""
        from soup_cli.utils.vllm import get_vllm_version

        mock_vllm = MagicMock(spec=[])  # no __version__ attr
        with patch.dict("sys.modules", {"vllm": mock_vllm}):
            assert get_vllm_version() == "unknown"


# ============================================================
# Serve command backend flag tests
# ============================================================


class TestServeBackendFlag:
    """Test --backend flag in serve command."""

    def test_invalid_backend_rejected(self, tmp_path):
        """serve --backend invalid should fail."""
        from typer.testing import CliRunner

        from soup_cli.cli import app

        runner = CliRunner()
        # Create a fake model dir
        model_dir = tmp_path / "model"
        model_dir.mkdir()

        result = runner.invoke(
            app, ["serve", "--model", str(model_dir), "--backend", "invalid"]
        )
        assert result.exit_code != 0
        assert "unknown backend" in result.output.lower() or result.exit_code != 0

    def test_vllm_backend_not_installed(self, tmp_path):
        """serve --backend vllm should fail if vllm not installed."""
        from typer.testing import CliRunner

        from soup_cli.cli import app

        runner = CliRunner()
        model_dir = tmp_path / "model"
        model_dir.mkdir()

        with patch("soup_cli.utils.vllm.is_vllm_available", return_value=False):
            result = runner.invoke(
                app, ["serve", "--model", str(model_dir), "--backend", "vllm"]
            )
            assert result.exit_code != 0

    def test_vllm_hint_shown_when_available(self, tmp_path):
        """When vllm is installed but not selected, show hint."""
        from io import StringIO

        from rich.console import Console

        from soup_cli.commands import serve as serve_mod

        # Save original console
        original_console = serve_mod.console
        output = StringIO()
        serve_mod.console = Console(file=output)

        model_dir = tmp_path / "model"
        model_dir.mkdir()

        try:
            with patch("soup_cli.utils.vllm.is_vllm_available", return_value=True):
                # Will fail at model loading but we can check for hint output
                try:
                    serve_mod.serve.__wrapped__(
                        model=str(model_dir),
                        base_model=None,
                        port=8000,
                        host="0.0.0.0",
                        device="cpu",
                        max_tokens_default=512,
                        backend="transformers",
                        tensor_parallel=1,
                        gpu_memory_utilization=0.9,
                    )
                except (SystemExit, Exception):
                    pass
        finally:
            serve_mod.console = original_console

        # Hint may or may not appear depending on how far execution gets
        # The key test is that no crash happens
        assert output.getvalue() is not None

    def test_default_backend_is_transformers(self):
        """Default backend should be transformers."""
        import inspect

        from soup_cli.commands.serve import serve

        sig = inspect.signature(serve)
        backend_param = sig.parameters.get("backend")
        assert backend_param is not None
        assert backend_param.default.default == "transformers"

    def test_backend_accepts_vllm(self):
        """Backend param should accept 'vllm' as a value."""
        import inspect

        from soup_cli.commands.serve import serve

        sig = inspect.signature(serve)
        backend_param = sig.parameters.get("backend")
        assert backend_param is not None
        # Just check the param exists and has help text mentioning vllm
        assert "vllm" in backend_param.default.help.lower()


class TestServeTensorParallel:
    """Test --tensor-parallel flag."""

    def test_tensor_parallel_param_exists(self):
        """serve should have --tensor-parallel param."""
        import inspect

        from soup_cli.commands.serve import serve

        sig = inspect.signature(serve)
        assert "tensor_parallel" in sig.parameters

    def test_tensor_parallel_default_is_one(self):
        """Default tensor parallel size should be 1."""
        import inspect

        from soup_cli.commands.serve import serve

        sig = inspect.signature(serve)
        tp_param = sig.parameters["tensor_parallel"]
        assert tp_param.default.default == 1

    def test_gpu_memory_param_exists(self):
        """serve should have --gpu-memory param."""
        import inspect

        from soup_cli.commands.serve import serve

        sig = inspect.signature(serve)
        assert "gpu_memory_utilization" in sig.parameters

    def test_gpu_memory_default(self):
        """Default GPU memory utilization should be 0.9."""
        import inspect

        from soup_cli.commands.serve import serve

        sig = inspect.signature(serve)
        param = sig.parameters["gpu_memory_utilization"]
        assert param.default.default == 0.9


# ============================================================
# vLLM engine creation tests
# ============================================================


class TestCreateVllmEngine:
    """Test vLLM engine creation logic."""

    def test_create_engine_full_model(self):
        """create_vllm_engine with full model should use model_path directly."""
        mock_engine = MagicMock()
        mock_args_cls = MagicMock()
        mock_engine_cls = MagicMock()
        mock_engine_cls.from_engine_args.return_value = mock_engine

        with patch.dict("sys.modules", {
            "vllm": MagicMock(
                AsyncEngineArgs=mock_args_cls,
                AsyncLLMEngine=mock_engine_cls,
            ),
        }):
            from importlib import reload

            import soup_cli.utils.vllm as vllm_mod

            reload(vllm_mod)

            engine, name = vllm_mod.create_vllm_engine(
                model_path="/path/to/model",
                is_adapter=False,
                tensor_parallel_size=2,
                gpu_memory_utilization=0.8,
            )

            assert engine == mock_engine
            assert name == "/path/to/model"
            mock_args_cls.assert_called_once()
            call_kwargs = mock_args_cls.call_args
            assert call_kwargs.kwargs["model"] == "/path/to/model"
            assert call_kwargs.kwargs["tensor_parallel_size"] == 2
            assert call_kwargs.kwargs["gpu_memory_utilization"] == 0.8

    def test_create_engine_adapter(self):
        """create_vllm_engine with adapter should use base_model and enable LoRA."""
        mock_engine = MagicMock()
        mock_args_cls = MagicMock()
        mock_engine_cls = MagicMock()
        mock_engine_cls.from_engine_args.return_value = mock_engine

        with patch.dict("sys.modules", {
            "vllm": MagicMock(
                AsyncEngineArgs=mock_args_cls,
                AsyncLLMEngine=mock_engine_cls,
            ),
        }):
            from importlib import reload

            import soup_cli.utils.vllm as vllm_mod

            reload(vllm_mod)

            engine, name = vllm_mod.create_vllm_engine(
                model_path="/path/to/adapter",
                base_model="meta-llama/Llama-3.1-8B",
                is_adapter=True,
            )

            assert engine == mock_engine
            assert name == "meta-llama/Llama-3.1-8B"
            call_kwargs = mock_args_cls.call_args
            assert call_kwargs.kwargs["model"] == "meta-llama/Llama-3.1-8B"
            assert call_kwargs.kwargs["enable_lora"] is True

    def test_create_engine_max_model_len(self):
        """create_vllm_engine should set max_model_len when provided."""
        mock_engine = MagicMock()
        mock_args_cls = MagicMock()
        mock_engine_cls = MagicMock()
        mock_engine_cls.from_engine_args.return_value = mock_engine

        with patch.dict("sys.modules", {
            "vllm": MagicMock(
                AsyncEngineArgs=mock_args_cls,
                AsyncLLMEngine=mock_engine_cls,
            ),
        }):
            from importlib import reload

            import soup_cli.utils.vllm as vllm_mod

            reload(vllm_mod)

            engine, name = vllm_mod.create_vllm_engine(
                model_path="/path/to/model",
                is_adapter=False,
                max_model_len=4096,
            )

            args_instance = mock_args_cls.return_value
            assert args_instance.max_model_len == 4096


# ============================================================
# vLLM app creation tests
# ============================================================


class TestCreateVllmApp:
    """Test vLLM FastAPI app creation."""

    def _make_mock_engine(self):
        """Create a mock vLLM engine."""
        engine = MagicMock()
        return engine

    def test_create_vllm_app_returns_fastapi(self):
        """create_vllm_app should return a FastAPI app."""
        try:
            from fastapi import FastAPI
        except ImportError:
            pytest.skip("FastAPI not installed")

        mock_vllm = MagicMock()
        mock_vllm.SamplingParams = MagicMock()

        with patch.dict("sys.modules", {
            "vllm": mock_vllm,
            "vllm.lora": MagicMock(),
            "vllm.lora.request": MagicMock(),
        }):
            from importlib import reload

            import soup_cli.utils.vllm as vllm_mod

            reload(vllm_mod)

            engine = self._make_mock_engine()
            app = vllm_mod.create_vllm_app(
                engine=engine,
                engine_model_name="test-model",
                model_name="test-model",
                max_tokens_default=256,
            )

            assert isinstance(app, FastAPI)

    def test_vllm_app_has_correct_routes(self):
        """vLLM app should have health, models, and chat completions routes."""
        try:
            import fastapi  # noqa: F401
        except ImportError:
            pytest.skip("FastAPI not installed")

        mock_vllm = MagicMock()
        mock_vllm.SamplingParams = MagicMock()

        with patch.dict("sys.modules", {
            "vllm": mock_vllm,
            "vllm.lora": MagicMock(),
            "vllm.lora.request": MagicMock(),
        }):
            from importlib import reload

            import soup_cli.utils.vllm as vllm_mod

            reload(vllm_mod)

            engine = self._make_mock_engine()
            app = vllm_mod.create_vllm_app(
                engine=engine,
                engine_model_name="test-model",
                model_name="test-model",
                max_tokens_default=256,
            )

            routes = [route.path for route in app.routes]
            assert "/health" in routes
            assert "/v1/models" in routes
            assert "/v1/chat/completions" in routes

    def test_vllm_health_endpoint(self):
        """Health endpoint should return backend: vllm."""
        try:
            from fastapi.testclient import TestClient
        except ImportError:
            pytest.skip("FastAPI not installed")

        mock_vllm = MagicMock()
        mock_vllm.SamplingParams = MagicMock()

        with patch.dict("sys.modules", {
            "vllm": mock_vllm,
            "vllm.lora": MagicMock(),
            "vllm.lora.request": MagicMock(),
        }):
            from importlib import reload

            import soup_cli.utils.vllm as vllm_mod

            reload(vllm_mod)

            engine = self._make_mock_engine()
            app = vllm_mod.create_vllm_app(
                engine=engine,
                engine_model_name="test-model",
                model_name="test-model",
                max_tokens_default=256,
            )

            client = TestClient(app)
            response = client.get("/health")
            assert response.status_code == 200
            data = response.json()
            assert data["status"] == "ok"
            assert data["model"] == "test-model"
            assert data["backend"] == "vllm"

    def test_vllm_models_endpoint(self):
        """Models endpoint should list the served model."""
        try:
            from fastapi.testclient import TestClient
        except ImportError:
            pytest.skip("FastAPI not installed")

        mock_vllm = MagicMock()
        mock_vllm.SamplingParams = MagicMock()

        with patch.dict("sys.modules", {
            "vllm": mock_vllm,
            "vllm.lora": MagicMock(),
            "vllm.lora.request": MagicMock(),
        }):
            from importlib import reload

            import soup_cli.utils.vllm as vllm_mod

            reload(vllm_mod)

            engine = self._make_mock_engine()
            app = vllm_mod.create_vllm_app(
                engine=engine,
                engine_model_name="my-model",
                model_name="my-model",
                max_tokens_default=256,
            )

            client = TestClient(app)
            response = client.get("/v1/models")
            assert response.status_code == 200
            data = response.json()
            assert data["object"] == "list"
            assert len(data["data"]) == 1
            assert data["data"][0]["id"] == "my-model"


# ============================================================
# Serve command integration with vLLM
# ============================================================


class TestServeVllmIntegration:
    """Test _serve_vllm helper."""

    def test_serve_vllm_creates_app(self, tmp_path):
        """_serve_vllm should create engine and return FastAPI app."""
        try:
            import fastapi  # noqa: F401
        except ImportError:
            pytest.skip("FastAPI not installed")

        mock_engine = MagicMock()
        mock_app = MagicMock()

        with patch(
            "soup_cli.utils.vllm.create_vllm_engine",
            return_value=(mock_engine, "base-model"),
        ) as mock_create_engine, patch(
            "soup_cli.utils.vllm.create_vllm_app",
            return_value=mock_app,
        ) as mock_create_app:
            from soup_cli.commands.serve import _serve_vllm

            model_path = tmp_path / "model"
            model_path.mkdir()

            app = _serve_vllm(
                model_path=model_path,
                base_model="base-model",
                is_adapter=False,
                max_tokens_default=512,
                tensor_parallel=2,
                gpu_memory_utilization=0.85,
            )

            assert app == mock_app
            mock_create_engine.assert_called_once_with(
                model_path=str(model_path),
                base_model="base-model",
                is_adapter=False,
                tensor_parallel_size=2,
                gpu_memory_utilization=0.85,
                speculative_model=None,
                num_speculative_tokens=5,
                enable_prefix_caching=False,
                quantization=None,  # default: model's native dtype
                trust_remote_code=False,  # v0.71.33 — default-deny gate
                max_model_len=None,  # #333 — new --max-model-len lever
            )
            mock_create_app.assert_called_once()

    def test_serve_vllm_with_adapter(self, tmp_path):
        """_serve_vllm with adapter should pass adapter_path."""
        try:
            import fastapi  # noqa: F401
        except ImportError:
            pytest.skip("FastAPI not installed")

        mock_engine = MagicMock()
        mock_app = MagicMock()

        with patch(
            "soup_cli.utils.vllm.create_vllm_engine",
            return_value=(mock_engine, "base-model"),
        ), patch(
            "soup_cli.utils.vllm.create_vllm_app",
            return_value=mock_app,
        ) as mock_create_app:
            from soup_cli.commands.serve import _serve_vllm

            model_path = tmp_path / "adapter"
            model_path.mkdir()

            _serve_vllm(
                model_path=model_path,
                base_model="base-model",
                is_adapter=True,
                max_tokens_default=256,
                tensor_parallel=1,
                gpu_memory_utilization=0.9,
            )

            # Check adapter_path was passed
            call_kwargs = mock_create_app.call_args
            assert call_kwargs.kwargs["adapter_path"] == str(model_path)


# ============================================================
# Existing serve tests still pass (transformers backend)
# ============================================================


class TestTransformersBackendUnchanged:
    """Verify transformers backend still works as before."""

    def test_create_app_returns_fastapi_instance(self):
        """_create_app should return a FastAPI app with correct endpoints."""
        try:
            from fastapi import FastAPI
        except ImportError:
            pytest.skip("FastAPI not installed")

        from soup_cli.commands.serve import _create_app

        mock_model = MagicMock()
        mock_tokenizer = MagicMock()

        app = _create_app(
            model_obj=mock_model,
            tokenizer=mock_tokenizer,
            device="cpu",
            model_name="test-model",
            max_tokens_default=256,
        )

        assert isinstance(app, FastAPI)

        routes = [route.path for route in app.routes]
        assert "/health" in routes
        assert "/v1/models" in routes
        assert "/v1/chat/completions" in routes

    def test_health_endpoint_transformers(self):
        """Health endpoint with transformers backend returns ok."""
        try:
            from fastapi.testclient import TestClient
        except ImportError:
            pytest.skip("FastAPI not installed")

        from soup_cli.commands.serve import _create_app

        app = _create_app(
            model_obj=MagicMock(),
            tokenizer=MagicMock(),
            device="cpu",
            model_name="test-model",
            max_tokens_default=256,
        )

        client = TestClient(app)
        response = client.get("/health")
        assert response.status_code == 200
        data = response.json()
        assert data["status"] == "ok"


# ============================================================
# CLI registration tests
# ============================================================


class TestServeCliRegistration:
    """Test serve command is properly registered."""

    @staticmethod
    def _strip_ansi(text: str) -> str:
        """Remove ANSI escape codes from text."""
        import re

        return re.sub(r"\x1b\[[0-9;]*m", "", text)

    def test_serve_help_shows_backend(self):
        """soup serve --help should mention --backend flag."""
        from typer.testing import CliRunner

        from soup_cli.cli import app

        runner = CliRunner()
        result = runner.invoke(app, ["serve", "--help"])
        assert result.exit_code == 0
        clean = self._strip_ansi(result.output)
        assert "--backend" in clean

    def test_serve_help_shows_tensor_parallel(self):
        """soup serve --help should mention --tensor-parallel flag."""
        from typer.testing import CliRunner

        from soup_cli.cli import app

        runner = CliRunner()
        result = runner.invoke(app, ["serve", "--help"])
        assert result.exit_code == 0
        clean = self._strip_ansi(result.output)
        assert "--tensor-parallel" in clean

    def test_serve_help_shows_gpu_memory(self):
        """soup serve --help should mention --gpu-memory flag."""
        from typer.testing import CliRunner

        from soup_cli.cli import app

        runner = CliRunner()
        result = runner.invoke(app, ["serve", "--help"])
        assert result.exit_code == 0
        clean = self._strip_ansi(result.output)
        assert "--gpu-memory" in clean


# ============================================================
# pyproject.toml extra tests
# ============================================================


class TestServeFastExtra:
    """Test serve-fast extra is properly defined."""

    def test_serve_fast_extra_in_pyproject(self):
        """pyproject.toml should have serve-fast extra with vllm."""
        toml_path = Path(__file__).parent.parent / "pyproject.toml"
        content = toml_path.read_text()
        assert "serve-fast" in content
        assert "vllm" in content


# ============================================================
# Version detection tests
# ============================================================


class TestVersionDetectsVllm:
    """Test that version --full detects vllm."""

    def test_version_full_checks_vllm(self):
        """version --full extras should include serve-fast check."""
        from typer.testing import CliRunner

        from soup_cli.cli import app

        runner = CliRunner()
        # Run version --full — vllm won't be installed but it shouldn't crash
        result = runner.invoke(app, ["version", "--full"])
        assert result.exit_code == 0
        # Should NOT show serve-fast since vllm not installed
        # But command should complete successfully