Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

16.2 KB
Raw Blame
"""Tests for SGLang backend — detection, runtime creation, serve --backend flag, FastAPI app."""

import json
import sys
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import patch as mock_patch

import pytest

from tests.test_issue360_sglang_prompt_finish_reason import _FakeTokenizer


def _has_fastapi():
    try:
        import fastapi  # noqa: F401
        return True
    except ImportError:
        return False


# ─── SGLang Detection Tests ──────────────────────────────────────────────


class TestSGLangDetection:
    """Test SGLang availability detection."""

    def test_check_sglang_available_when_installed(self):
        """check_sglang_available should return True when sglang is importable."""
        mock_sglang = MagicMock()
        with mock_patch.dict("sys.modules", {"sglang": mock_sglang}):
            from soup_cli.utils.sglang import check_sglang_available

            assert check_sglang_available() is True

    def test_check_sglang_available_when_not_installed(self):
        """check_sglang_available should return False when sglang import fails."""
        from soup_cli.utils.sglang import check_sglang_available

        # Just verify the function exists and is callable
        assert callable(check_sglang_available)

    def test_get_sglang_version_when_installed(self):
        """get_sglang_version should return version string."""
        mock_sglang = MagicMock()
        mock_sglang.__version__ = "0.3.0"
        with mock_patch.dict("sys.modules", {"sglang": mock_sglang}):
            from soup_cli.utils.sglang import get_sglang_version

            version = get_sglang_version()
            assert version == "0.3.0"

    def test_get_sglang_version_when_not_installed(self):
        """get_sglang_version should return 'not installed' when unavailable."""
        from soup_cli.utils.sglang import get_sglang_version

        # If sglang is not actually installed, should return "not installed"
        try:
            import sglang  # noqa: F401
        except ImportError:
            assert get_sglang_version() == "not installed"


# ─── SGLang Runtime Creation Tests ───────────────────────────────────────


class TestSGLangRuntimeCreation:
    """Test SGLang runtime creation."""

    def test_create_sglang_runtime_full_model(self):
        """create_sglang_runtime should create runtime for a full model."""
        mock_runtime = MagicMock()
        mock_sgl = MagicMock()
        mock_sgl.Runtime.return_value = mock_runtime

        with mock_patch.dict("sys.modules", {"sglang": mock_sgl}):
            from soup_cli.utils.sglang import create_sglang_runtime

            runtime, model_name, _ = create_sglang_runtime(
                model_path="/path/to/model",
            )

        mock_sgl.Runtime.assert_called_once()
        call_kwargs = mock_sgl.Runtime.call_args[1]
        assert call_kwargs["model_path"] == "/path/to/model"
        # Default-deny, matching the vLLM path. This asserted True until the
        # SGLang trust-contract fix: the runtime hardcoded trust_remote_code=True
        # at both call sites, so `soup serve --backend sglang` executed a
        # model's custom code whether or not the user passed
        # --trust-remote-code. See tests/test_sglang_trust_contract.py for the
        # opt-in path and the serve.py wiring.
        assert call_kwargs["trust_remote_code"] is False
        assert model_name == "/path/to/model"

    def test_create_sglang_runtime_with_adapter(self):
        """create_sglang_runtime should pass lora_paths for adapter."""
        mock_runtime = MagicMock()
        mock_sgl = MagicMock()
        mock_sgl.Runtime.return_value = mock_runtime

        with mock_patch.dict("sys.modules", {"sglang": mock_sgl}):
            from soup_cli.utils.sglang import create_sglang_runtime

            runtime, model_name, lora_name = create_sglang_runtime(
                model_path="/path/to/adapter",
                base_model="meta-llama/Llama-3.1-8B",
                is_adapter=True,
            )

        call_kwargs = mock_sgl.Runtime.call_args[1]
        assert call_kwargs["model_path"] == "meta-llama/Llama-3.1-8B"
        assert call_kwargs["lora_paths"] == [f"{lora_name}=/path/to/adapter"]
        assert model_name == "meta-llama/Llama-3.1-8B"

    def test_create_sglang_runtime_tensor_parallel(self):
        """create_sglang_runtime should pass tp_size."""
        mock_sgl = MagicMock()
        mock_sgl.Runtime.return_value = MagicMock()

        with mock_patch.dict("sys.modules", {"sglang": mock_sgl}):
            from soup_cli.utils.sglang import create_sglang_runtime

            create_sglang_runtime(
                model_path="/model",
                tensor_parallel_size=4,
            )

        call_kwargs = mock_sgl.Runtime.call_args[1]
        assert call_kwargs["tp_size"] == 4


# ─── SGLang FastAPI App Tests ────────────────────────────────────────────


@pytest.mark.skipif(
    not _has_fastapi(),
    reason="fastapi not installed",
)
class TestSGLangApp:
    """Test SGLang FastAPI app creation (requires fastapi)."""

    def test_create_sglang_app_returns_fastapi(self):
        """create_sglang_app should return a FastAPI application."""
        mock_runtime = MagicMock()

        from soup_cli.utils.sglang import create_sglang_app

        app = create_sglang_app(
            runtime=mock_runtime,
            runtime_model_name="test-model",
            model_name="test",
        )

        assert app is not None

    def test_sglang_app_has_health_endpoint(self):
        """SGLang app should have /health endpoint."""
        mock_runtime = MagicMock()

        from soup_cli.utils.sglang import create_sglang_app

        app = create_sglang_app(
            runtime=mock_runtime,
            runtime_model_name="test-model",
            model_name="test",
        )

        routes = [route.path for route in app.routes]
        assert "/health" in routes

    def test_sglang_app_has_models_endpoint(self):
        """SGLang app should have /v1/models endpoint."""
        mock_runtime = MagicMock()

        from soup_cli.utils.sglang import create_sglang_app

        app = create_sglang_app(
            runtime=mock_runtime,
            runtime_model_name="test-model",
            model_name="test",
        )

        routes = [route.path for route in app.routes]
        assert "/v1/models" in routes

    def test_sglang_app_has_chat_completions_endpoint(self):
        """SGLang app should have /v1/chat/completions endpoint."""
        mock_runtime = MagicMock()

        from soup_cli.utils.sglang import create_sglang_app

        app = create_sglang_app(
            runtime=mock_runtime,
            runtime_model_name="test-model",
            model_name="test",
        )

        routes = [route.path for route in app.routes]
        assert "/v1/chat/completions" in routes


# ─── Serve Command SGLang Tests ──────────────────────────────────────────


class TestServeSGLangCommand:
    """Test serve command with --backend sglang."""

    def test_serve_sglang_backend_in_help(self):
        """soup serve --help should mention sglang backend."""
        from typer.testing import CliRunner

        from soup_cli.cli import app

        runner = CliRunner()
        result = runner.invoke(app, ["serve", "--help"])
        assert "sglang" in result.output

    def test_serve_sglang_not_installed_shows_error(self, tmp_path):
        """serve --backend sglang should show install hint when sglang missing."""
        from typer.testing import CliRunner

        from soup_cli.cli import app

        # Create a dummy model path
        model_dir = tmp_path / "model"
        model_dir.mkdir()

        runner = CliRunner()
        with mock_patch(
            "soup_cli.utils.sglang.check_sglang_available", return_value=False
        ):
            result = runner.invoke(app, [
                "serve",
                "--model", str(model_dir),
                "--backend", "sglang",
            ])
        assert result.exit_code != 0

    @pytest.mark.skipif(not _has_fastapi(), reason="fastapi not installed")
    def test_serve_unknown_backend_shows_error(self, tmp_path):
        """serve --backend invalid should show error listing valid backends."""
        from typer.testing import CliRunner

        from soup_cli.cli import app

        model_dir = tmp_path / "model"
        model_dir.mkdir()

        runner = CliRunner()
        result = runner.invoke(app, [
            "serve",
            "--model", str(model_dir),
            "--backend", "invalid_backend",
        ])
        assert result.exit_code != 0
        assert "sglang" in result.output  # Should list valid backends


# ─── Doctor SGLang Detection Tests ───────────────────────────────────────


class TestDoctorSGLang:
    """Test that soup doctor checks for SGLang."""

    def test_doctor_includes_sglang(self):
        """DEPS list should include sglang."""
        from soup_cli.commands.doctor import DEPS

        pkg_names = [dep[1] for dep in DEPS]
        assert "sglang" in pkg_names

    def test_doctor_includes_librosa(self):
        """DEPS list should include librosa."""
        from soup_cli.commands.doctor import DEPS

        pkg_names = [dep[1] for dep in DEPS]
        assert "librosa" in pkg_names


# ─── SGLang SSRF Validation Tests ────────────────────────────────────────


class TestSGLangSSRF:
    """Test SSRF protection in SGLang runtime creation."""

    def test_create_runtime_blocks_http_model_path(self):
        """HTTP URLs for model_path should be rejected."""
        mock_sgl = MagicMock()

        with mock_patch.dict("sys.modules", {"sglang": mock_sgl}):
            from soup_cli.utils.sglang import create_sglang_runtime

            with pytest.raises(ValueError, match="not a URL"):
                create_sglang_runtime(
                    model_path="http://evil.com/model",
                )

    def test_create_runtime_blocks_http_base_model(self):
        """HTTP URLs for base_model should be rejected."""
        mock_sgl = MagicMock()

        with mock_patch.dict("sys.modules", {"sglang": mock_sgl}):
            from soup_cli.utils.sglang import create_sglang_runtime

            with pytest.raises(ValueError, match="not a URL"):
                create_sglang_runtime(
                    model_path="/path/to/adapter",
                    base_model="https://evil.com/model",
                    is_adapter=True,
                )

    def test_create_runtime_allows_hf_model_id(self):
        """HuggingFace model IDs should be allowed."""
        mock_sgl = MagicMock()
        mock_sgl.Runtime.return_value = MagicMock()

        with mock_patch.dict("sys.modules", {"sglang": mock_sgl}):
            from soup_cli.utils.sglang import create_sglang_runtime

            runtime, name, _ = create_sglang_runtime(
                model_path="meta-llama/Llama-3.1-8B",
            )
        assert name == "meta-llama/Llama-3.1-8B"


# ─── Adapter Selected On Every Request (#1724) ───────────────────────────


class _RecordingEngine:
    """The SGLang engine as the backend reaches it, with nothing behind it.

    ``sglang.Runtime(...)`` starts it; ``Runtime.generate`` is the string route
    and a POST to ``<url>/generate`` is the token-ids route (#995). Each request
    is recorded as the fields the engine would receive.
    """

    url = "http://127.0.0.1:30000"

    def __init__(self):
        self.started_with = None
        self.requests = []

    def start(self, **kwargs):  # sglang.Runtime(...)
        self.started_with = kwargs
        return self

    def generate(self, prompt, **kwargs):  # Runtime.generate
        self.requests.append({"route": "string", "text": prompt, **kwargs})
        return json.dumps({"text": "ok", "meta_info": {}})

    def post(self, url, **kwargs):  # httpx.post(<url>/generate, json=...)
        import httpx

        self.requests.append({"route": "ids", **kwargs["json"]})
        return httpx.Response(
            200, json={"text": "ok", "meta_info": {}}, request=httpx.Request("POST", url)
        )


def _tokenizer_for(route):
    """A rendered chat template sends token ids; no template sends the string."""
    return _FakeTokenizer() if route == "ids" else None


def _serve(monkeypatch, model_dir, *, adapter, tokenizer):
    """``_serve_sglang`` as ``soup serve --backend sglang`` runs it, on a recording engine."""
    import soup_cli.commands.serve as serve_mod

    engine = _RecordingEngine()
    monkeypatch.setitem(sys.modules, "sglang", SimpleNamespace(Runtime=engine.start))
    monkeypatch.setattr("httpx.post", engine.post)
    monkeypatch.setattr(serve_mod, "_load_serve_tokenizer", lambda **kwargs: tokenizer)
    app = serve_mod._serve_sglang(
        model_path=model_dir,
        base_model="org/base-model" if adapter else None,
        is_adapter=adapter,
        max_tokens_default=16,
        tensor_parallel=1,
        gpu_memory_utilization=0.9,
    )
    return app, engine


def _chat(app, *, stream):
    from fastapi.testclient import TestClient

    response = TestClient(app).post(
        "/v1/chat/completions",
        json={"messages": [{"role": "user", "content": "hi"}], "stream": stream},
    )
    assert response.status_code == 200, response.text


def _name_sglang_gives(entry):
    """The name SGLang registers a ``lora_paths`` entry under: ``name=path`` is
    split on its first ``=``, any other string names itself (sglang 0.5.21
    ``srt/arg_groups/lora_hook.py:60-75``)."""
    return entry.split("=", 1)[0] if "=" in entry else entry


@pytest.mark.skipif(not _has_fastapi(), reason="fastapi not installed")
class TestAdapterIsSelectedOnEveryRequest:
    """SGLang applies a LoRA only to a request whose ``lora_path`` names it; a
    request without one runs on the base weights. The backend registered the
    adapter at startup and never named it, so every answer came from the base
    model while the response reported the adapter's name (#1724)."""

    @pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"])
    @pytest.mark.parametrize("route", ["ids", "string"])
    def test_every_request_names_the_registered_adapter(
        self, monkeypatch, tmp_path, route, stream
    ):
        app, engine = _serve(
            monkeypatch, tmp_path / "lora", adapter=True, tokenizer=_tokenizer_for(route)
        )

        _chat(app, stream=stream)

        (entry,) = engine.started_with["lora_paths"]
        assert [request["route"] for request in engine.requests] == [route]
        assert engine.requests[0].get("lora_path") == _name_sglang_gives(entry)

    def test_the_adapter_is_registered_under_a_name_of_its_own(self, monkeypatch, tmp_path):
        """With the path doubling as the name, SGLang splits a path that holds
        ``=`` and loads what follows it; an explicit name keeps the whole path."""
        adapter_dir = tmp_path / "lora-r=16"
        _, engine = _serve(monkeypatch, adapter_dir, adapter=True, tokenizer=None)

        (entry,) = engine.started_with["lora_paths"]
        name, path = entry.split("=", 1)  # what SGLang does with the entry

        assert path == str(adapter_dir)
        assert name

    @pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"])
    @pytest.mark.parametrize("route", ["ids", "string"])
    def test_control_a_full_model_sends_no_lora_path(self, monkeypatch, tmp_path, route, stream):
        """CONTROL: a full model registers no adapter, so its requests carry no
        ``lora_path``; naming one SGLang has not loaded makes it refuse them."""
        app, engine = _serve(
            monkeypatch, tmp_path / "model", adapter=False, tokenizer=_tokenizer_for(route)
        )

        _chat(app, stream=stream)

        assert [request["route"] for request in engine.requests] == [route]
        assert "lora_path" not in engine.requests[0]