Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

17.0 KB
Raw Blame
"""SGLang backend utilities for soup serve."""

import json
import logging
from typing import Any, Optional

# The OpenAI finish_reason vocabulary Soup emits, imported rather than
# redefined: a second copy of this set is exactly the kind of duplicated source
# of truth that #372, #392 and #424 were each caused by. utils.vllm is
# torch-free at import time, so this does not weigh down CLI startup.
from soup_cli.utils.vllm import _FINISH_REASONS

logger = logging.getLogger(__name__)


def decode_sglang_response(response: Any) -> dict:
    """Normalise whatever ``Runtime.generate`` returned into a dict.

    #76 — sglang 0.5.16's ``Runtime.generate`` ends with
    ``return json.dumps(response.json())``, i.e. a **string**. Indexing it as a
    dict raised ``TypeError: string indices must be integers`` on EVERY request,
    so `--backend sglang` started cleanly and then 500'd on every generation.

    Older sglang returned the dict directly, so both shapes are accepted rather
    than one hard assumption being swapped for another.
    """
    if isinstance(response, dict):
        return response
    if isinstance(response, (str, bytes, bytearray)):
        try:
            decoded = json.loads(response)
        except (json.JSONDecodeError, UnicodeDecodeError) as exc:
            raise ValueError(
                "SGLang returned a response that is neither a dict nor JSON; "
                "the installed sglang may have changed its Runtime.generate "
                "contract again"
            ) from exc
        if not isinstance(decoded, dict):
            raise ValueError(
                f"SGLang returned JSON of type {type(decoded).__name__}, expected an object"
            )
        return decoded
    raise ValueError(
        f"SGLang returned an unsupported response type {type(response).__name__}"
    )


def generate_with_runtime(
    runtime: Any,
    prompt: str,
    prompt_token_ids: Optional[list[int]],
    sampling_params: dict,
    lora_name: Optional[str] = None,
) -> dict:
    """Run one generation on an SGLang ``Runtime`` and return the response dict (#785).

    ``Runtime.generate`` only ever posts ``{"text": prompt}``, and the server
    then tokenizes that string with its tokenizer's default
    ``add_special_tokens=True`` (``TokenizerManager._tokenize_texts``; there
    is no request or server flag to turn that off). A template that rendered
    ``{{ bos_token }}`` therefore reached the model with two of them:
    measured on a running SGLang 0.5.9 with ``unsloth/Llama-3.2-1B-Instruct``
    (``bos_token_id`` 128000), the engine ran on ``[128000, 128000, ...]``
    over 37 ids for the string Soup sent, against the 36 ids
    ``apply_chat_template(tokenize=True)`` returns; posting those 36 ids
    reproduced them exactly (``[128000, 128006, ...]``, ``prompt_tokens`` 36).

    So a prompt the template rendered is sent as the ids
    :func:`~soup_cli.utils.vllm.build_engine_prompt` encoded for it, through
    the same ``/generate`` endpoint ``Runtime.generate`` posts to, which uses
    ``input_ids`` verbatim (``TokenizerManager._tokenize_one_request``). A
    prompt no template rendered (``prompt_token_ids`` is None) still goes
    through ``Runtime.generate`` as the string it always was.

    Both routes return the same ``{"text": ..., "meta_info": {...}}`` dict:
    ``Runtime.generate`` re-encodes the server's JSON as a string (#76), the
    direct post has the dict already, and :func:`decode_sglang_response`
    accepts both.

    ``lora_name`` is the name :func:`create_sglang_runtime` registered the
    adapter under. SGLang applies a LoRA only to a request that names it in
    ``lora_path`` and runs any other request on the base weights, so both
    routes send the name whenever there is one (#1724). A full model has none,
    and its requests carry no ``lora_path`` at all.
    """
    # ``lora_path`` is SGLang's field name; it takes the registered name.
    selector = {} if lora_name is None else {"lora_path": lora_name}
    if prompt_token_ids is None:
        return decode_sglang_response(
            runtime.generate(prompt, sampling_params=sampling_params, **selector)
        )

    try:
        import httpx  # the repo's HTTP client (utils/webhooks.py, commands/generate.py)
    except ImportError as exc:
        raise ImportError("httpx is required to post token ids to the SGLang engine") from exc

    # ``timeout`` is mandatory: this is a blocking call on the app's event loop
    # (``chat_completions`` awaits it), so an unresponsive engine must be bounded
    # rather than wedging the whole server. 300s mirrors the server-generation
    # path in ``commands/generate.py``. ``rstrip`` guards a trailing slash on
    # ``runtime.url`` from producing ``//generate``.
    response = httpx.post(
        runtime.url.rstrip("/") + "/generate",
        json={"input_ids": prompt_token_ids, "sampling_params": sampling_params, **selector},
        timeout=300.0,
    )
    response.raise_for_status()
    return decode_sglang_response(response.json())


def resolve_sglang_finish_reason(meta_info: Any, max_tokens: Optional[int]) -> str:
    """Map SGLang ``meta_info`` to an OpenAI ``finish_reason`` (#360).

    Mirrors ``vllm.resolve_finish_reason``: trust the engine's own
    ``finish_reason`` when it maps to one Soup emits, otherwise derive it from
    the token count — a generation that spent the whole budget was truncated,
    not naturally stopped. The pre-fix code hardcoded ``"stop"``, so a client
    doing continue-on-length silently stopped early.

    SGLang reports ``finish_reason`` as ``{"type": "stop"|"length"|"abort"}`` in
    recent versions and as a bare string in older ones — both are accepted so a
    dict-only read cannot silently break a previously-working sglang.
    """
    info = meta_info if isinstance(meta_info, dict) else {}
    reported = info.get("finish_reason")
    if isinstance(reported, dict):
        reported = reported.get("type")
    if isinstance(reported, str) and reported in _FINISH_REASONS:
        return reported
    completion_tokens = info.get("completion_tokens")
    try:
        produced = int(completion_tokens) if completion_tokens is not None else 0
    except (TypeError, ValueError):
        produced = 0
    if max_tokens and produced >= int(max_tokens):
        return "length"
    return "stop"


def check_sglang_available() -> bool:
    """Check if SGLang is installed."""
    try:
        import sglang  # noqa: F401

        return True
    except ImportError:
        return False


def get_sglang_version() -> str:
    """Get installed SGLang version."""
    try:
        import sglang

        return getattr(sglang, "__version__", "unknown")
    except ImportError:
        return "not installed"


def create_sglang_runtime(
    model_path: str,
    base_model: Optional[str] = None,
    is_adapter: bool = False,
    tensor_parallel_size: int = 1,
    mem_fraction_static: float = 0.88,
    dtype: str = "auto",
    trust_remote_code: bool = False,
):
    """Create an SGLang Runtime for serving.

    Args:
        model_path: Path to model or LoRA adapter directory.
        base_model: Base model ID (required if model_path is a LoRA adapter).
        is_adapter: Whether model_path is a LoRA adapter.
        tensor_parallel_size: Number of GPUs for tensor parallelism.
        mem_fraction_static: Fraction of GPU memory for static allocation.
        dtype: Data type for model weights.
        trust_remote_code: Execute the model repo's custom code on load.
            Default ``False`` -- the same default-deny the vLLM path takes.
            ``serve.py`` resolves the v0.36.0 gate once per invocation and
            passes the result here. This was previously an unconditional
            ``True`` at both call sites, so the SGLang backend ran a model's
            custom code whether or not the user opted in.

    Returns:
        (runtime, runtime_model_name, lora_name) tuple. ``lora_name`` is the
        name the adapter is registered under, to be sent with every request
        (#1724); None for a full model.
    """
    import re

    import sglang as sgl

    # SSRF protection: block URL-based model paths
    for path_val in (model_path, base_model):
        if path_val and re.match(r'^https?://', path_val):
            raise ValueError(
                "model_path/base_model must be a local path or HuggingFace model ID, "
                "not a URL"
            )

    # For LoRA adapters, load the base model and register the adapter under a
    # name of its own: SGLang names a bare path entry after the whole string,
    # and splits one that holds ``=`` into a name and a different path.
    if is_adapter and base_model:
        lora_name = "adapter"  # the name the vLLM backend gives its LoRARequest
        runtime = sgl.Runtime(
            model_path=base_model,
            tp_size=tensor_parallel_size,
            mem_fraction_static=mem_fraction_static,
            dtype=dtype,
            trust_remote_code=trust_remote_code,
            lora_paths=[f"{lora_name}={model_path}"],
        )
        runtime_model_name = base_model
    else:
        lora_name = None
        runtime = sgl.Runtime(
            model_path=model_path,
            tp_size=tensor_parallel_size,
            mem_fraction_static=mem_fraction_static,
            dtype=dtype,
            trust_remote_code=trust_remote_code,
        )
        runtime_model_name = model_path

    return runtime, runtime_model_name, lora_name


def create_sglang_app(
    runtime,
    runtime_model_name: str,
    model_name: str,
    max_tokens_default: int = 512,
    tokenizer=None,
    lora_name: Optional[str] = None,
):
    """Create a FastAPI app using SGLang runtime for inference.

    Args:
        runtime: SGLang Runtime instance.
        runtime_model_name: Model name used by SGLang.
        model_name: Display model name for API responses.
        max_tokens_default: Default max tokens for generation.
        tokenizer: HF tokenizer for the served model — its chat template is
            applied by the shared ``build_engine_prompt`` (#360), which also
            encodes the rendered prompt so the server does not add a second
            BOS to it (#785). None degrades to the legacy role-prefixed
            format, same as the vLLM backend.
        lora_name: The name ``create_sglang_runtime`` registered the adapter
            under. Every request, streaming or not, names it as ``lora_path``,
            since SGLang runs a request that names no adapter on the base
            weights (#1724). None for a full model.

    Returns:
        FastAPI application.
    """
    import json
    import time
    import uuid

    from fastapi import FastAPI, HTTPException
    from fastapi.middleware.cors import CORSMiddleware
    from fastapi.responses import StreamingResponse
    from pydantic import BaseModel as PydanticBaseModel
    from pydantic import Field

    # THE single prompt builder shared with the transformers and vLLM backends
    # (#332), so the three cannot drift again. Was a hand-rolled third copy.
    from soup_cli.utils.vllm import build_engine_prompt

    app = FastAPI(title="Soup Inference Server (SGLang)", version="1.0.0")

    # Loopback-only CORS (parity with the transformers / vLLM servers). The
    # wildcard the loopback fix never reached let any web page read this local
    # server's responses.
    app.add_middleware(
        CORSMiddleware,
        allow_origin_regex=r"^https?://(localhost|127\.0\.0\.1)(:\d+)?$",
        allow_methods=["GET", "POST", "OPTIONS"],
        allow_headers=["*"],
    )

    class ChatMessage(PydanticBaseModel):
        role: str
        content: str

    class ChatCompletionRequest(PydanticBaseModel):
        model: str = model_name
        messages: list[ChatMessage]
        temperature: float = Field(default=0.7, ge=0.0, le=2.0)
        top_p: float = Field(default=0.9, ge=0.0, le=1.0)
        max_tokens: Optional[int] = Field(default=None, ge=1, le=16384)
        stream: bool = False

    @app.get("/health")
    def health():
        return {"status": "ok", "model": model_name, "backend": "sglang"}

    @app.get("/v1/models")
    def list_models():
        return {
            "object": "list",
            "data": [
                {
                    "id": model_name,
                    "object": "model",
                    "owned_by": "soup",
                }
            ],
        }

    @app.post("/v1/chat/completions")
    async def chat_completions(request: ChatCompletionRequest):
        max_tokens = request.max_tokens or max_tokens_default
        request_id = f"chatcmpl-{uuid.uuid4().hex[:8]}"

        # Apply the model's own chat template (legacy fallback when there is no
        # tokenizer / template), shared with the transformers + vLLM backends.
        # #785: and send the ids, not the string, whenever that template
        # rendered the prompt; the string route lets the server add a second
        # BOS on top of the one the template rendered (generate_with_runtime).
        prompt, prompt_token_ids = build_engine_prompt(request.messages, tokenizer)

        sampling_params = {
            "temperature": request.temperature,
            "top_p": request.top_p,
            "max_new_tokens": max_tokens,
        }

        if request.stream:
            return StreamingResponse(
                _stream_sglang_response(
                    runtime=runtime,
                    prompt=prompt,
                    prompt_token_ids=prompt_token_ids,
                    sampling_params=sampling_params,
                    request_id=request_id,
                    model_name=model_name,
                    lora_name=lora_name,
                ),
                media_type="text/event-stream",
            )

        # Non-streaming
        try:
            response = generate_with_runtime(
                runtime, prompt, prompt_token_ids, sampling_params, lora_name
            )
            response_text = response["text"]
            prompt_tokens = response.get("meta_info", {}).get("prompt_tokens", 0)
            completion_tokens = response.get(
                "meta_info", {},
            ).get("completion_tokens", len(response_text.split()))

            return {
                "id": request_id,
                "object": "chat.completion",
                "created": int(time.time()),
                "model": model_name,
                "choices": [
                    {
                        "index": 0,
                        "message": {
                            "role": "assistant",
                            "content": response_text,
                        },
                        "finish_reason": resolve_sglang_finish_reason(
                            response.get("meta_info"), max_tokens
                        ),
                    }
                ],
                "usage": {
                    "prompt_tokens": prompt_tokens,
                    "completion_tokens": completion_tokens,
                    "total_tokens": prompt_tokens + completion_tokens,
                },
            }

        except Exception:
            logger.exception("SGLang generation error")
            raise HTTPException(status_code=500, detail="Internal server error")

    async def _stream_sglang_response(
        runtime,
        prompt: str,
        prompt_token_ids: Optional[list[int]],
        sampling_params: dict,
        request_id: str,
        model_name: str,
        lora_name: Optional[str],
    ):
        """Stream SSE chunks from SGLang runtime."""
        created = int(time.time())

        try:
            response = generate_with_runtime(
                runtime, prompt, prompt_token_ids, sampling_params, lora_name
            )
            response_text = response["text"]
            meta_info = response.get("meta_info")
        except Exception:
            logger.exception("SGLang stream error")
            yield 'data: {"error": "Internal server error"}\n\n'
            return

        # Simulate streaming by sending word-by-word
        words = response_text.split(" ")
        for idx, word in enumerate(words):
            chunk_text = word if idx == 0 else f" {word}"
            chunk = {
                "id": request_id,
                "object": "chat.completion.chunk",
                "created": created,
                "model": model_name,
                "choices": [
                    {
                        "index": 0,
                        "delta": {"content": chunk_text},
                        "finish_reason": None,
                    }
                ],
            }
            yield f"data: {json.dumps(chunk)}\n\n"

        # Final chunk
        final_chunk = {
            "id": request_id,
            "object": "chat.completion.chunk",
            "created": created,
            "model": model_name,
            "choices": [
                {
                    "index": 0,
                    "delta": {},
                    "finish_reason": resolve_sglang_finish_reason(
                        meta_info, sampling_params.get("max_new_tokens")
                    ),
                }
            ],
        }
        yield f"data: {json.dumps(final_chunk)}\n\n"
        yield "data: [DONE]\n\n"

    return app