Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""SGLang backend utilities for soup serve."""
import json
import logging
from typing import Any, Optional
# The OpenAI finish_reason vocabulary Soup emits, imported rather than
# redefined: a second copy of this set is exactly the kind of duplicated source
# of truth that #372, #392 and #424 were each caused by. utils.vllm is
# torch-free at import time, so this does not weigh down CLI startup.
from soup_cli.utils.vllm import _FINISH_REASONS
logger = logging.getLogger(__name__)
def decode_sglang_response(response: Any) -> dict:
"""Normalise whatever ``Runtime.generate`` returned into a dict.
#76 — sglang 0.5.16's ``Runtime.generate`` ends with
``return json.dumps(response.json())``, i.e. a **string**. Indexing it as a
dict raised ``TypeError: string indices must be integers`` on EVERY request,
so `--backend sglang` started cleanly and then 500'd on every generation.
Older sglang returned the dict directly, so both shapes are accepted rather
than one hard assumption being swapped for another.
"""
if isinstance(response, dict):
return response
if isinstance(response, (str, bytes, bytearray)):
try:
decoded = json.loads(response)
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
raise ValueError(
"SGLang returned a response that is neither a dict nor JSON; "
"the installed sglang may have changed its Runtime.generate "
"contract again"
) from exc
if not isinstance(decoded, dict):
raise ValueError(
f"SGLang returned JSON of type {type(decoded).__name__}, expected an object"
)
return decoded
raise ValueError(
f"SGLang returned an unsupported response type {type(response).__name__}"
)
def generate_with_runtime(
runtime: Any,
prompt: str,
prompt_token_ids: Optional[list[int]],
sampling_params: dict,
lora_name: Optional[str] = None,
) -> dict:
"""Run one generation on an SGLang ``Runtime`` and return the response dict (#785).
``Runtime.generate`` only ever posts ``{"text": prompt}``, and the server
then tokenizes that string with its tokenizer's default
``add_special_tokens=True`` (``TokenizerManager._tokenize_texts``; there
is no request or server flag to turn that off). A template that rendered
``{{ bos_token }}`` therefore reached the model with two of them:
measured on a running SGLang 0.5.9 with ``unsloth/Llama-3.2-1B-Instruct``
(``bos_token_id`` 128000), the engine ran on ``[128000, 128000, ...]``
over 37 ids for the string Soup sent, against the 36 ids
``apply_chat_template(tokenize=True)`` returns; posting those 36 ids
reproduced them exactly (``[128000, 128006, ...]``, ``prompt_tokens`` 36).
So a prompt the template rendered is sent as the ids
:func:`~soup_cli.utils.vllm.build_engine_prompt` encoded for it, through
the same ``/generate`` endpoint ``Runtime.generate`` posts to, which uses
``input_ids`` verbatim (``TokenizerManager._tokenize_one_request``). A
prompt no template rendered (``prompt_token_ids`` is None) still goes
through ``Runtime.generate`` as the string it always was.
Both routes return the same ``{"text": ..., "meta_info": {...}}`` dict:
``Runtime.generate`` re-encodes the server's JSON as a string (#76), the
direct post has the dict already, and :func:`decode_sglang_response`
accepts both.
``lora_name`` is the name :func:`create_sglang_runtime` registered the
adapter under. SGLang applies a LoRA only to a request that names it in
``lora_path`` and runs any other request on the base weights, so both
routes send the name whenever there is one (#1724). A full model has none,
and its requests carry no ``lora_path`` at all.
"""
# ``lora_path`` is SGLang's field name; it takes the registered name.
selector = {} if lora_name is None else {"lora_path": lora_name}
if prompt_token_ids is None:
return decode_sglang_response(
runtime.generate(prompt, sampling_params=sampling_params, **selector)
)
try:
import httpx # the repo's HTTP client (utils/webhooks.py, commands/generate.py)
except ImportError as exc:
raise ImportError("httpx is required to post token ids to the SGLang engine") from exc
# ``timeout`` is mandatory: this is a blocking call on the app's event loop
# (``chat_completions`` awaits it), so an unresponsive engine must be bounded
# rather than wedging the whole server. 300s mirrors the server-generation
# path in ``commands/generate.py``. ``rstrip`` guards a trailing slash on
# ``runtime.url`` from producing ``//generate``.
response = httpx.post(
runtime.url.rstrip("/") + "/generate",
json={"input_ids": prompt_token_ids, "sampling_params": sampling_params, **selector},
timeout=300.0,
)
response.raise_for_status()
return decode_sglang_response(response.json())
def resolve_sglang_finish_reason(meta_info: Any, max_tokens: Optional[int]) -> str:
"""Map SGLang ``meta_info`` to an OpenAI ``finish_reason`` (#360).
Mirrors ``vllm.resolve_finish_reason``: trust the engine's own
``finish_reason`` when it maps to one Soup emits, otherwise derive it from
the token count — a generation that spent the whole budget was truncated,
not naturally stopped. The pre-fix code hardcoded ``"stop"``, so a client
doing continue-on-length silently stopped early.
SGLang reports ``finish_reason`` as ``{"type": "stop"|"length"|"abort"}`` in
recent versions and as a bare string in older ones — both are accepted so a
dict-only read cannot silently break a previously-working sglang.
"""
info = meta_info if isinstance(meta_info, dict) else {}
reported = info.get("finish_reason")
if isinstance(reported, dict):
reported = reported.get("type")
if isinstance(reported, str) and reported in _FINISH_REASONS:
return reported
completion_tokens = info.get("completion_tokens")
try:
produced = int(completion_tokens) if completion_tokens is not None else 0
except (TypeError, ValueError):
produced = 0
if max_tokens and produced >= int(max_tokens):
return "length"
return "stop"
def check_sglang_available() -> bool:
"""Check if SGLang is installed."""
try:
import sglang # noqa: F401
return True
except ImportError:
return False
def get_sglang_version() -> str:
"""Get installed SGLang version."""
try:
import sglang
return getattr(sglang, "__version__", "unknown")
except ImportError:
return "not installed"
def create_sglang_runtime(
model_path: str,
base_model: Optional[str] = None,
is_adapter: bool = False,
tensor_parallel_size: int = 1,
mem_fraction_static: float = 0.88,
dtype: str = "auto",
trust_remote_code: bool = False,
):
"""Create an SGLang Runtime for serving.
Args:
model_path: Path to model or LoRA adapter directory.
base_model: Base model ID (required if model_path is a LoRA adapter).
is_adapter: Whether model_path is a LoRA adapter.
tensor_parallel_size: Number of GPUs for tensor parallelism.
mem_fraction_static: Fraction of GPU memory for static allocation.
dtype: Data type for model weights.
trust_remote_code: Execute the model repo's custom code on load.
Default ``False`` -- the same default-deny the vLLM path takes.
``serve.py`` resolves the v0.36.0 gate once per invocation and
passes the result here. This was previously an unconditional
``True`` at both call sites, so the SGLang backend ran a model's
custom code whether or not the user opted in.
Returns:
(runtime, runtime_model_name, lora_name) tuple. ``lora_name`` is the
name the adapter is registered under, to be sent with every request
(#1724); None for a full model.
"""
import re
import sglang as sgl
# SSRF protection: block URL-based model paths
for path_val in (model_path, base_model):
if path_val and re.match(r'^https?://', path_val):
raise ValueError(
"model_path/base_model must be a local path or HuggingFace model ID, "
"not a URL"
)
# For LoRA adapters, load the base model and register the adapter under a
# name of its own: SGLang names a bare path entry after the whole string,
# and splits one that holds ``=`` into a name and a different path.
if is_adapter and base_model:
lora_name = "adapter" # the name the vLLM backend gives its LoRARequest
runtime = sgl.Runtime(
model_path=base_model,
tp_size=tensor_parallel_size,
mem_fraction_static=mem_fraction_static,
dtype=dtype,
trust_remote_code=trust_remote_code,
lora_paths=[f"{lora_name}={model_path}"],
)
runtime_model_name = base_model
else:
lora_name = None
runtime = sgl.Runtime(
model_path=model_path,
tp_size=tensor_parallel_size,
mem_fraction_static=mem_fraction_static,
dtype=dtype,
trust_remote_code=trust_remote_code,
)
runtime_model_name = model_path
return runtime, runtime_model_name, lora_name
def create_sglang_app(
runtime,
runtime_model_name: str,
model_name: str,
max_tokens_default: int = 512,
tokenizer=None,
lora_name: Optional[str] = None,
):
"""Create a FastAPI app using SGLang runtime for inference.
Args:
runtime: SGLang Runtime instance.
runtime_model_name: Model name used by SGLang.
model_name: Display model name for API responses.
max_tokens_default: Default max tokens for generation.
tokenizer: HF tokenizer for the served model — its chat template is
applied by the shared ``build_engine_prompt`` (#360), which also
encodes the rendered prompt so the server does not add a second
BOS to it (#785). None degrades to the legacy role-prefixed
format, same as the vLLM backend.
lora_name: The name ``create_sglang_runtime`` registered the adapter
under. Every request, streaming or not, names it as ``lora_path``,
since SGLang runs a request that names no adapter on the base
weights (#1724). None for a full model.
Returns:
FastAPI application.
"""
import json
import time
import uuid
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse
from pydantic import BaseModel as PydanticBaseModel
from pydantic import Field
# THE single prompt builder shared with the transformers and vLLM backends
# (#332), so the three cannot drift again. Was a hand-rolled third copy.
from soup_cli.utils.vllm import build_engine_prompt
app = FastAPI(title="Soup Inference Server (SGLang)", version="1.0.0")
# Loopback-only CORS (parity with the transformers / vLLM servers). The
# wildcard the loopback fix never reached let any web page read this local
# server's responses.
app.add_middleware(
CORSMiddleware,
allow_origin_regex=r"^https?://(localhost|127\.0\.0\.1)(:\d+)?$",
allow_methods=["GET", "POST", "OPTIONS"],
allow_headers=["*"],
)
class ChatMessage(PydanticBaseModel):
role: str
content: str
class ChatCompletionRequest(PydanticBaseModel):
model: str = model_name
messages: list[ChatMessage]
temperature: float = Field(default=0.7, ge=0.0, le=2.0)
top_p: float = Field(default=0.9, ge=0.0, le=1.0)
max_tokens: Optional[int] = Field(default=None, ge=1, le=16384)
stream: bool = False
@app.get("/health")
def health():
return {"status": "ok", "model": model_name, "backend": "sglang"}
@app.get("/v1/models")
def list_models():
return {
"object": "list",
"data": [
{
"id": model_name,
"object": "model",
"owned_by": "soup",
}
],
}
@app.post("/v1/chat/completions")
async def chat_completions(request: ChatCompletionRequest):
max_tokens = request.max_tokens or max_tokens_default
request_id = f"chatcmpl-{uuid.uuid4().hex[:8]}"
# Apply the model's own chat template (legacy fallback when there is no
# tokenizer / template), shared with the transformers + vLLM backends.
# #785: and send the ids, not the string, whenever that template
# rendered the prompt; the string route lets the server add a second
# BOS on top of the one the template rendered (generate_with_runtime).
prompt, prompt_token_ids = build_engine_prompt(request.messages, tokenizer)
sampling_params = {
"temperature": request.temperature,
"top_p": request.top_p,
"max_new_tokens": max_tokens,
}
if request.stream:
return StreamingResponse(
_stream_sglang_response(
runtime=runtime,
prompt=prompt,
prompt_token_ids=prompt_token_ids,
sampling_params=sampling_params,
request_id=request_id,
model_name=model_name,
lora_name=lora_name,
),
media_type="text/event-stream",
)
# Non-streaming
try:
response = generate_with_runtime(
runtime, prompt, prompt_token_ids, sampling_params, lora_name
)
response_text = response["text"]
prompt_tokens = response.get("meta_info", {}).get("prompt_tokens", 0)
completion_tokens = response.get(
"meta_info", {},
).get("completion_tokens", len(response_text.split()))
return {
"id": request_id,
"object": "chat.completion",
"created": int(time.time()),
"model": model_name,
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": response_text,
},
"finish_reason": resolve_sglang_finish_reason(
response.get("meta_info"), max_tokens
),
}
],
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
}
except Exception:
logger.exception("SGLang generation error")
raise HTTPException(status_code=500, detail="Internal server error")
async def _stream_sglang_response(
runtime,
prompt: str,
prompt_token_ids: Optional[list[int]],
sampling_params: dict,
request_id: str,
model_name: str,
lora_name: Optional[str],
):
"""Stream SSE chunks from SGLang runtime."""
created = int(time.time())
try:
response = generate_with_runtime(
runtime, prompt, prompt_token_ids, sampling_params, lora_name
)
response_text = response["text"]
meta_info = response.get("meta_info")
except Exception:
logger.exception("SGLang stream error")
yield 'data: {"error": "Internal server error"}\n\n'
return
# Simulate streaming by sending word-by-word
words = response_text.split(" ")
for idx, word in enumerate(words):
chunk_text = word if idx == 0 else f" {word}"
chunk = {
"id": request_id,
"object": "chat.completion.chunk",
"created": created,
"model": model_name,
"choices": [
{
"index": 0,
"delta": {"content": chunk_text},
"finish_reason": None,
}
],
}
yield f"data: {json.dumps(chunk)}\n\n"
# Final chunk
final_chunk = {
"id": request_id,
"object": "chat.completion.chunk",
"created": created,
"model": model_name,
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": resolve_sglang_finish_reason(
meta_info, sampling_params.get("max_new_tokens")
),
}
],
}
yield f"data: {json.dumps(final_chunk)}\n\n"
yield "data: [DONE]\n\n"
return app