Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""vLLM backend utilities for soup serve."""
import logging
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Optional
if TYPE_CHECKING: # pragma: no cover
from collections.abc import Generator
logger = logging.getLogger(__name__)
# OpenAI's finish_reason vocabulary that Soup emits. Anything the engine
# reports outside this set (vLLM also uses "abort") is normalised to "stop"
# rather than leaked to clients that branch on the value.
_FINISH_REASONS = frozenset({"stop", "length"})
def _normalise_messages(messages) -> list:
"""Accept pydantic ``ChatMessage`` objects OR plain dicts.
The vLLM app builds pydantic models; the transformers app passes dicts.
Both must render to the same prompt, so both funnel through here.
Dict messages are copied through WHOLE — a chat template may read keys
beyond role/content (``tool_calls``, ``name``, multimodal content parts),
and dropping them would silently change what the transformers backend
renders today.
"""
rows = []
for msg in messages or []:
if isinstance(msg, Mapping):
rows.append(dict(msg))
else:
content = getattr(msg, "content", "")
rows.append(
{
"role": str(getattr(msg, "role", "")),
"content": "" if content is None else content,
}
)
return rows
def _legacy_chat_prompt(messages) -> str:
"""The pre-#332 hand-rolled prompt.
Kept ONLY as the fallback for a model that ships no chat template — which
is exactly what the transformers backend has always done. Every other path
must render the model's own template; feeding this to a chat-tuned model is
what produced the Llama-3.1-8B run-on loop in #332.
"""
parts = []
for msg in _normalise_messages(messages):
role = msg.get("role", "")
content = msg.get("content", "")
if role == "system":
parts.append(f"System: {content}")
elif role == "user":
parts.append(f"User: {content}")
elif role == "assistant":
parts.append(f"Assistant: {content}")
parts.append("Assistant:")
return "\n".join(parts)
def _render_chat_prompt(
messages: Any, tokenizer: Any, *, fallback_on_error: bool
) -> tuple[str, bool]:
"""Render ``messages``; also report whether the model's template produced the text.
Only template-rendered text already carries the model's special tokens, which is
what :func:`encode_rendered_prompt` needs to know.
"""
template = getattr(tokenizer, "chat_template", None)
if tokenizer is not None and template and hasattr(tokenizer, "apply_chat_template"):
try:
text = tokenizer.apply_chat_template(
_normalise_messages(messages),
tokenize=False,
add_generation_prompt=True,
)
except Exception: # noqa: BLE001 — a broken template must not 500
if not fallback_on_error:
raise
logger.warning(
"chat template failed to render; falling back to the legacy "
"role-prefixed prompt",
exc_info=True,
)
else:
return text, True
return _legacy_chat_prompt(messages), False
def build_chat_prompt(messages, tokenizer=None) -> str:
"""Render chat messages into a prompt string (#332).
THE single prompt builder shared by ``soup serve``'s transformers and vLLM
backends, so the two cannot drift again. Applies the model's own chat
template when it has one, and falls back to the legacy role-prefixed
format only when it does not (or when no tokenizer could be loaded).
In-process callers must not hand the result to a bare ``tokenizer(...)``
call; use :func:`encode_chat_prompt` (#781). Backends whose engine would
tokenize this string itself use :func:`build_engine_prompt` (#785).
Args:
messages: chat messages — pydantic objects or dicts.
tokenizer: a HF tokenizer, or None when one could not be loaded.
Returns:
The prompt string to hand to the engine.
"""
return _render_chat_prompt(messages, tokenizer, fallback_on_error=True)[0]
def encode_rendered_prompt(
tokenizer: Any, text: str, *, templated: bool, **tokenizer_kwargs: Any
) -> Any:
"""Tokenize a prompt without re-adding the special tokens its template rendered (#781).
A chat template already emits every special token the model expects —
Llama-3, Gemma and Mistral render ``{{ bos_token }}`` — so letting the
tokenizer add its own on top sent ``[bos, bos, ...]``. Soup's training ids
are built with ``add_special_tokens=False`` (``data/loss_mask.py``), as are
HF's own ``apply_chat_template(tokenize=True)`` ids, and inference has to
match them. Text no template rendered is tokenized exactly as before.
"""
if templated:
return tokenizer(text, add_special_tokens=False, **tokenizer_kwargs)
return tokenizer(text, **tokenizer_kwargs)
def encode_chat_prompt(
messages: Any, tokenizer: Any, *, fallback_on_error: bool, **tokenizer_kwargs: Any
) -> Any:
"""Render ``messages`` with the model's template and tokenize them as Soup trains.
``fallback_on_error`` keeps each caller's existing contract: ``soup serve``
serves the legacy prompt when a template fails to render, while the CLI
commands surface the template's own error.
"""
text, templated = _render_chat_prompt(
messages, tokenizer, fallback_on_error=fallback_on_error
)
return encode_rendered_prompt(tokenizer, text, templated=templated, **tokenizer_kwargs)
def build_engine_prompt(
messages: Any, tokenizer: Any = None
) -> tuple[str, Optional[list[int]]]:
"""Render chat messages for a backend that tokenizes the prompt itself (#785).
The vLLM, SGLang and MII backends hand the engine a prompt STRING, and the
engine encodes it with its tokenizer's default ``add_special_tokens=True``.
A template that renders ``{{ bos_token }}`` therefore reached the model with
two of them. Measured on vLLM 0.29.0 + ``unsloth/Llama-3.2-1B-Instruct``
(``bos_token_id`` 128000): for the string Soup sends today the engine
reported ``prompt_token_ids[:2] == [128000, 128000]`` over 37 ids, against
the 36 ids ``apply_chat_template(tokenize=True)`` returns. Handing the
engine these ids instead reproduced that 36-id encoding exactly.
#782 fixed the same defect for every path that tokenizes in process; this is
the same rule for the paths that do not, so the chat template stays the
single source of the model's special tokens.
Returns:
``(text, token_ids)``. ``token_ids`` is what to send the engine in place
of ``text``, and is None when no template rendered the prompt: the
legacy role-prefixed fallback carries no special tokens of its own, so
the engine must keep adding its own exactly as it always has.
"""
text, templated = _render_chat_prompt(messages, tokenizer, fallback_on_error=True)
if not templated:
return text, None
encoded = encode_rendered_prompt(tokenizer, text, templated=True)
return text, list(encoded["input_ids"])
def resolve_finish_reason(output: Any, max_tokens: Optional[int]) -> str:
"""Map a vLLM ``CompletionOutput`` to an OpenAI ``finish_reason`` (#333).
Trusts the engine's own ``finish_reason`` when it is one Soup emits, and
otherwise derives it from the token count — an output that used the whole
budget was truncated, not naturally stopped. The pre-fix code hardcoded
``"stop"``, so a client doing continue-on-length silently stopped early.
"""
reported = getattr(output, "finish_reason", None)
if isinstance(reported, str):
if reported in _FINISH_REASONS:
return reported
if reported:
# e.g. vLLM's "abort" — not an OpenAI value; fall through to the
# token-count derivation rather than leaking it.
logger.debug("unmapped vLLM finish_reason %r", reported)
token_ids = getattr(output, "token_ids", None) or []
try:
produced = len(token_ids)
except TypeError: # pragma: no cover — defensive
produced = 0
if max_tokens and produced >= int(max_tokens):
return "length"
return "stop"
def is_vllm_available() -> bool:
"""Check if vLLM is installed."""
try:
import vllm # noqa: F401
return True
except ImportError:
return False
def get_vllm_version() -> str:
"""Get installed vLLM version."""
try:
import vllm
return getattr(vllm, "__version__", "unknown")
except ImportError:
return "not installed"
def create_vllm_engine(
model_path: str,
base_model: Optional[str] = None,
is_adapter: bool = False,
tensor_parallel_size: int = 1,
gpu_memory_utilization: float = 0.9,
max_model_len: Optional[int] = None,
dtype: str = "auto",
speculative_model: Optional[str] = None,
num_speculative_tokens: int = 5,
enable_prefix_caching: bool = False,
quantization: Optional[str] = None,
sleep_mode: bool = False,
trust_remote_code: bool = False,
):
"""Create a vLLM AsyncLLMEngine for serving.
Args:
model_path: Path to model or LoRA adapter directory.
base_model: Base model ID (required if model_path is a LoRA adapter).
is_adapter: Whether model_path is a LoRA adapter.
tensor_parallel_size: Number of GPUs for tensor parallelism.
gpu_memory_utilization: Fraction of GPU memory to use.
max_model_len: Maximum sequence length. Auto-detected if None.
dtype: Data type for model weights.
enable_prefix_caching: Enable vLLM's automatic prefix cache — big
win for RAG / agent workloads with shared system prompts.
sleep_mode: Enable vLLM sleep/standby support (v0.71.21 #124 —
requires vLLM >= 0.7; raises a friendly RuntimeError otherwise).
trust_remote_code: Execute custom repo code on model load. Default
False — the caller (``serve``) resolves this through the shared
v0.36.0 ``resolve_trust_remote_code`` gate (flag + warning panel)
so an untrusted HF repo never runs code silently.
Returns:
(engine, engine_model_name) tuple.
"""
from vllm import AsyncEngineArgs, AsyncLLMEngine
# For LoRA adapters, load the base model and apply LoRA at request time
if is_adapter and base_model:
engine_args = AsyncEngineArgs(
model=base_model,
enable_lora=True,
max_lora_rank=128,
tensor_parallel_size=tensor_parallel_size,
gpu_memory_utilization=gpu_memory_utilization,
dtype=dtype,
trust_remote_code=trust_remote_code,
enable_prefix_caching=enable_prefix_caching,
)
if max_model_len is not None:
engine_args.max_model_len = max_model_len
engine_model_name = base_model
else:
engine_args = AsyncEngineArgs(
model=model_path,
tensor_parallel_size=tensor_parallel_size,
gpu_memory_utilization=gpu_memory_utilization,
dtype=dtype,
trust_remote_code=trust_remote_code,
enable_prefix_caching=enable_prefix_caching,
)
if max_model_len is not None:
engine_args.max_model_len = max_model_len
engine_model_name = model_path
# vLLM accepts ``quantization`` on AsyncEngineArgs (string: awq/gptq/fp8).
# ``None`` is the default (baseline / model's native dtype).
if quantization:
if quantization not in ("awq", "gptq", "fp8"):
raise ValueError(
f"quantization must be one of awq/gptq/fp8 or None, "
f"got {quantization!r}"
)
engine_args.quantization = quantization
# Speculative decoding — use a smaller draft model for faster inference
if speculative_model:
import re
# SSRF protection: block URL-based model paths
if re.match(r'^https?://', speculative_model):
raise ValueError(
"speculative_model must be a local path or HuggingFace model ID, "
"not a URL"
)
engine_args.speculative_model = speculative_model
engine_args.num_speculative_tokens = num_speculative_tokens
# v0.71.21 #124 — vLLM sleep/standby support (engine.sleep()/wake_up()).
# apply_vllm_sleep_mode raises a friendly RuntimeError on vLLM < 0.7.
if sleep_mode:
from soup_cli.utils.grpo_long_context import apply_vllm_sleep_mode
apply_vllm_sleep_mode(engine_args)
engine = AsyncLLMEngine.from_engine_args(engine_args)
return engine, engine_model_name
def create_vllm_app(
engine,
engine_model_name: str,
model_name: str,
adapter_path: Optional[str] = None,
max_tokens_default: int = 512,
tokenizer=None,
enable_dashboard: bool = False,
):
"""Create a FastAPI app using vLLM engine for inference.
Args:
engine: vLLM AsyncLLMEngine instance.
engine_model_name: Model name used by vLLM engine.
model_name: Display model name for API responses.
adapter_path: Path to LoRA adapter (if using adapter).
max_tokens_default: Default max tokens for generation.
tokenizer: HF tokenizer for the served model — its chat template is
what turns messages into the prompt (#332). ``None`` degrades to
the legacy role-prefixed format.
enable_dashboard: Recorded on ``app.state`` for introspection.
``/metrics`` is served either way, mirroring the transformers
backend (#333 — the flag used to no-op entirely here).
Returns:
FastAPI application.
"""
import contextlib
import json
import time
import uuid
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse
from pydantic import BaseModel as PydanticBaseModel
from pydantic import Field
from vllm import SamplingParams
from soup_cli.utils.metrics import ServerMetrics
app = FastAPI(title="Soup Inference Server (vLLM)", version="1.0.0")
# #333 — shared metrics bucket. Created unconditionally so /metrics works
# whether or not --dashboard is set, exactly like the transformers app.
metrics = ServerMetrics()
# v0.53.7 M-G: restrict CORS to loopback — /v1/messages is a mutation
# endpoint and was previously reachable from any browser page via the
# legacy ``allow_origins=["*"]``. Matches v0.30.0 transformers-backend
# policy.
app.add_middleware(
CORSMiddleware,
allow_origin_regex=r"^https?://(localhost|127\.0\.0\.1)(:\d+)?$",
allow_methods=["GET", "POST", "OPTIONS"],
allow_headers=["*"],
)
class ChatMessage(PydanticBaseModel):
role: str
content: str
class ChatCompletionRequest(PydanticBaseModel):
model: str = model_name
messages: list[ChatMessage]
temperature: float = Field(default=0.7, ge=0.0, le=2.0)
top_p: float = Field(default=0.9, ge=0.0, le=1.0)
max_tokens: Optional[int] = Field(default=None, ge=1, le=16384)
stream: bool = False
@app.get("/health")
def health():
return {"status": "ok", "model": model_name, "backend": "vllm"}
@app.get("/metrics")
def metrics_endpoint():
"""Dashboard + Prometheus-style JSON scrape (#333)."""
return metrics.snapshot()
@app.get("/v1/models")
def list_models():
return {
"object": "list",
"data": [
{
"id": model_name,
"object": "model",
"owned_by": "soup",
}
],
}
@app.post("/v1/chat/completions")
async def chat_completions(request: ChatCompletionRequest):
max_tokens = request.max_tokens or max_tokens_default
request_id = f"chatcmpl-{uuid.uuid4().hex[:8]}"
# #332 — the model's OWN chat template, shared with the transformers
# backend. The pre-fix hand-rolled prompt made chat-tuned models loop.
# #785: and send the ids, not the string, whenever that template
# rendered the prompt: vLLM tokenizes a string prompt with its own
# add_special_tokens=True, which put a second BOS in front of the one
# the template had already rendered. A TokensPrompt is a plain
# ``{"prompt_token_ids": [...]}`` mapping, so no vLLM import is needed.
prompt, prompt_token_ids = build_engine_prompt(request.messages, tokenizer)
engine_prompt: Any = (
prompt if prompt_token_ids is None else {"prompt_token_ids": prompt_token_ids}
)
sampling_params = SamplingParams(
temperature=request.temperature,
top_p=request.top_p,
max_tokens=max_tokens,
)
# Build generate kwargs
generate_kwargs = {}
if adapter_path:
from vllm.lora.request import LoRARequest
generate_kwargs["lora_request"] = LoRARequest(
lora_name="adapter",
lora_int_id=1,
lora_path=adapter_path,
)
if request.stream:
return StreamingResponse(
_stream_vllm_response(
engine=engine,
prompt=engine_prompt,
sampling_params=sampling_params,
request_id=request_id,
model_name=model_name,
generate_kwargs=generate_kwargs,
max_tokens=max_tokens,
),
media_type="text/event-stream",
)
# Non-streaming
started = time.perf_counter()
with contextlib.ExitStack() as stack:
stack.enter_context(metrics.track_request())
try:
results_generator = engine.generate(
engine_prompt, sampling_params, request_id, **generate_kwargs
)
final_output = None
async for request_output in results_generator:
final_output = request_output
if final_output is None:
raise HTTPException(status_code=500, detail="No output generated")
output = final_output.outputs[0]
response_text = output.text
prompt_tokens = len(final_output.prompt_token_ids)
completion_tokens = len(output.token_ids)
metrics.record_tokens(completion_tokens)
return {
"id": request_id,
"object": "chat.completion",
"created": int(time.time()),
"model": model_name,
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": response_text,
},
# #333 — was hardcoded "stop", so a length
# truncation looked like a natural stop.
"finish_reason": resolve_finish_reason(
output, max_tokens
),
}
],
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
}
except HTTPException:
raise
except Exception:
logger.exception("vLLM generation error")
raise HTTPException(status_code=500, detail="Internal server error")
finally:
# Record latency on error paths too, or the dashboard shows a
# blind spot exactly when things are going wrong.
metrics.record_latency((time.perf_counter() - started) * 1000)
async def _stream_vllm_response(
engine,
prompt: Any,
sampling_params,
request_id: str,
model_name: str,
generate_kwargs: dict,
max_tokens: Optional[int] = None,
):
"""Stream SSE chunks from vLLM engine."""
created = int(time.time())
previous_text = ""
last_output = None
started = time.perf_counter()
with metrics.track_request():
try:
results_generator = engine.generate(
prompt, sampling_params, request_id, **generate_kwargs
)
async for request_output in results_generator:
output = request_output.outputs[0]
last_output = output
new_text = output.text[len(previous_text):]
previous_text = output.text
if new_text:
chunk = {
"id": request_id,
"object": "chat.completion.chunk",
"created": created,
"model": model_name,
"choices": [
{
"index": 0,
"delta": {"content": new_text},
"finish_reason": None,
}
],
}
yield f"data: {json.dumps(chunk)}\n\n"
if last_output is not None:
metrics.record_tokens(len(getattr(last_output, "token_ids", ()) or ()))
finally:
metrics.record_latency((time.perf_counter() - started) * 1000)
# Final chunk — #333: report the real reason, not a hardcoded "stop".
final_chunk = {
"id": request_id,
"object": "chat.completion.chunk",
"created": created,
"model": model_name,
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": resolve_finish_reason(last_output, max_tokens),
}
],
}
yield f"data: {json.dumps(final_chunk)}\n\n"
yield "data: [DONE]\n\n"
# ----- v0.53.7 #102 — Anthropic /v1/messages on vLLM backend -----
# Mirrors the v0.53.6 transformers-backend route. Reuses the v0.45.0
# Part B converter + the existing chat_completions handler; streaming
# emits the Anthropic event shape.
@app.post("/v1/messages")
async def anthropic_messages(payload: dict):
from soup_cli.utils.anthropic_messages import (
from_anthropic,
validate_anthropic_payload,
)
wants_stream = isinstance(payload, dict) and bool(payload.get("stream"))
try:
validate_anthropic_payload(payload)
openai_payload = from_anthropic(payload)
openai_payload.pop("stream", None)
req = ChatCompletionRequest(**openai_payload)
except (TypeError, ValueError) as exc:
logger.debug("vLLM /v1/messages invalid request: %s", exc)
raise HTTPException(status_code=400, detail="Invalid request")
except Exception as exc: # noqa: BLE001
logger.debug("vLLM /v1/messages pydantic error: %s", exc)
raise HTTPException(status_code=400, detail="Invalid request")
chat_response = await chat_completions(req)
text = ""
if isinstance(chat_response, dict):
try:
text = chat_response["choices"][0]["message"]["content"]
except (KeyError, IndexError, TypeError):
text = ""
usage = (
chat_response.get("usage", {}) if isinstance(chat_response, dict) else {}
)
msg_id = (
chat_response.get("id", "") if isinstance(chat_response, dict) else ""
)
out_model = openai_payload.get("model", model_name)
in_tokens = int(usage.get("prompt_tokens", 0) or 0)
out_tokens = int(usage.get("completion_tokens", 0) or 0)
# #333 — carry the real finish reason across to Anthropic's vocabulary
# instead of always claiming a natural end_turn.
stop_reason = _anthropic_stop_reason(chat_response)
if wants_stream:
# v0.53.7 L-C: disable intermediate caching on SSE streams.
return StreamingResponse(
_stream_anthropic_messages_vllm(
msg_id=msg_id,
model=out_model,
text=text,
input_tokens=in_tokens,
output_tokens=out_tokens,
stop_reason=stop_reason,
),
media_type="text/event-stream",
headers={
"Cache-Control": "no-store",
"X-Accel-Buffering": "no",
},
)
return {
"id": msg_id,
"type": "message",
"role": "assistant",
"model": out_model,
"content": [{"type": "text", "text": text}],
"stop_reason": stop_reason,
"usage": {
"input_tokens": in_tokens,
"output_tokens": out_tokens,
},
}
# Introspection parity with the transformers app.
app.state.enable_dashboard = enable_dashboard
app.state.has_chat_template = bool(getattr(tokenizer, "chat_template", None))
return app
def _anthropic_stop_reason(chat_response) -> str:
"""Map an OpenAI ``finish_reason`` to Anthropic's ``stop_reason``."""
reason = None
if isinstance(chat_response, Mapping):
try:
reason = chat_response["choices"][0]["finish_reason"]
except (KeyError, IndexError, TypeError):
reason = None
return "max_tokens" if reason == "length" else "end_turn"
def _stream_anthropic_messages_vllm(
*,
msg_id: str,
model: str,
text: str,
input_tokens: int,
output_tokens: int,
stop_reason: str = "end_turn",
) -> "Generator[str, None, None]":
"""Yield Anthropic event-shape SSE frames (vLLM backend)."""
import json as _json
# M-A: sanitise caller-influenced fields before SSE embedding.
def _sanitise(value: str, max_len: int) -> str:
if not isinstance(value, str):
return ""
return value.replace("\r", "").replace("\n", "").replace("\x00", "")[:max_len]
msg_id = _sanitise(msg_id, 64)
model = _sanitise(model, 200)
def _frame(event_type: str, data: dict) -> str:
return f"event: {event_type}\ndata: {_json.dumps(data)}\n\n"
yield _frame(
"message_start",
{
"type": "message_start",
"message": {
"id": msg_id,
"type": "message",
"role": "assistant",
"content": [],
"model": model,
"stop_reason": None,
"usage": {
"input_tokens": int(input_tokens),
"output_tokens": 0,
},
},
},
)
words = (text or "").split(" ")
for idx, word in enumerate(words):
chunk_text = word if idx == 0 else f" {word}"
yield _frame(
"content_block_delta",
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": chunk_text},
},
)
yield _frame(
"message_delta",
{
"type": "message_delta",
"delta": {"stop_reason": stop_reason},
"usage": {"output_tokens": int(output_tokens)},
},
)
yield _frame("message_stop", {"type": "message_stop"})