Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
"""GET /metrics in the Prometheus text format, with vLLM's metric names, so the dashboards and alerts written for a
vLLM server read a Strata server unchanged. Everything comes from the server's own record (Service.metrics(), the
same dict the JSON /metrics returns) plus three latency histograms observed when a request finishes.
What vLLM has no name for is the JSON's own facts under `strata:`, named after the JSON's keys (live.tok_s ->
strata:live_tok_s, totals.decode_ms -> strata:totals_decode_seconds_total, hardware.gpu_util -> strata:gpu_util), so
the two formats say the same thing and a panel can be moved from the Monitor tab to Grafana by its key."""
import threading
# vLLM's buckets for the request latencies (seconds)
BUCKETS = (0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0,
80.0, 160.0, 640.0, 2560.0)
CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
def wants_prometheus(accept: str, query: str) -> bool:
"""Prometheus asks with Accept: text/plain or application/openmetrics-text; a person can add ?format=prometheus.
Anything else (the Monitor tab, curl without a header) keeps the JSON."""
return "format=prometheus" in (query or "") or "openmetrics" in (accept or "") or "text/plain" in (accept or "")
class Histogram:
def __init__(self):
self.counts = [0] * len(BUCKETS)
self.sum = 0.0
self.n = 0
def observe(self, v: float, times: int = 1):
if v is None or times <= 0:
return
for i, b in enumerate(BUCKETS):
if v <= b:
self.counts[i] += times
self.sum += v * times
self.n += times
class Latencies:
"""Time to first token, the time between tokens and the whole request, per finished request."""
def __init__(self):
self.lock = threading.Lock()
self.ttft, self.itl, self.e2e = Histogram(), Histogram(), Histogram()
def observe(self, ttft_s, output_tokens, decode_s, total_s):
with self.lock:
self.ttft.observe(ttft_s)
if output_tokens and output_tokens > 1 and decode_s:
# the mean gap, once per gap: the engine reports the decode time, not each token's
self.itl.observe(decode_s / (output_tokens - 1), output_tokens - 1)
self.e2e.observe(total_s)
def snapshot(self):
with self.lock:
return {k: (list(h.counts), h.sum, h.n) for k, h in
(("ttft", self.ttft), ("itl", self.itl), ("e2e", self.e2e))}
def _num(v, default=0):
return v if isinstance(v, (int, float)) and not isinstance(v, bool) else default
def render(m: dict, lat: dict) -> str:
"""The text for one scrape. `m` is Service.metrics(), `lat` Latencies.snapshot()."""
model = str((m.get("engine") or {}).get("model") or "strata").replace("\\", "\\\\").replace('"', '\\"')
lab = f'model_name="{model}"'
live, totals = m.get("live") or {}, m.get("totals") or {}
out = []
def metric(name, kind, help_, value):
out.append(f"# HELP {name} {help_}\n# TYPE {name} {kind}\n{name}{{{lab}}} {value}")
busy = live.get("state") in ("reading", "generating")
ctx = _num((m.get("engine") or {}).get("max_context"), 0)
used = _num(live.get("prompt_tokens")) + _num(live.get("generated")) if busy else 0
# "parallel": live.running / live.waiting count the requests together (waiting: for a slot or the control lines)
metric("vllm:num_requests_running", "gauge", "Requests reading their prompt or generating.",
_num(live.get("running"), int(busy)))
metric("vllm:num_requests_waiting", "gauge", "Requests waiting for their turn.",
_num(live.get("queued")) + _num(live.get("waiting")))
metric("vllm:kv_cache_usage_perc", "gauge", "The running request's share of the context (1 = full).",
round(min(1.0, used / ctx), 4) if ctx else 0)
metric("vllm:prompt_tokens_total", "counter", "Prompt tokens of the finished requests.",
_num(totals.get("prompt_tokens")))
metric("vllm:generation_tokens_total", "counter", "Generated tokens.", _num(totals.get("output_tokens")))
metric("vllm:request_success_total", "counter", "Finished requests.", _num(totals.get("requests")))
metric("vllm:prefix_cache_queries_total", "counter", "Prompt tokens looked up in the prompt cache.",
_num(totals.get("prompt_tokens")))
metric("vllm:prefix_cache_hits_total", "counter", "Prompt tokens the prompt cache already held.",
_num(totals.get("reused")))
metric("vllm:spec_decode_num_draft_tokens_total", "counter", "MTP draft tokens offered.",
_num(totals.get("drafts_offered")))
metric("vllm:spec_decode_num_accepted_tokens_total", "counter", "MTP draft tokens accepted.",
_num(totals.get("drafts_accepted")))
metric("vllm:num_preemptions_total", "counter", "Preempted requests (Strata does not preempt).", 0)
for name, key, help_ in (("vllm:time_to_first_token_seconds", "ttft", "Time to the first token."),
("vllm:inter_token_latency_seconds", "itl", "Time between two tokens."),
("vllm:e2e_request_latency_seconds", "e2e", "A request from start to end.")):
counts, total, n = lat[key]
out.append(f"# HELP {name} {help_}\n# TYPE {name} histogram")
for b, c in zip(BUCKETS, counts):
out.append(f'{name}_bucket{{{lab},le="{b}"}} {c}')
out.append(f'{name}_bucket{{{lab},le="+Inf"}} {n}')
out.append(f"{name}_sum{{{lab}}} {round(total, 6)}\n{name}_count{{{lab}}} {n}")
strata(m, out, lab)
return "\n".join(out) + "\n"
def strata(m: dict, out: list, lab: str):
"""The JSON's facts vLLM has no name for, under `strata:` and the JSON's key."""
live, totals = m.get("live") or {}, m.get("totals") or {}
hw, eng = m.get("hardware") or {}, m.get("engine") or {}
reqs = m.get("requests") or []
typed = set()
def metric(name, kind, help_, value, labels=""):
if not isinstance(value, (int, float)) or isinstance(value, bool):
return # unknown here (no GPU telemetry, an older engine): no sample
if name not in typed: # one HELP / TYPE per family, before its first sample
typed.add(name)
out.append(f"# HELP strata:{name} {help_}\n# TYPE strata:{name} {kind}")
out.append(f"strata:{name}{{{lab}{labels}}} {value}")
for state in ("unloaded", "idle", "reading", "generating"):
metric("live_state", "gauge", "1 for what the engine is doing (live.state).", int(live.get("state") == state),
f',state="{state}"')
metric("live_tok_s", "gauge", "Decode rate over the last seconds (live.tok_s).", live.get("tok_s") or 0)
metric("live_prefill_tok_s_mean", "gauge", "Prompt reading rate of the running request (live.prefill_tok_s_mean).",
live.get("prefill_tok_s_mean") or 0)
metric("live_prompt_read", "gauge", "Prompt tokens read so far by the running request (live.prompt_read).",
live.get("prompt_read") or 0)
for state in ("idle", "reading", "generating"):
slots = live.get("slots")
if isinstance(slots, list) and slots:
metric("live_slots", "gauge", "The batch slots in each state (live.slots).",
sum(1 for x in slots if isinstance(x, dict) and x.get("state") == state), f',state="{state}"')
metric("engine_max_context", "gauge", "The engine's context (engine.max_context).", eng.get("max_context"))
metric("totals_prompt_seconds_total", "counter", "Time spent reading prompts (totals.prompt_ms).",
round((totals.get("prompt_ms") or 0) / 1000, 3))
metric("totals_decode_seconds_total", "counter", "Time spent generating (totals.decode_ms).",
round((totals.get("decode_ms") or 0) / 1000, 3))
last = reqs[0] if reqs else {}
metric("last_hit_rate", "gauge", "The last request's expert cache hit rate (requests[0].hit_rate).",
last.get("hit_rate"))
metric("last_decode_tok_s", "gauge", "The last request's decode rate (requests[0].decode_tok_s).",
last.get("decode_tok_s"))
gpus = hw.get("gpus") or ([{"index": 0, "util": hw.get("gpu_util"), "mem_used": hw.get("gpu_mem_used"),
"mem_total": hw.get("gpu_mem_total"), "temp": hw.get("gpu_temp"),
"power": hw.get("gpu_power")}] if hw.get("gpu_util") is not None else [])
for name, key, help_ in (("gpu_util", "util", "GPU busy, percent (hardware.gpu_util)."),
("gpu_mem_used_bytes", "mem_used", "GPU memory in use (hardware.gpu_mem_used)."),
("gpu_mem_total_bytes", "mem_total", "GPU memory (hardware.gpu_mem_total)."),
("gpu_temp_celsius", "temp", "GPU temperature (hardware.gpu_temp)."),
("gpu_power_watts", "power", "GPU power draw (hardware.gpu_power).")):
for g in gpus: # a family's samples together, one per card
metric(name, "gauge", help_, g.get(key), f',gpu="{g.get("index")}"')
metric("cpu", "gauge", "CPU busy, percent (hardware.cpu).", hw.get("cpu"))
metric("ram_used_bytes", "gauge", "RAM in use (hardware.ram_used).", hw.get("ram_used"))
metric("ram_total_bytes", "gauge", "RAM (hardware.ram_total).", hw.get("ram_total"))