Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
{
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"title": "Strata (and vLLM)",
"uid": "strata-serving",
"tags": [
"strata",
"vllm",
"llm"
],
"timezone": "",
"schemaVersion": 39,
"version": 1,
"refresh": "30s",
"time": {
"from": "now-6h",
"to": "now"
},
"editable": true,
"templating": {
"list": [
{
"name": "model",
"label": "Model",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"query": {
"query": "label_values(vllm:generation_tokens_total, model_name)",
"refId": "model"
},
"definition": "label_values(vllm:generation_tokens_total, model_name)",
"sort": 1
},
{
"name": "instance",
"label": "Instance",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"query": {
"query": "label_values(vllm:generation_tokens_total{model_name=~\"$model\"}, instance)",
"refId": "instance"
},
"definition": "label_values(vllm:generation_tokens_total{model_name=~\"$model\"}, instance)",
"sort": 1
}
]
},
"panels": [
{
"id": 1,
"type": "row",
"title": "Requests and throughput (vLLM names: the same panels read a vLLM server)",
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"panels": []
},
{
"id": 2,
"type": "timeseries",
"title": "Requests running / waiting",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 1
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (model_name) (vllm:num_requests_running{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "{{model_name}} running"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (model_name) (vllm:num_requests_waiting{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "{{model_name}} waiting"
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 3,
"type": "timeseries",
"title": "Tokens per second",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 1
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (model_name) (rate(vllm:generation_tokens_total{model_name=~\"$model\",instance=~\"$instance\"}[1m]))",
"legendFormat": "{{model_name}} generated"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (model_name) (rate(vllm:prompt_tokens_total{model_name=~\"$model\",instance=~\"$instance\"}[1m]))",
"legendFormat": "{{model_name}} prompt"
}
],
"fieldConfig": {
"defaults": {
"unit": "suffix: tok/s",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 4,
"type": "timeseries",
"title": "Time to first token (p50 / p95)",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum by (le, model_name) (rate(vllm:time_to_first_token_seconds_bucket{model_name=~\"$model\",instance=~\"$instance\"}[5m])))",
"legendFormat": "{{model_name}} p95"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.5, sum by (le, model_name) (rate(vllm:time_to_first_token_seconds_bucket{model_name=~\"$model\",instance=~\"$instance\"}[5m])))",
"legendFormat": "{{model_name}} p50"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 5,
"type": "timeseries",
"title": "Time between tokens (p50 / p95)",
"description": "Strata observes the mean gap of each request (the engine reports its decode time, not each token's).",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum by (le, model_name) (rate(vllm:inter_token_latency_seconds_bucket{model_name=~\"$model\",instance=~\"$instance\"}[5m])))",
"legendFormat": "{{model_name}} p95"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.5, sum by (le, model_name) (rate(vllm:inter_token_latency_seconds_bucket{model_name=~\"$model\",instance=~\"$instance\"}[5m])))",
"legendFormat": "{{model_name}} p50"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 6,
"type": "timeseries",
"title": "Request duration (p50 / p95)",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 15
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum by (le, model_name) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=~\"$model\",instance=~\"$instance\"}[5m])))",
"legendFormat": "{{model_name}} p95"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.5, sum by (le, model_name) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=~\"$model\",instance=~\"$instance\"}[5m])))",
"legendFormat": "{{model_name}} p50"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 7,
"type": "timeseries",
"title": "Speed of one answer",
"description": "1 / the mean time between tokens: what one user sees, against the total above.",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 15
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (model_name) (rate(vllm:inter_token_latency_seconds_count{model_name=~\"$model\",instance=~\"$instance\"}[5m])) / sum by (model_name) (rate(vllm:inter_token_latency_seconds_sum{model_name=~\"$model\",instance=~\"$instance\"}[5m]))",
"legendFormat": "{{model_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "suffix: tok/s",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 8,
"type": "timeseries",
"title": "Prompt tokens served from the cache",
"description": "Strata: the prompt cache and parked conversations; vLLM: its prefix-cache blocks.",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 22
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (model_name) (rate(vllm:prefix_cache_hits_total{model_name=~\"$model\",instance=~\"$instance\"}[15m])) / sum by (model_name) (rate(vllm:prefix_cache_queries_total{model_name=~\"$model\",instance=~\"$instance\"}[15m]))",
"legendFormat": "{{model_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 9,
"type": "timeseries",
"title": "MTP drafts accepted",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 22
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (model_name) (rate(vllm:spec_decode_num_accepted_tokens_total{model_name=~\"$model\",instance=~\"$instance\"}[15m])) / sum by (model_name) (rate(vllm:spec_decode_num_draft_tokens_total{model_name=~\"$model\",instance=~\"$instance\"}[15m]))",
"legendFormat": "{{model_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 10,
"type": "timeseries",
"title": "Requests finished per minute",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 29
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (model_name) (rate(vllm:request_success_total{model_name=~\"$model\",instance=~\"$instance\"}[5m])) * 60",
"legendFormat": "{{model_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "suffix: req/min",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 11,
"type": "timeseries",
"title": "Context in use",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 29
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max by (model_name) (vllm:kv_cache_usage_perc{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "{{model_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 12,
"type": "row",
"title": "Strata engine (strata: names, after the /metrics JSON keys)",
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 36
},
"panels": []
},
{
"id": 13,
"type": "timeseries",
"title": "Engine state",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 37
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max by (state) (strata:live_state{model_name=~\"$model\",instance=~\"$instance\"}) > 0",
"legendFormat": "{{state}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"custom": {
"fillOpacity": 60,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 14,
"type": "timeseries",
"title": "Batch slots by state",
"description": "\"parallel\": N / --batch only.",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 37
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (state) (strata:live_slots{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "{{state}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"custom": {
"fillOpacity": 60,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 15,
"type": "timeseries",
"title": "Engine rates",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 44
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max(strata:live_tok_s{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "decode tok/s"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max(strata:live_prefill_tok_s_mean{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "prompt tok/s"
}
],
"fieldConfig": {
"defaults": {
"unit": "suffix: tok/s",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 16,
"type": "timeseries",
"title": "Expert cache hit rate (last request)",
"description": "Experts found in VRAM. Below ~90 % the CPU and the PCIe link compute the rest and decode slows down.",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 44
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max(strata:last_hit_rate{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "hit rate"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 17,
"type": "timeseries",
"title": "Engine time: prompts / decoding",
"description": "Seconds of work per second: 1 = busy all the time (more with requests running together).",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 51
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(strata:totals_prompt_seconds_total{model_name=~\"$model\",instance=~\"$instance\"}[5m]))",
"legendFormat": "reading prompts"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(strata:totals_decode_seconds_total{model_name=~\"$model\",instance=~\"$instance\"}[5m]))",
"legendFormat": "decoding"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": {
"fillOpacity": 60,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 18,
"type": "timeseries",
"title": "GPU busy, per card",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 51
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max by (gpu) (strata:gpu_util{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "GPU {{gpu}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 19,
"type": "timeseries",
"title": "GPU memory, per card",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 58
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max by (gpu) (strata:gpu_mem_used_bytes{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "GPU {{gpu}} used"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max by (gpu) (strata:gpu_mem_total_bytes{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "GPU {{gpu}} total"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 20,
"type": "timeseries",
"title": "Host RAM",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 58
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max(strata:ram_used_bytes{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "used"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max(strata:ram_total_bytes{model_name=~\"$model\",instance=~\"$instance\"})",
"legendFormat": "total"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"custom": {
"fillOpacity": 10,
"showPoints": "never"
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
}
]
}