Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(serve): expose Prometheus scheduler metrics

ZacharyZcR90ff72bb773e
diff --git a/c/openai_server.py b/c/openai_server.py
index d3de8dea..57888ac7 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -110,6 +110,8 @@ def _engine_error(fields, message):
class GenerationScheduler:
"""Bounded FIFO admission for the engine's independent KV contexts."""
+ _buckets = (0.001, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 300, math.inf)
+
def __init__(self, max_queue=8, queue_timeout=300, capacity=1):
if max_queue < 0:
raise ValueError("max_queue cannot be negative")
@@ -127,9 +129,12 @@ class GenerationScheduler:
self.closed = False
self.admitted = 0
self.completed = 0
+ self.failed = 0
self.rejected = 0
self.timed_out = 0
self.cancelled = 0
+ self.timings = {name: {"sum": 0.0, "buckets": [0] * len(self._buckets)}
+ for name in ("queue_wait_seconds", "slot_duration_seconds")}
@contextlib.contextmanager
def admit(self, cancelled=None, slot=None):
@@ -187,21 +192,22 @@ class GenerationScheduler:
self.free_slots.remove(available)
self.active += 1
self.admitted += 1
- wait_seconds = time.monotonic() - queued_at
- cancelled_after_admission = False
+ admitted_at = time.monotonic()
+ wait_seconds = admitted_at - queued_at
+ self._observe("queue_wait_seconds", wait_seconds)
+ outcome = "failed"
try:
yield wait_seconds, available
+ outcome = "completed"
except ClientCancelled:
- cancelled_after_admission = True
+ outcome = "cancelled"
raise
finally:
with self.condition:
self.active -= 1
self.free_slots.add(available)
- if cancelled_after_admission:
- self.cancelled += 1
- else:
- self.completed += 1
+ setattr(self, outcome, getattr(self, outcome) + 1)
+ self._observe("slot_duration_seconds", time.monotonic() - admitted_at)
self.condition.notify_all()
def snapshot(self):
@@ -209,10 +215,51 @@ class GenerationScheduler:
return {"active": self.active, "queued": len(self.queue),
"capacity": self.capacity,
"max_queue": self.max_queue, "queue_timeout_seconds": self.queue_timeout,
- "admitted": self.admitted, "completed": self.completed,
+ "admitted": self.admitted, "completed": self.completed, "failed": self.failed,
"rejected": self.rejected, "timed_out": self.timed_out,
"cancelled": self.cancelled}
+ def _observe(self, name, seconds):
+ # Called with condition held. Cumulative buckets need no request history.
+ timing = self.timings[name]
+ timing["sum"] += seconds
+ for i, bound in enumerate(self._buckets):
+ if seconds <= bound:
+ timing["buckets"][i] += 1
+
+ def prometheus(self):
+ """One consistent, bounded snapshot; no prompt or request-ID labels."""
+ gauges = {"active": "Currently admitted requests.",
+ "queued": "Requests waiting for a KV slot.",
+ "capacity": "Concurrent KV slots configured.",
+ "max_queue": "Maximum waiting requests configured."}
+ counters = {"admitted": "Requests admitted to a KV slot.",
+ "completed": "Admitted requests that returned normally.",
+ "failed": "Admitted requests that raised an error.",
+ "rejected": "Requests rejected because the queue was full.",
+ "timed_out": "Requests that timed out waiting for a slot.",
+ "cancelled": "Requests cancelled while queued or admitted."}
+ lines = []
+ with self.condition:
+ for kind, fields in (("gauge", gauges), ("counter", counters)):
+ for field, help_text in fields.items():
+ name = "colibri_scheduler_" + field + ("_total" if kind == "counter" else "")
+ value = len(self.queue) if field == "queued" else getattr(self, field)
+ lines.extend((f"# HELP {name} {help_text}", f"# TYPE {name} {kind}",
+ f"{name} {value}"))
+ for field, help_text in (
+ ("queue_wait_seconds", "Queue wait of admitted requests only."),
+ ("slot_duration_seconds", "Slot occupancy of finished admitted requests, including errors and cancellation.")):
+ name = "colibri_scheduler_" + field
+ timing = self.timings[field]
+ lines.extend((f"# HELP {name} {help_text}", f"# TYPE {name} histogram"))
+ for bound, count in zip(self._buckets, timing["buckets"]):
+ label = "+Inf" if math.isinf(bound) else str(bound)
+ lines.append(f'{name}_bucket{{le="{label}"}} {count}')
+ lines.extend((f'{name}_sum {timing["sum"]}',
+ f'{name}_count {timing["buckets"][-1]}'))
+ return "\n".join(lines) + "\n"
+
def close(self):
with self.condition:
self.closed = True
@@ -3886,6 +3933,16 @@ class APIHandler(BaseHTTPRequestHandler):
try:
self._check_host()
path = urlsplit(self.path).path
+ if path == "/metrics":
+ self.require_auth()
+ data = self.server.scheduler.prometheus().encode("utf-8")
+ self.send_response(200)
+ self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+ self.send_header("Content-Length", str(len(data)))
+ self.send_header("Cache-Control", "no-store")
+ self.end_headers()
+ self.wfile.write(data)
+ return
if path == "/health":
# Liveness is always public; hardware/scheduler internals only when a
# request is authed (or no key set), so a configured key isn't leaked
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 75e691e7..581b5443 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -533,6 +533,48 @@ class ProtocolTest(unittest.TestCase):
class SchedulerTest(unittest.TestCase):
+ def test_engine_failure_is_not_a_completed_request(self):
+ scheduler = GenerationScheduler()
+ with self.assertRaisesRegex(RuntimeError, "engine failed"):
+ with scheduler.admit():
+ raise RuntimeError("engine failed")
+ stats = scheduler.snapshot()
+ self.assertEqual(stats["failed"], 1)
+ self.assertEqual(stats["completed"], 0)
+ self.assertEqual(stats["active"], 0)
+ with scheduler.admit():
+ pass
+ self.assertEqual(scheduler.snapshot()["completed"], 1)
+
+ def test_prometheus_histograms_measure_admission_and_slot_occupancy(self):
+ scheduler = GenerationScheduler()
+ with patch("openai_server.time.monotonic", side_effect=[10, 10.25, 12.25]):
+ with scheduler.admit():
+ active = scheduler.prometheus()
+ self.assertIn("colibri_scheduler_active 1\n", active)
+ self.assertIn("colibri_scheduler_slot_duration_seconds_count 0\n", active)
+ metrics = scheduler.prometheus()
+ self.assertIn("# TYPE colibri_scheduler_completed_total counter\n", metrics)
+ self.assertIn("colibri_scheduler_completed_total 1\n", metrics)
+ self.assertIn("colibri_scheduler_queue_wait_seconds_sum 0.25\n", metrics)
+ self.assertIn('colibri_scheduler_queue_wait_seconds_bucket{le="0.1"} 0\n', metrics)
+ self.assertIn('colibri_scheduler_queue_wait_seconds_bucket{le="0.5"} 1\n', metrics)
+ self.assertIn('colibri_scheduler_queue_wait_seconds_bucket{le="+Inf"} 1\n', metrics)
+ self.assertIn("colibri_scheduler_slot_duration_seconds_sum 2.0\n", metrics)
+ self.assertIn("colibri_scheduler_slot_duration_seconds_count 1\n", metrics)
+
+ def test_failed_and_cancelled_admissions_are_timed(self):
+ scheduler = GenerationScheduler()
+ for error in (RuntimeError("failed"), ClientCancelled()):
+ with self.assertRaises(type(error)):
+ with scheduler.admit():
+ raise error
+ metrics = scheduler.prometheus()
+ self.assertIn("colibri_scheduler_failed_total 1\n", metrics)
+ self.assertIn("colibri_scheduler_cancelled_total 1\n", metrics)
+ self.assertIn("colibri_scheduler_completed_total 0\n", metrics)
+ self.assertIn("colibri_scheduler_slot_duration_seconds_count 2\n", metrics)
+
def test_admits_up_to_capacity_without_serializing(self):
scheduler = GenerationScheduler(max_queue=0, queue_timeout=1, capacity=2)
with scheduler.admit() as first:
@@ -1346,6 +1388,41 @@ class HTTPTest(unittest.TestCase):
self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 401)
+ def test_metrics_counts_http_engine_failure_without_success(self):
+ before = self.server.scheduler.snapshot()
+ with patch.object(self.engine, "generate", side_effect=RuntimeError("injected failure")):
+ with self.assertRaises(HTTPError) as caught:
+ self.request("/v1/chat/completions", {
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "hello"}], "max_tokens": 1})
+ self.addCleanup(caught.exception.close)
+ self.assertEqual(caught.exception.code, 500)
+ after = self.server.scheduler.snapshot()
+ self.assertEqual(after["failed"], before["failed"] + 1)
+ self.assertEqual(after["completed"], before["completed"])
+ with self.request("/metrics") as response:
+ text = response.read().decode()
+ self.assertIn(f'colibri_scheduler_failed_total {after["failed"]}\n', text)
+ self.assertIn("colibri_scheduler_active 0\n", text)
+
+ def test_metrics_exposes_prometheus_text_with_auth(self):
+ with self.request("/metrics") as response:
+ self.assertEqual(response.headers["Content-Type"],
+ "text/plain; version=0.0.4; charset=utf-8")
+ self.assertEqual(response.headers["Cache-Control"], "no-store")
+ text = response.read().decode()
+ self.assertIn("colibri_scheduler_capacity 2\n", text)
+ self.assertIn("# TYPE colibri_scheduler_queue_wait_seconds histogram\n", text)
+ for key in ("wrong", ""):
+ with self.assertRaises(HTTPError) as caught:
+ self.request("/metrics", key=key)
+ self.addCleanup(caught.exception.close)
+ self.assertEqual(caught.exception.code, 401)
+ with self.assertRaises(HTTPError) as caught:
+ urlopen(self.base + "/metrics", timeout=2)
+ self.addCleanup(caught.exception.close)
+ self.assertEqual(caught.exception.code, 401)
+
def test_health_reports_scheduler_and_kv_slots(self):
with self.request("/health") as response:
health = json.load(response)
diff --git a/docs/api.md b/docs/api.md
index 3d014bbc..b2cbaa4b 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -99,6 +99,46 @@ errors before streaming headers are sent. `GET /health` exposes
active/queued/completed/rejected counters, and successful generation responses
include `x-colibri-queue-wait-ms`.
+## Prometheus metrics
+
+`GET /metrics` returns Prometheus text exposition (version 0.0.4). When an
+API key is configured, supply the same `Authorization: Bearer ...` or
+`x-api-key` header used for generation; missing or invalid credentials return
+401. Without an API key, the endpoint follows the server's usual unauthenticated
+access policy. Metrics contain no prompts, model paths, or request-ID labels.
+
+All names start with `colibri_scheduler_`:
+
+| Suffix | Type | Meaning |
+|---|---|---|
+| `active`, `queued`, `capacity`, `max_queue` | gauge | Admitted requests, waiters, KV slot capacity, and queue limit |
+| `admitted_total` | counter | Requests admitted to a KV slot |
+| `completed_total` | counter | Admitted requests that returned normally |
+| `failed_total` | counter | Admitted requests that raised an error, excluding `ClientCancelled` |
+| `rejected_total`, `timed_out_total` | counter | Queue-full refusals and queue timeouts |
+| `cancelled_total` | counter | Cancellations while queued or admitted |
+| `queue_wait_seconds` | histogram | Wait until admission, for admitted requests only |
+| `slot_duration_seconds` | histogram | Slot occupancy until completion, failure, or cancellation |
+
+Histogram buckets are 0.001, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 300 seconds,
+and `+Inf`; each histogram exposes `_bucket`, `_sum`, and `_count`.
+Counters reset when the gateway restarts. Collection does not call the engine
+or consume a generation slot. `failed` is also included in `/health`'s
+authenticated scheduler snapshot; failures no longer increment `completed`.
+
+These are admission/slot metrics, not TTFT, per-token latency, or GPU kernel
+measurements. Slot occupancy includes any response handling while the slot is
+held. Validation/authentication failures before admission are not counted.
+`completed` means the admitted handler returned normally, not that the client
+received every response byte. Request exceptions can include client input or
+transport errors as well as engine failures.
+
+Example PromQL for the admitted-request queue-wait p95:
+
+```promql
+histogram_quantile(0.95, sum by (le) (rate(colibri_scheduler_queue_wait_seconds_bucket[5m])))
+```
+
## Anthropic-protocol endpoint (`/v1/messages`)
The same server also speaks the **Anthropic Messages API**, so clients that only talk