Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(serve): expose Prometheus scheduler metrics
ZacharyZcR90ff72bb773e
diff --git a/c/openai_server.py b/c/openai_server.pyindex d3de8dea..57888ac7 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -110,6 +110,8 @@ def _engine_error(fields, message):class GenerationScheduler:"""Bounded FIFO admission for the engine's independent KV contexts."""+ _buckets = (0.001, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 300, math.inf)+def __init__(self, max_queue=8, queue_timeout=300, capacity=1):if max_queue < 0:raise ValueError("max_queue cannot be negative")@@ -127,9 +129,12 @@ class GenerationScheduler:self.closed = Falseself.admitted = 0self.completed = 0+ self.failed = 0self.rejected = 0self.timed_out = 0self.cancelled = 0+ self.timings = {name: {"sum": 0.0, "buckets": [0] * len(self._buckets)}+ for name in ("queue_wait_seconds", "slot_duration_seconds")}@contextlib.contextmanagerdef admit(self, cancelled=None, slot=None):@@ -187,21 +192,22 @@ class GenerationScheduler:self.free_slots.remove(available)self.active += 1self.admitted += 1- wait_seconds = time.monotonic() - queued_at- cancelled_after_admission = False+ admitted_at = time.monotonic()+ wait_seconds = admitted_at - queued_at+ self._observe("queue_wait_seconds", wait_seconds)+ outcome = "failed"try:yield wait_seconds, available+ outcome = "completed"except ClientCancelled:- cancelled_after_admission = True+ outcome = "cancelled"raisefinally:with self.condition:self.active -= 1self.free_slots.add(available)- if cancelled_after_admission:- self.cancelled += 1- else:- self.completed += 1+ setattr(self, outcome, getattr(self, outcome) + 1)+ self._observe("slot_duration_seconds", time.monotonic() - admitted_at)self.condition.notify_all()def snapshot(self):@@ -209,10 +215,51 @@ class GenerationScheduler:return {"active": self.active, "queued": len(self.queue),"capacity": self.capacity,"max_queue": self.max_queue, "queue_timeout_seconds": self.queue_timeout,- "admitted": self.admitted, "completed": self.completed,+ "admitted": self.admitted, "completed": self.completed, "failed": self.failed,"rejected": self.rejected, "timed_out": self.timed_out,"cancelled": self.cancelled}+ def _observe(self, name, seconds):+ # Called with condition held. Cumulative buckets need no request history.+ timing = self.timings[name]+ timing["sum"] += seconds+ for i, bound in enumerate(self._buckets):+ if seconds <= bound:+ timing["buckets"][i] += 1++ def prometheus(self):+ """One consistent, bounded snapshot; no prompt or request-ID labels."""+ gauges = {"active": "Currently admitted requests.",+ "queued": "Requests waiting for a KV slot.",+ "capacity": "Concurrent KV slots configured.",+ "max_queue": "Maximum waiting requests configured."}+ counters = {"admitted": "Requests admitted to a KV slot.",+ "completed": "Admitted requests that returned normally.",+ "failed": "Admitted requests that raised an error.",+ "rejected": "Requests rejected because the queue was full.",+ "timed_out": "Requests that timed out waiting for a slot.",+ "cancelled": "Requests cancelled while queued or admitted."}+ lines = []+ with self.condition:+ for kind, fields in (("gauge", gauges), ("counter", counters)):+ for field, help_text in fields.items():+ name = "colibri_scheduler_" + field + ("_total" if kind == "counter" else "")+ value = len(self.queue) if field == "queued" else getattr(self, field)+ lines.extend((f"# HELP {name} {help_text}", f"# TYPE {name} {kind}",+ f"{name} {value}"))+ for field, help_text in (+ ("queue_wait_seconds", "Queue wait of admitted requests only."),+ ("slot_duration_seconds", "Slot occupancy of finished admitted requests, including errors and cancellation.")):+ name = "colibri_scheduler_" + field+ timing = self.timings[field]+ lines.extend((f"# HELP {name} {help_text}", f"# TYPE {name} histogram"))+ for bound, count in zip(self._buckets, timing["buckets"]):+ label = "+Inf" if math.isinf(bound) else str(bound)+ lines.append(f'{name}_bucket{{le="{label}"}} {count}')+ lines.extend((f'{name}_sum {timing["sum"]}',+ f'{name}_count {timing["buckets"][-1]}'))+ return "\n".join(lines) + "\n"+def close(self):with self.condition:self.closed = True@@ -3886,6 +3933,16 @@ class APIHandler(BaseHTTPRequestHandler):try:self._check_host()path = urlsplit(self.path).path+ if path == "/metrics":+ self.require_auth()+ data = self.server.scheduler.prometheus().encode("utf-8")+ self.send_response(200)+ self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")+ self.send_header("Content-Length", str(len(data)))+ self.send_header("Cache-Control", "no-store")+ self.end_headers()+ self.wfile.write(data)+ returnif path == "/health":# Liveness is always public; hardware/scheduler internals only when a# request is authed (or no key set), so a configured key isn't leakeddiff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 75e691e7..581b5443 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -533,6 +533,48 @@ class ProtocolTest(unittest.TestCase):class SchedulerTest(unittest.TestCase):+ def test_engine_failure_is_not_a_completed_request(self):+ scheduler = GenerationScheduler()+ with self.assertRaisesRegex(RuntimeError, "engine failed"):+ with scheduler.admit():+ raise RuntimeError("engine failed")+ stats = scheduler.snapshot()+ self.assertEqual(stats["failed"], 1)+ self.assertEqual(stats["completed"], 0)+ self.assertEqual(stats["active"], 0)+ with scheduler.admit():+ pass+ self.assertEqual(scheduler.snapshot()["completed"], 1)++ def test_prometheus_histograms_measure_admission_and_slot_occupancy(self):+ scheduler = GenerationScheduler()+ with patch("openai_server.time.monotonic", side_effect=[10, 10.25, 12.25]):+ with scheduler.admit():+ active = scheduler.prometheus()+ self.assertIn("colibri_scheduler_active 1\n", active)+ self.assertIn("colibri_scheduler_slot_duration_seconds_count 0\n", active)+ metrics = scheduler.prometheus()+ self.assertIn("# TYPE colibri_scheduler_completed_total counter\n", metrics)+ self.assertIn("colibri_scheduler_completed_total 1\n", metrics)+ self.assertIn("colibri_scheduler_queue_wait_seconds_sum 0.25\n", metrics)+ self.assertIn('colibri_scheduler_queue_wait_seconds_bucket{le="0.1"} 0\n', metrics)+ self.assertIn('colibri_scheduler_queue_wait_seconds_bucket{le="0.5"} 1\n', metrics)+ self.assertIn('colibri_scheduler_queue_wait_seconds_bucket{le="+Inf"} 1\n', metrics)+ self.assertIn("colibri_scheduler_slot_duration_seconds_sum 2.0\n", metrics)+ self.assertIn("colibri_scheduler_slot_duration_seconds_count 1\n", metrics)++ def test_failed_and_cancelled_admissions_are_timed(self):+ scheduler = GenerationScheduler()+ for error in (RuntimeError("failed"), ClientCancelled()):+ with self.assertRaises(type(error)):+ with scheduler.admit():+ raise error+ metrics = scheduler.prometheus()+ self.assertIn("colibri_scheduler_failed_total 1\n", metrics)+ self.assertIn("colibri_scheduler_cancelled_total 1\n", metrics)+ self.assertIn("colibri_scheduler_completed_total 0\n", metrics)+ self.assertIn("colibri_scheduler_slot_duration_seconds_count 2\n", metrics)+def test_admits_up_to_capacity_without_serializing(self):scheduler = GenerationScheduler(max_queue=0, queue_timeout=1, capacity=2)with scheduler.admit() as first:@@ -1346,6 +1388,41 @@ class HTTPTest(unittest.TestCase):self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 401)+ def test_metrics_counts_http_engine_failure_without_success(self):+ before = self.server.scheduler.snapshot()+ with patch.object(self.engine, "generate", side_effect=RuntimeError("injected failure")):+ with self.assertRaises(HTTPError) as caught:+ self.request("/v1/chat/completions", {+ "model": "test-model",+ "messages": [{"role": "user", "content": "hello"}], "max_tokens": 1})+ self.addCleanup(caught.exception.close)+ self.assertEqual(caught.exception.code, 500)+ after = self.server.scheduler.snapshot()+ self.assertEqual(after["failed"], before["failed"] + 1)+ self.assertEqual(after["completed"], before["completed"])+ with self.request("/metrics") as response:+ text = response.read().decode()+ self.assertIn(f'colibri_scheduler_failed_total {after["failed"]}\n', text)+ self.assertIn("colibri_scheduler_active 0\n", text)++ def test_metrics_exposes_prometheus_text_with_auth(self):+ with self.request("/metrics") as response:+ self.assertEqual(response.headers["Content-Type"],+ "text/plain; version=0.0.4; charset=utf-8")+ self.assertEqual(response.headers["Cache-Control"], "no-store")+ text = response.read().decode()+ self.assertIn("colibri_scheduler_capacity 2\n", text)+ self.assertIn("# TYPE colibri_scheduler_queue_wait_seconds histogram\n", text)+ for key in ("wrong", ""):+ with self.assertRaises(HTTPError) as caught:+ self.request("/metrics", key=key)+ self.addCleanup(caught.exception.close)+ self.assertEqual(caught.exception.code, 401)+ with self.assertRaises(HTTPError) as caught:+ urlopen(self.base + "/metrics", timeout=2)+ self.addCleanup(caught.exception.close)+ self.assertEqual(caught.exception.code, 401)+def test_health_reports_scheduler_and_kv_slots(self):with self.request("/health") as response:health = json.load(response)diff --git a/docs/api.md b/docs/api.mdindex 3d014bbc..b2cbaa4b 100644--- a/docs/api.md+++ b/docs/api.md@@ -99,6 +99,46 @@ errors before streaming headers are sent. `GET /health` exposesactive/queued/completed/rejected counters, and successful generation responsesinclude `x-colibri-queue-wait-ms`.+## Prometheus metrics++`GET /metrics` returns Prometheus text exposition (version 0.0.4). When an+API key is configured, supply the same `Authorization: Bearer ...` or+`x-api-key` header used for generation; missing or invalid credentials return+401. Without an API key, the endpoint follows the server's usual unauthenticated+access policy. Metrics contain no prompts, model paths, or request-ID labels.++All names start with `colibri_scheduler_`:++| Suffix | Type | Meaning |+|---|---|---|+| `active`, `queued`, `capacity`, `max_queue` | gauge | Admitted requests, waiters, KV slot capacity, and queue limit |+| `admitted_total` | counter | Requests admitted to a KV slot |+| `completed_total` | counter | Admitted requests that returned normally |+| `failed_total` | counter | Admitted requests that raised an error, excluding `ClientCancelled` |+| `rejected_total`, `timed_out_total` | counter | Queue-full refusals and queue timeouts |+| `cancelled_total` | counter | Cancellations while queued or admitted |+| `queue_wait_seconds` | histogram | Wait until admission, for admitted requests only |+| `slot_duration_seconds` | histogram | Slot occupancy until completion, failure, or cancellation |++Histogram buckets are 0.001, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 300 seconds,+and `+Inf`; each histogram exposes `_bucket`, `_sum`, and `_count`.+Counters reset when the gateway restarts. Collection does not call the engine+or consume a generation slot. `failed` is also included in `/health`'s+authenticated scheduler snapshot; failures no longer increment `completed`.++These are admission/slot metrics, not TTFT, per-token latency, or GPU kernel+measurements. Slot occupancy includes any response handling while the slot is+held. Validation/authentication failures before admission are not counted.+`completed` means the admitted handler returned normally, not that the client+received every response byte. Request exceptions can include client input or+transport errors as well as engine failures.++Example PromQL for the admitted-request queue-wait p95:++```promql+histogram_quantile(0.95, sum by (le) (rate(colibri_scheduler_queue_wait_seconds_bucket[5m])))+```+## Anthropic-protocol endpoint (`/v1/messages`)The same server also speaks the **Anthropic Messages API**, so clients that only talk