Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(serve): enforce zero queue limit for busy pinned slots
ZacharyZcR5ef694d57af2
diff --git a/c/openai_server.py b/c/openai_server.pyindex 00bd6cf5..7c2856eb 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -146,7 +146,8 @@ class GenerationScheduler:if self.closed:raise APIError(503, "The inference scheduler is shutting down.", None,"scheduler_closed", "server_error")- if (self.active >= self.capacity or self.queue) and len(self.queue) >= self.max_queue:+ slot_busy = not self.free_slots if slot is None else slot not in self.free_slots+ if (slot_busy or self.queue) and len(self.queue) >= self.max_queue:self.rejected += 1raise APIError(429, "The inference queue is full.", None, "queue_full","rate_limit_error", {"Retry-After": "1"})diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 089d4d5a..8a4d5a14 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -650,6 +650,20 @@ class SchedulerTest(unittest.TestCase):self.assertEqual(caught.exception.code, "queue_full")self.assertEqual(scheduler.snapshot()["rejected"], 1)+ def test_zero_queue_rejects_busy_pinned_slot_with_spare_capacity(self):+ scheduler = GenerationScheduler(max_queue=0, queue_timeout=0.01, capacity=2)+ with scheduler.admit(slot=0):+ with self.assertRaises(APIError) as caught:+ with scheduler.admit(slot=0):+ self.fail("busy pinned slot admitted")+ self.assertEqual(caught.exception.code, "queue_full")+ with scheduler.admit(slot=1) as (_, slot):+ self.assertEqual(slot, 1)+ stats = scheduler.snapshot()+ self.assertEqual((stats["rejected"], stats["timed_out"], stats["queued"]), (1, 0, 0))+ self.assertEqual((stats["admitted"], stats["completed"], stats["active"]), (2, 2, 0))+ self.assertIn("colibri_scheduler_queue_wait_seconds_count 2\n", scheduler.prometheus())+def test_times_out_and_cancels_queued_requests(self):scheduler = GenerationScheduler(max_queue=2, queue_timeout=0.02)with scheduler.admit():diff --git a/docs/api.md b/docs/api.mdindex be551c53..41f88596 100644--- a/docs/api.md+++ b/docs/api.md@@ -95,7 +95,9 @@ admission queue instead of pretending to run unsafe parallel sequences.Configure it with `--max-queue N` (default 8) and `--queue-timeout SECONDS`(default 300), or the `COLI_MAX_QUEUE` / `COLI_QUEUE_TIMEOUT` environmentvariables. Saturated and timed-out requests receive OpenAI-shaped HTTP 429-errors before streaming headers are sent. `GET /health` exposes+errors before streaming headers are sent. With `--max-queue 0`, a request+pinned to an occupied KV slot is rejected immediately even if another slot+is free. `GET /health` exposesactive/queued/completed/rejected counters, and successful generation responsesinclude `x-colibri-queue-wait-ms`.