Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(serve): enforce zero queue limit for busy pinned slots

ZacharyZcR5ef694d57af2
diff --git a/c/openai_server.py b/c/openai_server.py
index 00bd6cf5..7c2856eb 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -146,7 +146,8 @@ class GenerationScheduler:
if self.closed:
raise APIError(503, "The inference scheduler is shutting down.", None,
"scheduler_closed", "server_error")
- if (self.active >= self.capacity or self.queue) and len(self.queue) >= self.max_queue:
+ slot_busy = not self.free_slots if slot is None else slot not in self.free_slots
+ if (slot_busy or self.queue) and len(self.queue) >= self.max_queue:
self.rejected += 1
raise APIError(429, "The inference queue is full.", None, "queue_full",
"rate_limit_error", {"Retry-After": "1"})
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 089d4d5a..8a4d5a14 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -650,6 +650,20 @@ class SchedulerTest(unittest.TestCase):
self.assertEqual(caught.exception.code, "queue_full")
self.assertEqual(scheduler.snapshot()["rejected"], 1)
+ def test_zero_queue_rejects_busy_pinned_slot_with_spare_capacity(self):
+ scheduler = GenerationScheduler(max_queue=0, queue_timeout=0.01, capacity=2)
+ with scheduler.admit(slot=0):
+ with self.assertRaises(APIError) as caught:
+ with scheduler.admit(slot=0):
+ self.fail("busy pinned slot admitted")
+ self.assertEqual(caught.exception.code, "queue_full")
+ with scheduler.admit(slot=1) as (_, slot):
+ self.assertEqual(slot, 1)
+ stats = scheduler.snapshot()
+ self.assertEqual((stats["rejected"], stats["timed_out"], stats["queued"]), (1, 0, 0))
+ self.assertEqual((stats["admitted"], stats["completed"], stats["active"]), (2, 2, 0))
+ self.assertIn("colibri_scheduler_queue_wait_seconds_count 2\n", scheduler.prometheus())
+
def test_times_out_and_cancels_queued_requests(self):
scheduler = GenerationScheduler(max_queue=2, queue_timeout=0.02)
with scheduler.admit():
diff --git a/docs/api.md b/docs/api.md
index be551c53..41f88596 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -95,7 +95,9 @@ admission queue instead of pretending to run unsafe parallel sequences.
Configure it with `--max-queue N` (default 8) and `--queue-timeout SECONDS`
(default 300), or the `COLI_MAX_QUEUE` / `COLI_QUEUE_TIMEOUT` environment
variables. Saturated and timed-out requests receive OpenAI-shaped HTTP 429
-errors before streaming headers are sent. `GET /health` exposes
+errors before streaming headers are sent. With `--max-queue 0`, a request
+pinned to an occupied KV slot is rejected immediately even if another slot
+is free. `GET /health` exposes
active/queued/completed/rejected counters, and successful generation responses
include `x-colibri-queue-wait-ms`.