Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(serve): check cancellation before admitting an available slot
ZacharyZcR24788ac9e5ef
diff --git a/c/openai_server.py b/c/openai_server.pyindex 2128f9da..00bd6cf5 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -158,6 +158,11 @@ class GenerationScheduler:self.condition.notify_all()raise APIError(503, "The inference scheduler is shutting down.", None,"scheduler_closed", "server_error")+ if cancelled and cancelled():+ self.queue.remove(entry)+ self.cancelled += 1+ self.condition.notify_all()+ raise ClientCancelled()available = min(self.free_slots) if slot is None and self.free_slots else slot# (#B2) Admit as soon as our target slot is free AND no strictly-earlier# waiter also wants it (an earlier waiter "wants" it if it is any-slot or@@ -176,11 +181,6 @@ class GenerationScheduler:breakif can_admit:break- if cancelled and cancelled():- self.queue.remove(entry)- self.cancelled += 1- self.condition.notify_all()- raise ClientCancelled()remaining = deadline - time.monotonic()if remaining <= 0:self.queue.remove(entry)diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 4d7065ff..089d4d5a 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -664,6 +664,51 @@ class SchedulerTest(unittest.TestCase):self.assertEqual(stats["timed_out"], 1)self.assertEqual(stats["cancelled"], 1)+ def test_cancelled_request_does_not_acquire_a_free_slot(self):+ scheduler = GenerationScheduler()+ with self.assertRaises(ClientCancelled):+ with scheduler.admit(lambda: True):+ self.fail("cancelled request admitted")+ stats = scheduler.snapshot()+ self.assertEqual((stats["active"], stats["queued"], stats["admitted"], stats["cancelled"]),+ (0, 0, 0, 1))+ self.assertIn("colibri_scheduler_queue_wait_seconds_count 0\n", scheduler.prometheus())+ with scheduler.admit():+ pass++ def test_cancellation_wins_when_a_waiting_slot_becomes_free(self):+ scheduler = GenerationScheduler(queue_timeout=1)+ waiting = threading.Event()+ cancelled = threading.Event()+ outcomes = []+ holder = scheduler.admit()+ holder.__enter__()+ def is_cancelled():+ waiting.set()+ return cancelled.is_set()+ def run():+ try:+ with scheduler.admit(is_cancelled):+ outcomes.append("admitted")+ except ClientCancelled:+ outcomes.append("cancelled")+ thread = threading.Thread(target=run)+ thread.start()+ observed = waiting.wait(1)+ # Publish cancellation and release capacity under the same lock, so+ # the waiter must observe both on its next scheduling pass.+ with scheduler.condition:+ cancelled.set()+ holder.__exit__(None, None, None)+ thread.join(2)+ self.assertTrue(observed)+ self.assertFalse(thread.is_alive())+ self.assertEqual(outcomes, ["cancelled"])+ stats = scheduler.snapshot()+ self.assertEqual((stats["admitted"], stats["completed"], stats["cancelled"]), (1, 1, 1))+ self.assertEqual((stats["active"], stats["queued"]), (0, 0))+ self.assertIn("colibri_scheduler_slot_duration_seconds_count 1\n", scheduler.prometheus())+def test_counts_admitted_client_cancellation_without_completion(self):scheduler = GenerationScheduler(max_queue=0, queue_timeout=1)with self.assertRaises(ClientCancelled):diff --git a/docs/api.md b/docs/api.mdindex e4cdfced..be551c53 100644--- a/docs/api.md+++ b/docs/api.md@@ -116,7 +116,7 @@ All names start with `colibri_scheduler_`:| `completed_total` | counter | Admitted requests that returned normally || `failed_total` | counter | Admitted requests that raised an error, excluding `ClientCancelled` || `rejected_total`, `timed_out_total` | counter | Queue-full refusals and queue timeouts |-| `cancelled_total` | counter | Cancellations while queued or admitted |+| `cancelled_total` | counter | Cancellations detected before admission or during admitted work || `queue_wait_seconds` | histogram | Wait until admission, for admitted requests only || `slot_duration_seconds` | histogram | Slot occupancy until completion, failure, or cancellation || `first_output_seconds` | histogram | Engine-call start to first nonempty text or tool-output callback |@@ -128,6 +128,13 @@ Counters reset when the gateway restarts. Collection does not call the engineor consume a generation slot. `failed` is also included in `/health`'sauthenticated scheduler snapshot; failures no longer increment `completed`.+Cancellation is checked before acquiring an available KV slot, including when a+waiting request wakes as capacity becomes free. A request cancelled at this+point increments `cancelled_total`, but not `admitted_total`, and contributes+no admission-wait or slot-duration sample. Queue-full and scheduler-closed+checks can still reject a request before the cancellation check is reached.++The engine-call histograms exclude admission queue wait and prompt rendering.First output is observed before the gateway's stop filtering, reasoning split,or HTTP serialization: it can be reasoning or tool data, not necessarily