Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(serve): check cancellation before admitting an available slot

ZacharyZcR24788ac9e5ef
diff --git a/c/openai_server.py b/c/openai_server.py
index 2128f9da..00bd6cf5 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -158,6 +158,11 @@ class GenerationScheduler:
self.condition.notify_all()
raise APIError(503, "The inference scheduler is shutting down.", None,
"scheduler_closed", "server_error")
+ if cancelled and cancelled():
+ self.queue.remove(entry)
+ self.cancelled += 1
+ self.condition.notify_all()
+ raise ClientCancelled()
available = min(self.free_slots) if slot is None and self.free_slots else slot
# (#B2) Admit as soon as our target slot is free AND no strictly-earlier
# waiter also wants it (an earlier waiter "wants" it if it is any-slot or
@@ -176,11 +181,6 @@ class GenerationScheduler:
break
if can_admit:
break
- if cancelled and cancelled():
- self.queue.remove(entry)
- self.cancelled += 1
- self.condition.notify_all()
- raise ClientCancelled()
remaining = deadline - time.monotonic()
if remaining <= 0:
self.queue.remove(entry)
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 4d7065ff..089d4d5a 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -664,6 +664,51 @@ class SchedulerTest(unittest.TestCase):
self.assertEqual(stats["timed_out"], 1)
self.assertEqual(stats["cancelled"], 1)
+ def test_cancelled_request_does_not_acquire_a_free_slot(self):
+ scheduler = GenerationScheduler()
+ with self.assertRaises(ClientCancelled):
+ with scheduler.admit(lambda: True):
+ self.fail("cancelled request admitted")
+ stats = scheduler.snapshot()
+ self.assertEqual((stats["active"], stats["queued"], stats["admitted"], stats["cancelled"]),
+ (0, 0, 0, 1))
+ self.assertIn("colibri_scheduler_queue_wait_seconds_count 0\n", scheduler.prometheus())
+ with scheduler.admit():
+ pass
+
+ def test_cancellation_wins_when_a_waiting_slot_becomes_free(self):
+ scheduler = GenerationScheduler(queue_timeout=1)
+ waiting = threading.Event()
+ cancelled = threading.Event()
+ outcomes = []
+ holder = scheduler.admit()
+ holder.__enter__()
+ def is_cancelled():
+ waiting.set()
+ return cancelled.is_set()
+ def run():
+ try:
+ with scheduler.admit(is_cancelled):
+ outcomes.append("admitted")
+ except ClientCancelled:
+ outcomes.append("cancelled")
+ thread = threading.Thread(target=run)
+ thread.start()
+ observed = waiting.wait(1)
+ # Publish cancellation and release capacity under the same lock, so
+ # the waiter must observe both on its next scheduling pass.
+ with scheduler.condition:
+ cancelled.set()
+ holder.__exit__(None, None, None)
+ thread.join(2)
+ self.assertTrue(observed)
+ self.assertFalse(thread.is_alive())
+ self.assertEqual(outcomes, ["cancelled"])
+ stats = scheduler.snapshot()
+ self.assertEqual((stats["admitted"], stats["completed"], stats["cancelled"]), (1, 1, 1))
+ self.assertEqual((stats["active"], stats["queued"]), (0, 0))
+ self.assertIn("colibri_scheduler_slot_duration_seconds_count 1\n", scheduler.prometheus())
+
def test_counts_admitted_client_cancellation_without_completion(self):
scheduler = GenerationScheduler(max_queue=0, queue_timeout=1)
with self.assertRaises(ClientCancelled):
diff --git a/docs/api.md b/docs/api.md
index e4cdfced..be551c53 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -116,7 +116,7 @@ All names start with `colibri_scheduler_`:
| `completed_total` | counter | Admitted requests that returned normally |
| `failed_total` | counter | Admitted requests that raised an error, excluding `ClientCancelled` |
| `rejected_total`, `timed_out_total` | counter | Queue-full refusals and queue timeouts |
-| `cancelled_total` | counter | Cancellations while queued or admitted |
+| `cancelled_total` | counter | Cancellations detected before admission or during admitted work |
| `queue_wait_seconds` | histogram | Wait until admission, for admitted requests only |
| `slot_duration_seconds` | histogram | Slot occupancy until completion, failure, or cancellation |
| `first_output_seconds` | histogram | Engine-call start to first nonempty text or tool-output callback |
@@ -128,6 +128,13 @@ Counters reset when the gateway restarts. Collection does not call the engine
or consume a generation slot. `failed` is also included in `/health`'s
authenticated scheduler snapshot; failures no longer increment `completed`.
+Cancellation is checked before acquiring an available KV slot, including when a
+waiting request wakes as capacity becomes free. A request cancelled at this
+point increments `cancelled_total`, but not `admitted_total`, and contributes
+no admission-wait or slot-duration sample. Queue-full and scheduler-closed
+checks can still reject a request before the cancellation check is reached.
+
+
The engine-call histograms exclude admission queue wait and prompt rendering.
First output is observed before the gateway's stop filtering, reasoning split,
or HTTP serialization: it can be reasoning or tool data, not necessarily