Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(qwen36): wake cv_take waiters on tier shutdown (#1340)
Courtney Andrew Richardson246ce2cd1cca
diff --git a/c/Makefile b/c/Makefileindex 3e5d7bd3..1c61bd51 100644--- a/c/Makefile+++ b/c/Makefile@@ -1650,6 +1650,12 @@ tests/test_qwen36_tier_int8$(EXE): tests/test_qwen36_tier_int8.c tests/qwen36_fatests/test_qwen36_tier_multidev$(EXE): tests/test_qwen36_tier_multidev.c tests/qwen36_fake_cuda.h qwen36_tier.c qwen36_tier.h backend_cuda.h$(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)+# Same fake backend, single device: proves qt_shutdown returns (under an+# alarm(10) watchdog) instead of hanging forever when a group is still open+# and an LFRU swap is parked waiting for cv_take (#1340).+tests/test_qwen36_tier_shutdown$(EXE): tests/test_qwen36_tier_shutdown.c tests/qwen36_fake_cuda.h qwen36_tier.c qwen36_tier.h backend_cuda.h+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)+tests/test_serve_poll$(EXE): tests/test_serve_poll.c serve_poll.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/qwen36_tier.c b/c/qwen36_tier.cindex 4ac3209f..e032972f 100644--- a/c/qwen36_tier.c+++ b/c/qwen36_tier.c@@ -97,6 +97,17 @@ static void *uploader(void *arg){/* LFRU swap: free the victim only when no group is in flight */while(G.issue_open && !G.th_stop) pthread_cond_wait(&G.cv_take,&G.mx);QSlot *v=qs(vl,ve);+ if(G.th_stop && G.issue_open){+ /* Shutting down with a group still open: qt_take() -- the only+ * thing that clears issue_open -- will never come. Abandon+ * this swap instead of freeing a victim tensor the in-flight+ * group may still reference; qt_lfru_tick_locked already+ * cleared the victim's resident flag before enqueueing, so+ * restore it to keep the flag consistent with the tensor it+ * still holds. */+ v->resident=1; qs(layer,eid)->queued=0;+ pthread_mutex_unlock(&G.mx); free(w); free(sc); continue;+ }ColiCudaTensor *a=v->tg,*b=v->tu,*ct=v->td;v->tg=v->tu=v->td=NULL;pthread_mutex_unlock(&G.mx);@@ -497,7 +508,10 @@ void qt_shutdown(void){fprintf(stderr,"[qtier] HEAT_FILE saved: %s\n",hf);}}- pthread_mutex_lock(&G.mx); G.th_stop=1; pthread_cond_signal(&G.cv); pthread_mutex_unlock(&G.mx);+ /* Wake cv_take too: the uploader's LFRU victim wait (and qt_note_block /+ * qt_note_planned / qt_fill_wait, all waiting on the same condvar) would+ * otherwise never notice th_stop and pthread_join below would hang (#1340). */+ pthread_mutex_lock(&G.mx); G.th_stop=1; pthread_cond_signal(&G.cv); pthread_cond_broadcast(&G.cv_take); pthread_mutex_unlock(&G.mx);pthread_join(G.th,NULL);G.on=0;coli_cuda_shutdown();diff --git a/c/tests/test_qwen36_tier_shutdown.c b/c/tests/test_qwen36_tier_shutdown.cnew file mode 100644index 00000000..fad38885--- /dev/null+++ b/c/tests/test_qwen36_tier_shutdown.c@@ -0,0 +1,132 @@+/* qt_shutdown hangs when a group is open and a swap is queued (#1340).+ *+ * The defect: qt_shutdown sets G.th_stop and signals only G.cv, then+ * pthread_join()s the uploader. But the uploader's LFRU victim path waits on+ * G.cv_take -- "while(G.issue_open && !G.th_stop) pthread_cond_wait(&G.cv_take,+ * &G.mx)" -- and nothing ever broadcasts G.cv_take from qt_shutdown. If a+ * caller issued a group (qt_issue, which sets G.issue_open=1) and never took+ * it (qt_take is what clears issue_open and broadcasts G.cv_take), a queued+ * swap parks the uploader on that wait forever, and qt_shutdown's+ * pthread_join never returns.+ *+ * This test reproduces exactly that sequence with the fake CUDA backend+ * (tests/qwen36_fake_cuda.h, no GPU needed): budget one expert in, issue it+ * without taking it, queue an LFRU swap directly (what qt_lfru_tick_locked+ * does), then call qt_shutdown() under an alarm(10) so a hang fails loudly+ * instead of wedging CI. */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <signal.h>+#include <unistd.h>+#include <time.h>++#include "qwen36_fake_cuda.h"++#include "../qwen36_tier.c"++static int fails;+static void check(int ok, const char *what) {+ if (!ok) { printf(" FAIL: %s\n", what); fails++; }+}++static int issue_ok(int device, int count, const float *x) {+ (void)device; (void)count; (void)x; return 1;+}++static void on_alarm(int sig) {+ (void)sig;+ (void)!write(2,"FAIL: qt_shutdown hung\n",23);+ _exit(1);+}++/* Poll a G.mx-guarded predicate without a busy spin. */+static int wait_for_resident(int layer, int eid, int want, int max_polls) {+ for (int i = 0; i < max_polls; i++) {+ pthread_mutex_lock(&G.mx);+ int r = qs(layer, eid)->resident;+ pthread_mutex_unlock(&G.mx);+ if (r == want) return 1;+ struct timespec ts = {0, 2000000}; /* 2 ms */+ nanosleep(&ts, NULL);+ }+ return 0;+}++int main(void) {+ enum { D = 64 };+ setenv("COLI_CUDA", "1", 1);+ setenv("COLI_GPUS", "0", 1);+ setenv("QT_NO_WARMSTART", "1", 1);+ /* exp_bytes = 3*D*IH/2 + (2*IH+D)*4 + 4096 = 3072+512+4096 = 7680 B.+ * 0.00001 GiB ~= 10737 B: enough for one expert (7680), not two (15360). */+ setenv("CUDA_EXPERT_GB", "0.00001", 1);+ fake_ndev = 1;++ if (!qt_init(1, 2, D, 32, 2, 1, 0 /* per-row */, 1 /* int4 */)) {+ printf(" FAIL: il tier non parte\n");+ return 1;+ }++ static unsigned char g4[2][D * 32 / 2], u4[2][D * 32 / 2], d4[2][D * 32 / 2];+ static float sc[2][2 * 32 + D];+ for (int eid = 0; eid < 2; eid++) {+ memset(g4[eid], (unsigned char)(eid + 1), sizeof g4[eid]);+ memset(u4[eid], (unsigned char)(eid + 2), sizeof u4[eid]);+ memset(d4[eid], (unsigned char)(eid + 3), sizeof d4[eid]);+ for (int i = 0; i < 2 * 32 + D; i++) sc[eid][i] = 1.0f;+ }+ /* Expert 0 fits the budget and gets uploaded; expert 1's pointers are+ * stored (needed for the later swap) but the budget refuses its upload. */+ qt_note(0, 0, g4[0], u4[0], d4[0], sc[0], sc[0] + 32, sc[0] + 2 * 32);+ qt_note(0, 1, g4[1], u4[1], d4[1], sc[1], sc[1] + 32, sc[1] + 2 * 32);+ wait_for_resident(0, 0, 1, 500);+ check(qt_is_resident(0, 0), "expert 0 should have fit the one-expert budget");+ check(!qt_is_resident(0, 1), "expert 1 should have been refused by the exhausted budget");++ /* Open a group on the resident expert and deliberately never take it --+ * that is what leaves G.issue_open set going into shutdown. */+ fake_issue_hook = issue_ok;+ float x[D]; for (int i = 0; i < D; i++) x[i] = (float)i;+ int resident_eid = 0;+ uint32_t mask = qt_issue(0, &resident_eid, 1, x);+ check(mask == 1u, "issuing the resident expert should set mask bit 0");++ /* Queue an LFRU swap directly -- exactly what qt_lfru_tick_locked does. */+ pthread_mutex_lock(&G.mx);+ qs(0, resident_eid)->resident = 0;+ int queued = enqueue_locked(0, 1, 0, resident_eid, 0);+ pthread_mutex_unlock(&G.mx);+ check(queued, "the swap should have been enqueued (queue has room, victim provided)");++ /* Give the uploader a moment to dequeue the swap and park on the victim+ * wait (poll G.qn==0 under the lock, up to 1s). */+ int parked = 0;+ for (int i = 0; i < 500; i++) {+ pthread_mutex_lock(&G.mx);+ int qn = G.qn;+ pthread_mutex_unlock(&G.mx);+ if (qn == 0) { parked = 1; break; }+ struct timespec ts = {0, 2000000};+ nanosleep(&ts, NULL);+ }+ check(parked, "the uploader should have dequeued the swap within 1s");++ signal(SIGALRM, on_alarm);+ alarm(10);+ qt_shutdown();+ alarm(0);+ check(!G.on, "shutdown_returns_while_a_group_is_open");+ /* The abandoned swap must leave the victim exactly as the open group left+ * it, and must not have driven the incoming expert's upload after+ * shutdown began. The uploader is already joined here, so reading G+ * needs no lock. */+ check(qs(0, resident_eid)->resident && qs(0, resident_eid)->tg,+ "shutdown_abandons_the_swap_instead_of_freeing_the_victim");+ check(!qs(0, 1)->queued && !qs(0, 1)->resident && G.uploads == 1,+ "shutdown_abandons_the_swap_instead_of_uploading_the_incoming_expert");++ if (fails) { printf("test_qwen36_tier_shutdown: %d fallimenti\n", fails); return 1; }+ printf("test_qwen36_tier_shutdown: ok\n");+ return 0;+}