Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen36 tier: QT_HOME=layer, expert homes per layer range -- two cards as a pipeline
kreuzzelg14336c699e68
diff --git a/c/Makefile b/c/Makefileindex 3577f7ea..016ae2f4 100644--- a/c/Makefile+++ b/c/Makefile@@ -2399,6 +2399,10 @@ tests/test_qwen36_dnproj_batch$(EXE): tests/test_qwen36_dnproj_batch.c qwen36.ctests/test_qwen36_tier_withdraw$(EXE): tests/test_qwen36_tier_withdraw.c qwen36_tier.c$(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)+# QT_HOME=layer: expert homes per layer range, the cards as a pipeline (fake backend, two devices).+tests/test_qwen36_tier_layer_home$(EXE): tests/test_qwen36_tier_layer_home.c qwen36_tier.c+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)+tests/test_qwen36_tier_int8_engine$(EXE): tests/test_qwen36_tier_int8_engine.c qwen36.c qwen36_tier.c $(VK_OBJ)$(CC) $(CFLAGS) -DCOLI_CUDA $< $(VK_OBJ) -o $@ $(LDFLAGS)diff --git a/c/qwen36_tier.c b/c/qwen36_tier.cindex ad1f2f4d..c81f1972 100644--- a/c/qwen36_tier.c+++ b/c/qwen36_tier.c@@ -76,7 +76,13 @@ static void wait_take_locked(void){}static QSlot *qs(int layer, int eid){ return &G.slot[(size_t)layer*G.ne + eid]; }-static int home(int eid){ return eid % G.ndev; }+/* Expert homes. Default: expert eid lives on device index eid % ndev in every+ * layer (each layer's group joins across the cards). QT_HOME=layer: every+ * expert of layer l lives on one device, chosen per layer range, so a token+ * runs each layer on one card and the cards form a pipeline -- no per-layer+ * join, which on unequal cards is what the slower one paced (measured: take+ * 5.2 ms/token on two cards against 0.2 on one). */+static int home2(int layer, int eid);/* Staging: packed int4 (g|u|d) two's-complement -> offset-binary (XOR 0x88,* the upload format of backend_cuda fmt=2) + copy the scales (gs|us|ds). */@@ -183,7 +189,7 @@ static void *uploader(void *arg){if(a)coli_cuda_tensor_free(a); if(b)coli_cuda_tensor_free(b); if(ct)coli_cuda_tensor_free(ct);} else pthread_mutex_unlock(&G.mx);- int dv = G.dev[home(eid)];+ int dv = G.dev[home2(layer,eid)];/* passo fra le tre matrici nello staging: int4 impacchettato = mezzo* byte per elemento, int8 = uno. */size_t mb=(size_t)G.D*G.Ih/(G.wfmt==4?2:1);@@ -220,7 +226,7 @@ static void *uploader(void *arg){pthread_mutex_lock(&G.mx);QSlot *s=qs(layer,eid);if(ok){ s->tg=tg; s->tu=tu; s->td=td; s->resident=1; G.uploads++; }- else { int hd=home(eid); G.used[hd]-=G.exp_bytes;+ else { int hd=home2(layer,eid); G.used[hd]-=G.exp_bytes;G.budget[hd]=G.used[hd]; /* device genuinely full: stop trying */ }s->queued=0;G.inflight--;@@ -304,6 +310,10 @@ static int qt_place_named(const char *component){* so the array is n_layers wide and the attention slots stay empty. */#define QT_DN_MAX_LAYERS 128static struct { ColiCudaTensor *t; int dev, on; } G_dnp[QT_DN_MAX_LAYERS];+static int G_home_layer; /* QT_HOME=layer */+static uint8_t G_layer_dev[QT_DN_MAX_LAYERS]; /* device INDEX per layer in that mode */+static int home(int eid){ return eid % G.ndev; } /* the expert-index mapping (tests read it too) */+static int home2(int layer, int eid){ return G_home_layer ? G_layer_dev[layer] : home(eid); }/* ---- automatic placement (COLI_PLACE unset or "auto") ------------------ *//* The hand-written list above is a measurement tool. Nobody running a 6 GB@@ -391,8 +401,7 @@ static double auto_displaced_value(int di, size_t room, int k, size_t exp_bytes,int nl, int ne, int topk, const uint32_t *heat0,double *p_marginal_out){size_t homed = 0;- for(int e = 0; e < ne; e++) if(e % G.ndev == di) homed++;- homed *= (size_t)nl;+ for(int l = 0; l < nl; l++) for(int e = 0; e < ne; e++) if(home2(l, e) == di) homed++;size_t fit = room / exp_bytes;if(fit >= homed){ *p_marginal_out = 0; return 0; } /* room to spare: displaces nothing */if(k <= 0){ *p_marginal_out = 0; return 0; }@@ -409,7 +418,7 @@ static double auto_displaced_value(int di, size_t room, int k, size_t exp_bytes,double sum = 0;for(int e = 0; e < ne; e++) sum += (double)heat0[(size_t)l*ne + e];for(int e = 0; e < ne; e++){- if(e % G.ndev != di) continue;+ if(home2(l, e) != di) continue;double pe = sum > 0 ? (double)topk * heat0[(size_t)l*ne + e] / sum : (double)topk / ne;p[m++] = pe > 1.0 ? 1.0 : pe;}@@ -465,6 +474,9 @@ static void auto_place(int nl, int ne, int topk, const size_t *capacity, const usize_t bytes = G_offer[o].bytes;int di = 0;for(int i = 1; i < G.ndev; i++) if(room[i] > room[di]) di = i;+ /* pipeline homes: a layer's trunk goes where its experts live, lm_head+ * to the last layer's card -- the token then changes card once */+ if(G_home_layer) di = is_lmh ? G_layer_dev[nl-1] : G_layer_dev[G_offer[o].layer < QT_DN_MAX_LAYERS ? G_offer[o].layer : nl-1];/* a layer's out_proj goes where its in_proj went when that card has* the room: with both on one device the whole DeltaNet layer can run* there (qt_dn_gpu_init), which is worth more than balancing bytes */@@ -667,6 +679,27 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,}}if(auto_mode()){+ /* QT_HOME=layer: expert homes per layer range, proportional to the cards'+ * allowances unless QT_LAYER_SPLIT=<n> says how many layers the first card+ * takes (two cards). Layers on one card form a pipeline stage. */+ G_home_layer = 0;+ { const char *hm = getenv("QT_HOME");+ if(hm && !strcmp(hm, "layer") && G.ndev > 1 && nl <= QT_DN_MAX_LAYERS){+ G_home_layer = 1;+ double tot = 0; for(int i = 0; i < G.ndev; i++) tot += (double)capacity[i];+ const char *sp = getenv("QT_LAYER_SPLIT"); int k = (sp && *sp) ? atoi(sp) : -1;+ for(int l = 0; l < nl; l++){+ int di = G.ndev - 1;+ if(k >= 0 && k <= nl && G.ndev == 2) di = l < k ? 0 : 1;+ else { double f = (l + 0.5) / nl, acc = 0; for(int i = 0; i < G.ndev; i++){ acc += tot > 0 ? capacity[i] / tot : 1.0 / G.ndev; if(f < acc){ di = i; break; } } }+ G_layer_dev[l] = (uint8_t)di;+ }+ for(int i = 0; i < G.ndev; i++){+ int lo = -1, hi = -1, n = 0;+ for(int l = 0; l < nl; l++) if(G_layer_dev[l] == i){ if(lo < 0) lo = l; hi = l; n++; }+ fprintf(stderr,"[qtier] homes: layers %d-%d (%d) with their experts and trunk on dev %d\n", lo, hi, n, G.dev[i]);+ }+ } }if(G_offer_n) auto_place(nl, ne, topk, capacity, G.heat0);else { G_auto_on = 1; G_auto_lmh = QT_PLACE_CPU; for(int l=0;l<QT_DN_MAX_LAYERS;l++) G_auto_dnp[l]=QT_PLACE_CPU; }} else {@@ -709,7 +742,10 @@ int qt_init(int nl, int ne, int D, int Ih, int cap, int topk, int expert_gs,* -- including the slow card, whose take() paces every layer (the* measured reason asymmetric expert placement lost). */int ed=qt_place_of("experts",0);- if(ed==QT_PLACE_ALL){+ if(G_home_layer){+ /* layer homes: every card keeps the experts of its own layers,+ * whatever a hand-written list reserved on it */+ } else if(ed==QT_PLACE_ALL){/* experts=all: every COLI_GPUS card keeps its experts, reserved or* not -- the form for "trunk on the fast card, experts on both" */fprintf(stderr,"[place] experts=all -> Experten auf allen %d Karten, auch den reservierten\n",G.ndev);@@ -933,7 +969,7 @@ static int enqueue_locked(int layer,int eid,int v_layer,int v_eid,int reserved){if(G.th_stop) return 0;if(s->resident||s->queued||!s->g4) return 0;if(G.qn>=QT_QCAP){ G.q_full_skips++; return 0; }- int hd=home(eid);+ int hd=home2(layer,eid);if(!reserved && v_eid<0 && G.used[hd]+G.exp_bytes>G.budget[hd]) return 0;size_t mb=(size_t)G.D*G.Ih/(G.wfmt==4?2:1); /* buffer di staging: int8/fp8 = 1 byte/elemento */uint8_t *w=malloc(3*mb); float *sc=malloc((2*G.sc_gu+G.sc_d)*sizeof(float));@@ -963,12 +999,12 @@ static void stream_forget(QSlot *s){ s->g4=s->u4=s->d4=NULL; s->gs=s->us=s->ds=Nstatic void stream_promote_locked(int layer,int eid){QSlot *s=qs(layer,eid);if(s->resident||s->queued) return;- int hd=home(eid);+ int hd=home2(layer,eid);if(G.used[hd]+G.exp_bytes<=G.budget[hd]){ enqueue_locked(layer,eid,-1,-1,0); return; }size_t n=(size_t)G.nl*G.ne; int cold=-1; uint32_t ch=0;for(size_t i=0;i<n;i++){QSlot *c=&G.slot[i];- if(home((int)(i%G.ne))!=hd || !c->resident || c->queued) continue;+ if(home2((int)(i/G.ne),(int)(i%G.ne))!=hd || !c->resident || c->queued) continue;if(cold<0||c->heat<ch){ cold=(int)i; ch=c->heat; }}if(cold<0 || !tier_should_promote(s->heat,ch)) return;@@ -1032,7 +1068,7 @@ int qt_fill_next(int *layer,int *eid){}while((size_t)G.fill_cur<n){int gi=G.fill_order[G.fill_cur];- int l=gi/G.ne, e=gi%G.ne, hd=home(e);+ int l=gi/G.ne, e=gi%G.ne, hd=home2(l,e);QSlot *s=qs(l,e);int full=1; for(int i=0;i<G.ndev;i++) if(G.used[i]+G.exp_bytes<=G.budget[i]) full=0;if(full){ pthread_mutex_unlock(&G.mx); return 0; }@@ -1065,7 +1101,7 @@ int qt_plan_fill(int *layers,int *eids,int max){int full=1; for(int i=0;i<G.ndev;i++) if(G.used[i]+G.exp_bytes<=G.budget[i]) full=0;if(full) break;int gi=G.fill_order[G.fill_cur++];- int l=gi/G.ne, e=gi%G.ne, hd=home(e);+ int l=gi/G.ne, e=gi%G.ne, hd=home2(l,e);QSlot *s=qs(l,e);if(s->resident||s->queued||s->planned) continue;if(G.used[hd]+G.exp_bytes>G.budget[hd]) continue;@@ -1094,7 +1130,7 @@ void qt_note_planned(int layer,int eid,* both keeps the bytes out of the budget for the life of the process* and "if(resident||queued||planned) continue" never reconsiders the* expert. #1331 was this leak for every expert of an int8 container. */- if(s->planned){ G.used[home(eid)]-=G.exp_bytes; s->planned=0; }+ if(s->planned){ G.used[home2(layer,eid)]-=G.exp_bytes; s->planned=0; }pthread_mutex_unlock(&G.mx);return;}@@ -1103,7 +1139,7 @@ void qt_note_planned(int layer,int eid,while(G.qn>=QT_QCAP && !G.th_stop) wait_take_locked();if(!enqueue_locked(layer,eid,-1,-1,1)){/* not enqueueable (e.g. already resident): return the reservation */- if(s->planned) G.used[home(eid)]-=G.exp_bytes;+ if(s->planned) G.used[home2(layer,eid)]-=G.exp_bytes;}s->planned=0;if(G_fp8_stream) stream_forget(s);@@ -1137,7 +1173,7 @@ static void qt_lfru_tick_locked(void){for(size_t i=0;i<n;i++){QSlot *s=&G.slot[i];int e=(int)(i%G.ne);- if(home(e)!=di) continue;+ if(home2((int)(i/G.ne),e)!=di) continue;if(s->resident && !s->queued){ if(cold<0||s->heat<ch){ cold=(int)i; ch=s->heat; } }else if(!s->resident && !s->queued && s->g4){ if(hot<0||s->heat>hh){ hot=(int)i; hh=s->heat; } }}@@ -1173,7 +1209,7 @@ uint32_t qt_issue(int layer,const int *eids,int K,const float *x){for(int k=0;k<K;k++){QSlot *s=qs(layer,eids[k]);if(s->resident){- int di=home(eids[k]); int c=G.is_cnt[di];+ int di=home2(layer,eids[k]); int c=G.is_cnt[di];tg[di][c]=s->tg; tu[di][c]=s->tu; td[di][c]=s->td;G.is_k[di][c]=k; G.is_cnt[di]=c+1;mask|=1u<<k; G.hits[di]++;diff --git a/c/tests/test_qwen36_tier_layer_home.c b/c/tests/test_qwen36_tier_layer_home.cnew file mode 100644index 00000000..42e84b82--- /dev/null+++ b/c/tests/test_qwen36_tier_layer_home.c@@ -0,0 +1,92 @@+/* QT_HOME=layer: expert homes per layer range, the cards as a pipeline.+ *+ * By default expert eid lives on device eid % ndev in every layer, so every+ * layer's group joins across the cards and the slower card paces each layer+ * (measured: take 5.2 ms/token on two unequal cards against 0.2 on one). With+ * QT_HOME=layer every expert of a layer lives on that layer's device, the+ * automatic placer puts the layer's trunk components on the same device and+ * lm_head on the last layer's, and a group is issued to exactly one device.+ * Fake CUDA backend, two fake devices, no GPU. */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>++#include "../compat.h"+#include "qwen36_fake_cuda.h"++#include "../qwen36_tier.c"++static int fails;+static void check(int ok, const char *what) { if (!ok) { printf(" FAIL: %s\n", what); fails++; } }++enum { NL = 8, NE = 8, D = 64, IH = 32, TOPK = 2 };+#define MiB (1024ull * 1024ull)++static int issued_dev[64], issued_n;+static int record_issue(int device, int count, const float *x) { (void)count; (void)x; if (issued_n < 64) issued_dev[issued_n++] = device; return 1; }++int main(void) {+ setenv("COLI_CUDA", "1", 1); setenv("COLI_GPUS", "0,1", 1); setenv("QT_NO_WARMSTART", "1", 1);+ setenv("HEAT_FILE", "", 1); setenv("COLI_PLACE", "", 1); setenv("QT_HOME", "layer", 1); setenv("QT_LAYER_SPLIT", "", 1);+ setenv("CUDA_EXPERT_GB", "0.0625", 1); /* 64 MiB per card */+ fake_ndev = 2; fake_uploads = 0; fake_issue_hook = record_issue;++ /* offers: lmhead once, dnproj and dnout for the DeltaNet layers (every layer but 3 and 7) */+ G_offer_n = 0; G_auto_on = 0;+ qt_trunk_offer("lmhead", 0, 4 * MiB);+ for (int l = 0; l < NL; l++) if (l % 4 != 3) { qt_trunk_offer("dnproj", l, 2 * MiB); qt_trunk_offer("dnout", l, 1 * MiB); }+ check(qt_init(NL, NE, D, IH, NE, TOPK, 0, 1), "tier starts on two fake devices in layer mode");+ check(G_home_layer && G.ndev == 2, "layer homes are on, both cards kept");+ /* equal allowances: layers 0-3 on device index 0, 4-7 on 1 */+ int ok = 1; for (int l = 0; l < NL; l++) if (G_layer_dev[l] != (l < 4 ? 0 : 1)) ok = 0;+ check(ok, "equal allowances split the layers in half");+ ok = 1; for (int l = 0; l < NL; l++) for (int e = 0; e < NE; e++) if (home2(l, e) != G_layer_dev[l]) ok = 0;+ check(ok, "every expert of a layer is homed on that layer's device");+ check(qt_place_of("dnproj", 0) == 0 && qt_place_of("dnout", 0) == 0 && qt_place_of("dnproj", 6) == 1 && qt_place_of("dnout", 6) == 1,+ "the automatic placer puts a layer's trunk on the layer's device");+ check(qt_place_of("lmhead", 0) == 1, "lm_head goes to the last layer's device");+ check(G.budget[0] == 64 * MiB - 9 * MiB && G.budget[1] == 64 * MiB - 13 * MiB, "each card's budget charges its own layers' trunk (3 x 3 MiB; 3 x 3 + 4 MiB)");++ /* residents follow the homes: note every expert, wait, read back */+ static unsigned char g4[NL][NE][D * IH / 2], u4[NL][NE][D * IH / 2], d4[NL][NE][D * IH / 2];+ static float sc[NL][NE][2 * IH + D];+ for (int l = 0; l < NL; l++) for (int e = 0; e < NE; e++) {+ memset(g4[l][e], 1, sizeof g4[l][e]); memset(u4[l][e], 2, sizeof u4[l][e]); memset(d4[l][e], 3, sizeof d4[l][e]);+ for (int i = 0; i < 2 * IH + D; i++) sc[l][e][i] = 1.f;+ qt_note_block(l, e, g4[l][e], u4[l][e], d4[l][e], sc[l][e], sc[l][e] + IH, sc[l][e] + 2 * IH);+ }+ qt_fill_wait();+ int res0 = 0, res1 = 0;+ pthread_mutex_lock(&G.mx);+ for (int l = 0; l < NL; l++) for (int e = 0; e < NE; e++) if (qs(l, e)->resident) { if (G_layer_dev[l] == 0) res0++; else res1++; }+ pthread_mutex_unlock(&G.mx);+ check(res0 == 4 * NE && res1 == 4 * NE, "all 64 experts resident, 32 per card, each on its layer's card");+ check(G.used[0] == 4 * NE * G.exp_bytes && G.used[1] == 4 * NE * G.exp_bytes, "the bytes were charged to the layer's card");++ /* a group goes to one device: layer 0 to the first card, layer 7 to the second, mixed eids notwithstanding */+ float x[D]; for (int i = 0; i < D; i++) x[i] = 0.5f;+ int eids[TOPK] = { 1, 6 }; const float val[TOPK] = { 0.5f, 0.5f }; float out[D];+ issued_n = 0;+ uint32_t m0 = qt_issue(0, eids, TOPK, x); qt_take(m0, val, TOPK, out);+ uint32_t m7 = qt_issue(7, eids, TOPK, x); qt_take(m7, val, TOPK, out);+ check(m0 == 3u && m7 == 3u, "both experts of each group hit VRAM");+ check(issued_n == 2 && issued_dev[0] == 0 && issued_dev[1] == 1, "layer 0's group went to device 0 alone, layer 7's to device 1 alone");+ qt_shutdown();++ /* QT_LAYER_SPLIT overrides the proportional split */+ setenv("QT_LAYER_SPLIT", "6", 1); G_offer_n = 0; G_auto_on = 0; fake_uploads = 0;+ check(qt_init(NL, NE, D, IH, NE, TOPK, 0, 1), "tier starts with an explicit split");+ ok = 1; for (int l = 0; l < NL; l++) if (G_layer_dev[l] != (l < 6 ? 0 : 1)) ok = 0;+ check(ok, "QT_LAYER_SPLIT=6: layers 0-5 on the first card, 6-7 on the second");+ qt_shutdown();++ /* one card: layer mode is a no-op, expert homes as before */+ setenv("COLI_GPUS", "0", 1); fake_ndev = 1; G_offer_n = 0; G_auto_on = 0;+ check(qt_init(NL, NE, D, IH, NE, TOPK, 0, 1), "tier starts on one card");+ check(!G_home_layer && home2(3, 5) == 0, "one card: no layer homes needed");+ qt_shutdown();++ if (fails) { printf("test_qwen36_tier_layer_home: %d failure(s)\n", fails); return 1; }+ printf("OK test_qwen36_tier_layer_home: experts and trunk homed per layer range, one device per group\n");+ return 0;+}