Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(qwen36): keep CUDA tier heat adaptive
ZacharyZcRfc31c8ae00e7
diff --git a/c/qwen36_tier.c b/c/qwen36_tier.cindex 29963316..89d60c46 100644--- a/c/qwen36_tier.c+++ b/c/qwen36_tier.c@@ -6,6 +6,7 @@#include <pthread.h>#include "qwen36_tier.h"#include "backend_cuda.h"+#include "tier.h"#define QT_MAX_DEV 8#define QT_QCAP 48 /* upload queue depth (staging ~1.6 MB/entry) */@@ -330,11 +331,15 @@ void qt_fill_wait(void){pthread_mutex_unlock(&G.mx);}-/* LFRU swap check (every 16 ticks = tokens): per device, coldest resident vs- * hottest non-resident, with the tier.h hysteresis. */+/* Adaptive swap check (every 16 ticks = tokens): per device, coldest resident+ * vs hottest non-resident. Decay every 1024 ticks so an old workload cannot+ * permanently own the tier; admission uses the shared tier.h contract. */static void qt_lfru_tick_locked(void){- if(++G.tick % 16) return;size_t n=(size_t)G.nl*G.ne;+ G.tick++;+ if(!(G.tick%1024))+ for(size_t i=0;i<n;i++) G.slot[i].heat=tier_decay_value(G.slot[i].heat);+ if(G.tick%16) return;for(int di=0;di<G.ndev;di++){int cold=-1, hot=-1; uint32_t ch=0, hh=0;for(size_t i=0;i<n;i++){@@ -345,7 +350,7 @@ static void qt_lfru_tick_locked(void){else if(!s->resident && !s->queued && s->g4){ if(hot<0||s->heat>hh){ hot=(int)i; hh=s->heat; } }}if(cold<0||hot<0) continue;- if(hh<=ch+(ch>>2)+4) continue; /* hysteresis as in tier.h */+ if(!tier_should_promote(hh,ch)) continue;QSlot *v=&G.slot[cold];v->resident=0; /* CPU fallback from now on */if(enqueue_locked(hot/G.ne,hot%G.ne,cold/G.ne,cold%G.ne,0)) G.swaps++;diff --git a/c/tests/test_tier.c b/c/tests/test_tier.cindex eb6f1ac9..0f43c849 100644--- a/c/tests/test_tier.c+++ b/c/tests/test_tier.c@@ -7,6 +7,11 @@ static int fail(const char *message){}int main(void){+ if(!tier_should_promote(130,100)) return fail("shared hysteresis admits hot expert");+ if(tier_should_promote(129,100)) return fail("shared hysteresis blocks marginal expert");+ if(tier_should_promote(UINT32_MAX-1,UINT32_MAX))+ return fail("saturated hysteresis threshold wrapped");+ if(tier_decay_value(9)!=4) return fail("shared heat decay");uint32_t heat[6]={20,2,8,3,30,1};int pinned[2]={0,1}, slot=-1, eid=-1; long gain=0;if(!tier_pick_swap(heat,6,pinned,2,&slot,&eid,&gain)) return fail("hot expert not promoted");diff --git a/c/tier.h b/c/tier.hindex 859cacf6..01fa4b80 100644--- a/c/tier.h+++ b/c/tier.h@@ -3,6 +3,16 @@#include <stdint.h>+/* Shared admission contract for every adaptive resident tier. Widen before+ * adding the margin: a saturated uint32 heat counter must become sticky, not+ * wrap the threshold and admit a colder expert. */+static int tier_should_promote(uint32_t hot, uint32_t cold){+ uint64_t threshold=(uint64_t)cold+((uint64_t)cold>>2)+4u;+ return (uint64_t)hot>threshold;+}++static uint32_t tier_decay_value(uint32_t heat){ return heat>>1; }+/* Pick one RAM/VRAM hot-store slot to replace from recent routing heat.* The fixed margin handles tiny samples; the 25% margin prevents ping-pong. */static int tier_pick_swap(const uint32_t *heat, int nexpert,@@ -19,7 +29,7 @@ static int tier_pick_swap(const uint32_t *heat, int nexpert,}if(hot<0) return 0;uint32_t fc=heat[pinned[cold]];- if(fh<=fc+(fc>>2)+4) return 0;+ if(!tier_should_promote(fh,fc)) return 0;*slot=cold; *eid=hot; *gain=(long)fh-(long)fc;return 1;}@@ -54,7 +64,7 @@ static int tier_pick_lfru(const uint32_t *heat, const uint32_t *last, uint32_t c}static void tier_decay(uint32_t *heat, int nexpert){- for(int e=0;e<nexpert;e++) heat[e]>>=1;+ for(int e=0;e<nexpert;e++) heat[e]=tier_decay_value(heat[e]);}#endifdiff --git a/docs/qwen36-cuda-tier.md b/docs/qwen36-cuda-tier.mdindex 21684a6c..0941c7a7 100644--- a/docs/qwen36-cuda-tier.md+++ b/docs/qwen36-cuda-tier.md@@ -10,7 +10,9 @@ more GPUs and computed there through the existing shared CUDA backend- **Home device:** expert `eid` lives on GPU `eid % n_gpus`; no duplicates.- **Placement:** routing heat decides who earns VRAM (LFRU semantics from- `tier.h`, 25%+4 hysteresis). A parallel **warmstart** fills the per-device+ `tier.h`, 25%+4 hysteresis). Runtime heat is halved every 1024 decode ticks,+ so a long-lived process can replace experts from an old workload instead of+ permanently freezing its initial hot set. A parallel **warmstart** fills the per-devicebudget before the first token — ordered by a persisted heat table(`HEAT_FILE`) when present, so a second run starts fully placed.- **Decode:** per (token, layer) the resident experts are issued as async