Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(qwen36): keep CUDA tier heat adaptive

ZacharyZcRfc31c8ae00e7
diff --git a/c/qwen36_tier.c b/c/qwen36_tier.c
index 29963316..89d60c46 100644
--- a/c/qwen36_tier.c
+++ b/c/qwen36_tier.c
@@ -6,6 +6,7 @@
#include <pthread.h>
#include "qwen36_tier.h"
#include "backend_cuda.h"
+#include "tier.h"
#define QT_MAX_DEV 8
#define QT_QCAP 48 /* upload queue depth (staging ~1.6 MB/entry) */
@@ -330,11 +331,15 @@ void qt_fill_wait(void){
pthread_mutex_unlock(&G.mx);
}
-/* LFRU swap check (every 16 ticks = tokens): per device, coldest resident vs
- * hottest non-resident, with the tier.h hysteresis. */
+/* Adaptive swap check (every 16 ticks = tokens): per device, coldest resident
+ * vs hottest non-resident. Decay every 1024 ticks so an old workload cannot
+ * permanently own the tier; admission uses the shared tier.h contract. */
static void qt_lfru_tick_locked(void){
- if(++G.tick % 16) return;
size_t n=(size_t)G.nl*G.ne;
+ G.tick++;
+ if(!(G.tick%1024))
+ for(size_t i=0;i<n;i++) G.slot[i].heat=tier_decay_value(G.slot[i].heat);
+ if(G.tick%16) return;
for(int di=0;di<G.ndev;di++){
int cold=-1, hot=-1; uint32_t ch=0, hh=0;
for(size_t i=0;i<n;i++){
@@ -345,7 +350,7 @@ static void qt_lfru_tick_locked(void){
else if(!s->resident && !s->queued && s->g4){ if(hot<0||s->heat>hh){ hot=(int)i; hh=s->heat; } }
}
if(cold<0||hot<0) continue;
- if(hh<=ch+(ch>>2)+4) continue; /* hysteresis as in tier.h */
+ if(!tier_should_promote(hh,ch)) continue;
QSlot *v=&G.slot[cold];
v->resident=0; /* CPU fallback from now on */
if(enqueue_locked(hot/G.ne,hot%G.ne,cold/G.ne,cold%G.ne,0)) G.swaps++;
diff --git a/c/tests/test_tier.c b/c/tests/test_tier.c
index eb6f1ac9..0f43c849 100644
--- a/c/tests/test_tier.c
+++ b/c/tests/test_tier.c
@@ -7,6 +7,11 @@ static int fail(const char *message){
}
int main(void){
+ if(!tier_should_promote(130,100)) return fail("shared hysteresis admits hot expert");
+ if(tier_should_promote(129,100)) return fail("shared hysteresis blocks marginal expert");
+ if(tier_should_promote(UINT32_MAX-1,UINT32_MAX))
+ return fail("saturated hysteresis threshold wrapped");
+ if(tier_decay_value(9)!=4) return fail("shared heat decay");
uint32_t heat[6]={20,2,8,3,30,1};
int pinned[2]={0,1}, slot=-1, eid=-1; long gain=0;
if(!tier_pick_swap(heat,6,pinned,2,&slot,&eid,&gain)) return fail("hot expert not promoted");
diff --git a/c/tier.h b/c/tier.h
index 859cacf6..01fa4b80 100644
--- a/c/tier.h
+++ b/c/tier.h
@@ -3,6 +3,16 @@
#include <stdint.h>
+/* Shared admission contract for every adaptive resident tier. Widen before
+ * adding the margin: a saturated uint32 heat counter must become sticky, not
+ * wrap the threshold and admit a colder expert. */
+static int tier_should_promote(uint32_t hot, uint32_t cold){
+ uint64_t threshold=(uint64_t)cold+((uint64_t)cold>>2)+4u;
+ return (uint64_t)hot>threshold;
+}
+
+static uint32_t tier_decay_value(uint32_t heat){ return heat>>1; }
+
/* Pick one RAM/VRAM hot-store slot to replace from recent routing heat.
* The fixed margin handles tiny samples; the 25% margin prevents ping-pong. */
static int tier_pick_swap(const uint32_t *heat, int nexpert,
@@ -19,7 +29,7 @@ static int tier_pick_swap(const uint32_t *heat, int nexpert,
}
if(hot<0) return 0;
uint32_t fc=heat[pinned[cold]];
- if(fh<=fc+(fc>>2)+4) return 0;
+ if(!tier_should_promote(fh,fc)) return 0;
*slot=cold; *eid=hot; *gain=(long)fh-(long)fc;
return 1;
}
@@ -54,7 +64,7 @@ static int tier_pick_lfru(const uint32_t *heat, const uint32_t *last, uint32_t c
}
static void tier_decay(uint32_t *heat, int nexpert){
- for(int e=0;e<nexpert;e++) heat[e]>>=1;
+ for(int e=0;e<nexpert;e++) heat[e]=tier_decay_value(heat[e]);
}
#endif
diff --git a/docs/qwen36-cuda-tier.md b/docs/qwen36-cuda-tier.md
index 21684a6c..0941c7a7 100644
--- a/docs/qwen36-cuda-tier.md
+++ b/docs/qwen36-cuda-tier.md
@@ -10,7 +10,9 @@ more GPUs and computed there through the existing shared CUDA backend
- **Home device:** expert `eid` lives on GPU `eid % n_gpus`; no duplicates.
- **Placement:** routing heat decides who earns VRAM (LFRU semantics from
- `tier.h`, 25%+4 hysteresis). A parallel **warmstart** fills the per-device
+ `tier.h`, 25%+4 hysteresis). Runtime heat is halved every 1024 decode ticks,
+ so a long-lived process can replace experts from an old workload instead of
+ permanently freezing its initial hot set. A parallel **warmstart** fills the per-device
budget before the first token — ordered by a persisted heat table
(`HEAT_FILE`) when present, so a second run starts fully placed.
- **Decode:** per (token, layer) the resident experts are issued as async