Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vk-int3 hardening from first real-container runs

Steve Markgraf4222bf11a2bf
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index 8c0cd5ed..c3fa9ac9 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -1325,33 +1325,37 @@ int main(int argc, char **argv) {
bad |= run_expert_group(2, 6144, 2048, 1);
bad |= run_expert_group(2, 6144, 2048, 8);
bad |= run_expert_group(2, 6144, 2048, 32);
- /* int3-g64 expert group: correctness + the 0.86x-bytes throughput question */
+ /* int3-g64 expert group: correctness + the 0.86x-bytes throughput question.
+ * count=1 included — it is the SHARED-expert path shape in the engine. */
+ bad |= run_expert_group(5, 6144, 2048, 1);
bad |= run_expert_group(5, 6144, 2048, 8);
bad |= run_expert_group(5, 6144, 2048, 32);
- /* Fused same-input matmul pair (the q_a + kv_a prologue pattern). */
- {
+ /* Fused same-input matmul pair (the q_a + kv_a prologue pattern), int4 AND int3. */
+ for (int pi = 0; pi < 2; pi++) {
+ int pf = pi == 0 ? 2 : 5;
int I = 6144, O1 = 2048, O2 = 576, S = 1;
- size_t rb = (size_t)(I + 1) / 2;
+ size_t rb = ref_rowbytes(pf, I);
+ size_t n1 = ref_scales(pf, I, O1), n2 = ref_scales(pf, I, O2);
uint8_t *w1 = malloc(rb * O1), *w2 = malloc(rb * O2);
- float *s1 = malloc((size_t)O1 * 4), *s2 = malloc((size_t)O2 * 4), *x = malloc((size_t)I * 4);
+ float *s1 = malloc(n1 * 4), *s2 = malloc(n2 * 4), *x = malloc((size_t)I * 4);
float *y1 = malloc((size_t)O1 * 4), *y2 = malloc((size_t)O2 * 4);
float *c1 = malloc((size_t)O1 * 4), *c2 = malloc((size_t)O2 * 4);
for (size_t i = 0; i < rb * (size_t)O1; i++) w1[i] = rand() & 0xff;
for (size_t i = 0; i < rb * (size_t)O2; i++) w2[i] = rand() & 0xff;
- for (int o = 0; o < O1; o++) s1[o] = 0.01f + (rand() % 100) / 10000.0f;
- for (int o = 0; o < O2; o++) s2[o] = 0.01f + (rand() % 100) / 10000.0f;
+ for (size_t o = 0; o < n1; o++) s1[o] = 0.01f + (rand() % 100) / 10000.0f;
+ for (size_t o = 0; o < n2; o++) s2[o] = 0.01f + (rand() % 100) / 10000.0f;
for (int i = 0; i < I; i++) x[i] = (rand() % 200 - 100) / 100.0f;
ColiVkTensor *t1 = NULL, *t2 = NULL;
- if (!coli_vk_matmul_pair(&t1, y1, w1, s1, O1, &t2, y2, w2, s2, O2, 2, x, S, I)) {
- printf("matmul_pair failed\n"); bad = 1;
+ if (!coli_vk_matmul_pair(&t1, y1, w1, s1, O1, &t2, y2, w2, s2, O2, pf, x, S, I)) {
+ printf("matmul_pair fmt=%d failed\n", pf); bad = 1;
} else {
- cpu_ref(c1, x, w1, s1, 2, S, I, O1); cpu_ref(c2, x, w2, s2, 2, S, I, O2);
+ cpu_ref(c1, x, w1, s1, pf, S, I, O1); cpu_ref(c2, x, w2, s2, pf, S, I, O2);
double mr = 0;
for (int i = 0; i < O1; i++) { double e = fabs(y1[i]-c1[i]); if (fabs(c1[i])>1e-2) { double r=e/fabs(c1[i]); if (r>mr) mr=r; } }
for (int i = 0; i < O2; i++) { double e = fabs(y2[i]-c2[i]); if (fabs(c2[i])>1e-2) { double r=e/fabs(c2[i]); if (r>mr) mr=r; } }
double t0 = now();
- for (int k = 0; k < 30; k++) coli_vk_matmul_pair(&t1, y1, w1, s1, O1, &t2, y2, w2, s2, O2, 2, x, S, I);
- printf("matmul_pair 6144->(2048,576) | maxrel=%.4g | %.3f ms/pair-call\n", mr, (now()-t0)*1000/30);
+ for (int k = 0; k < 30; k++) coli_vk_matmul_pair(&t1, y1, w1, s1, O1, &t2, y2, w2, s2, O2, pf, x, S, I);
+ printf("matmul_pair fmt=%d 6144->(2048,576) | maxrel=%.4g | %.3f ms/pair-call\n", pf, mr, (now()-t0)*1000/30);
bad |= mr > 1e-3;
}
coli_vk_tensor_free(t1); coli_vk_tensor_free(t2);
diff --git a/c/colibri.c b/c/colibri.c
index 3a81493f..93533c1e 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -3631,7 +3631,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
}
}
#ifdef COLI_VULKAN
- /* Vulkan expert tier (COLI_VULKAN=1): upload routed int4 experts to the GPU once
+ /* Vulkan expert tier (COLI_VULKAN=1): upload routed int4/int3-g64 experts to the GPU once
* (capped by COLI_VK_EXPERTS), then compute the resident ones as one batched
* coli_vk_expert_group (fused gate+up+silu -> down, on-device); the rest + misses
* fall back to the CPU below. Decode-only (S<=4). Measured ~35% faster than ROCm. */
@@ -7398,7 +7398,7 @@ int main(int argc, char **argv){
g_vk_budget = getenv("COLI_VK_EXPERTS") ? atoi(getenv("COLI_VK_EXPERTS")) : 320;
g_vk_dense = getenv("COLI_VK_DENSE") ? atoi(getenv("COLI_VK_DENSE")) : 0;
g_vk_attn = getenv("COLI_VK_ATTN") ? atoi(getenv("COLI_VK_ATTN")) : 0;
- fprintf(stderr,"[VK] expert tier active: routed int4 experts on the GPU (budget %d)%s%s\n",
+ fprintf(stderr,"[VK] expert tier active: routed quantized experts on the GPU (budget %d)%s%s\n",
g_vk_budget, g_vk_dense ? " + dense projections + shared expert" : "",
g_vk_attn ? " + absorb attention core" : "");
}
diff --git a/c/sample.h b/c/sample.h
index 3ca2c09f..c1187cf3 100644
--- a/c/sample.h
+++ b/c/sample.h
@@ -97,6 +97,23 @@ static int dist_sample(int V, int ban){
/* next token from logits: greedy if g_temp<=0, sampling otherwise.
* ban = token excluded because it was rejected by speculative verification. */
static int pick_tok(const float *lo, int V, int ban){
+ /* COLI_LOGIT_DUMP=1: top-5 (id:logit) per step to stderr — for comparing two
+ * engine configs on identical forced context (backend-exactness triage). */
+ static int dump = -1;
+ if (dump < 0) dump = getenv("COLI_LOGIT_DUMP") ? 1 : 0;
+ if (dump){
+ int id[5]={-1,-1,-1,-1,-1}; float v[5]={-3e38f,-3e38f,-3e38f,-3e38f,-3e38f};
+ for (int t = 0; t < V; t++){
+ float x = lo[t];
+ for (int k = 0; k < 5; k++) if (x > v[k]){
+ for (int j = 4; j > k; j--){ v[j]=v[j-1]; id[j]=id[j-1]; }
+ v[k]=x; id[k]=t; break;
+ }
+ }
+ fprintf(stderr,"[LOGITS]");
+ for (int k = 0; k < 5; k++) fprintf(stderr," %d:%.6f", id[k], v[k]);
+ fprintf(stderr,"\n");
+ }
if (g_temp <= 0) return argmax_v(lo, V);
dist_build(lo, V);
return dist_sample(V, ban);