Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen38: the BF16 and F32 resident rows on Vulkan (fmt 11, fmt 10)

JustVugg7461fc3f7445
diff --git a/c/qwen38_core.h b/c/qwen38_core.h
index 6a49639d..31a63c05 100644
--- a/c/qwen38_core.h
+++ b/c/qwen38_core.h
@@ -56,7 +56,8 @@ typedef struct {
unsigned owns_data:1, owns_scales:1;
int gpu; /* 0 = CPU; else 1 + tier handle of an int8 copy resident in VRAM (decode, S == 1) */
int8_t *q8; float *q8sc; /* the trunk's int8 rows on the CPU (default; Q38_TRUNK_CPU_INT8=0 keeps BF16): the same rows the GPU holds, met by an int8 activation in idot.h */
- void *vk; int vk_off; /* COLI_VULKAN=1: device copy of q8 (uploaded at the first matmul), vk_off = upload failed, stays on the CPU */
+ void *vk; int vk_off; /* COLI_VULKAN=1: device copy of q8, or of the BF16/F32 rows (uploaded at the first matmul), vk_off = upload failed, stays on the CPU */
+ int vk_res; /* COLI_VULKAN=1: a resident matrix (q38_load_weight), never an expert slot that is refilled in place */
} Q38Weight;
typedef struct { float *norm; Q38Weight down, up, inject; } GatedResidual;
@@ -355,14 +356,24 @@ static void q38_matmul_fp8(float *y,const float *x,const uint8_t *q8,const float
}
#ifdef COLI_VULKAN
-/* COLI_VULKAN=1: a trunk matrix (q8 present: the int8 rows with one scale per
- * row the CPU reads, fmt 1 for the shader) answers from the Vulkan device,
- * decode and prefill alike. The device copy lives in weight->vk from the first
- * call; a failed upload sets vk_off and the matrix stays on the CPU. The
- * activation stays f32 there, where the CPU's integer kernel rounds it to int8:
- * same weights, the activation not rounded. BF16 rows (Q38_TRUNK_CPU_INT8=0, or
- * matrices under Q38_TRUNK_MIN_KB) and the routed experts have no q8 and never
- * come here. The backend has one command buffer: never from a parallel region. */
+/* COLI_VULKAN=1: a resident matrix answers from the Vulkan device, decode and
+ * prefill alike, with the weights the CPU would read and in the order
+ * q38_weight_matmul picks them: the trunk's int8 rows when there are (q8, one
+ * scale per row, fmt 1), else the rows as loaded, BF16 (fmt 11) or F32
+ * (fmt 10, Q38_NATIVE_BF16=0), times the f32 activation like
+ * q38_matmul_bf16 / q38_matmul. For the int8 rows the activation stays f32
+ * on the device where the CPU's integer kernel rounds it to int8; the float
+ * rows differ from the CPU only in the order of the sums. Only matrices
+ * loaded by q38_load_weight (vk_res) qualify: the routed experts' slots are
+ * refilled in place, so a copy cached in them would go stale, and they stay
+ * on the CPU in every format. The device copy lives in weight->vk from the
+ * first call; a failed upload sets vk_off and the matrix stays on the CPU.
+ * The backend has one command buffer: never from a parallel region. */
+static int q38_vk_eligible(const Q38Weight *w) {
+ return w->q8 || (w->vk_res && w->data &&
+ (w->kind==Q38_WEIGHT_BF16 || w->kind==Q38_WEIGHT_F32));
+}
+static unsigned g_q38_vk_placed[3]; /* uploads by format: int8 rows, bf16, f32 */
static int q38_vk_matmul(float *y,const float *x,const Q38Weight *weight,int S,int I,int O) {
#ifdef _OPENMP
if(omp_in_parallel())return 0;
@@ -370,8 +381,14 @@ static int q38_vk_matmul(float *y,const float *x,const Q38Weight *weight,int S,i
if(weight->vk_off||S<1||S>65535)return 0;
Q38Weight *w=(Q38Weight*)weight; /* vk is a cache in a weight the forward pass treats as read-only */
ColiVkTensor **t=(ColiVkTensor**)&w->vk;
- if(!*t&&!coli_vk_tensor_ensure(t,w->q8,w->q8sc,1,I,O,0)){w->vk_off=1;return 0;}
- return coli_vk_matmul(t,y,x,w->q8,w->q8sc,1,S,I,O,0);
+ int fmt=w->q8?1:w->kind==Q38_WEIGHT_BF16?11:10;
+ const void *wq=w->q8?(const void*)w->q8:(const void*)w->data;
+ const float *sc=w->q8?w->q8sc:NULL; /* fmt 10/11: no scales */
+ if(!*t){
+ if(!coli_vk_tensor_ensure(t,wq,sc,fmt,I,O,0)){w->vk_off=1;return 0;}
+ g_q38_vk_placed[fmt==1?0:fmt==11?1:2]++;
+ }
+ return coli_vk_matmul(t,y,x,wq,sc,fmt,S,I,O,0);
}
/* One line at the end of a run or a serve turn: how many matmuls the device
* really answered, so a test can tell a used path from an initialised one. */
@@ -379,8 +396,9 @@ static void q38_vk_report(void) {
if(!g_vk_ready)return;
size_t bytes=0,tensors=0;
coli_vk_mem_info(&bytes,&tensors);
- fprintf(stderr,"[VK] qwen38: %llu matmuls on the GPU (%zu matrices resident, %.1f MiB)\n",
- coli_vk_matmul_calls(),tensors,bytes/1048576.0);
+ fprintf(stderr,"[VK] qwen38: %llu matmuls on the GPU (%zu matrices resident, %.1f MiB; placed int8 %u, bf16 %u, f32 %u)\n",
+ coli_vk_matmul_calls(),tensors,bytes/1048576.0,
+ g_q38_vk_placed[0],g_q38_vk_placed[1],g_q38_vk_placed[2]);
}
#endif
static void q38_weight_matmul(float *y,const float *x,const Q38Weight *weight,
@@ -391,7 +409,7 @@ static void q38_weight_matmul(float *y,const float *x,const Q38Weight *weight,
if(S==1&&weight&&weight->gpu&&weight->rows==O&&weight->cols==I&&
qt_dense_matmul(weight->gpu-1,y,x,I,O))return;
#ifdef COLI_VULKAN
- if(g_vk_ready&&weight&&weight->q8&&weight->rows==O&&weight->cols==I&&
+ if(g_vk_ready&&weight&&weight->rows==O&&weight->cols==I&&q38_vk_eligible(weight)&&
q38_vk_matmul(y,x,weight,S,I,O))return;
#endif
if(weight&&weight->q8&&weight->rows==O&&weight->cols==I){
@@ -750,6 +768,9 @@ static Q38Weight q38_load_weight(Model *m,const char *name,int rows,int cols) {
st_read_f32(&m->S,name,(float*)weight.data,1);
}
m->resident_weight_bytes+=q38_weight_bytes(&weight);
+#ifdef COLI_VULKAN
+ weight.vk_res=1; /* resident for the life of the model: its rows may live on the device */
+#endif
return weight;
}