Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda+engine: full fmt=4 (grouped int4 gs=64) support + diagnostic harness

woolcoxm6e29260635f1
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index a0083f40..30a1f4a4 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -21,6 +21,7 @@ struct ColiCudaTensor {
size_t weight_bytes;
int fmt, I, O, device;
int gs; /* quant group size; 0 = per-row scales (#334) */
+ int ng; /* number of scale groups per row = ceil(I/gs) for fmt=4 */
size_t scale_count; /* floats in `scales`: O per-row, O*ng grouped */
int tracked;
RaggedKVEntry ragged[512];
@@ -85,7 +86,7 @@ static int select_ctx(DeviceContext *ctx) {
__host__ __device__ static size_t row_bytes(int fmt, int I) {
if (fmt == 0) return (size_t)I * sizeof(float);
if (fmt == 1) return (size_t)I;
- if (fmt == 2) return (size_t)(I + 1) / 2;
+ if (fmt == 2 || fmt == 4) return (size_t)(I + 1) / 2; /* fmt=4: same packed int4 */
if (fmt == 3) return (size_t)(I + 3) / 4;
if (fmt == 4) return (size_t)(I + 1) / 2; /* grouped int4: nibbles like fmt 2 */
return 0;
@@ -96,7 +97,7 @@ __device__ static float weight_at(const void *weights, int fmt, size_t row, int
if (fmt == 0) return reinterpret_cast<const float *>(base)[i];
if (fmt == 1) return static_cast<float>(reinterpret_cast<const int8_t *>(base)[i]);
const uint8_t *q = base;
- if (fmt == 2) {
+ if (fmt == 2 || fmt == 4) { /* fmt=4: same nibble layout */
uint8_t v = q[i >> 1];
int n=(i&1)?(v>>4):(v&15); return static_cast<float>(n&8?n-16:n);
}
@@ -110,14 +111,27 @@ __global__ static void offset_to_signed_s4(uint8_t *q,size_t n){
__global__ static void quant_matmul(float *y, const float *x, const void *weights,
const float *scales, int fmt, int S, int I, int O,
- size_t rb) {
+ size_t rb, int gs, int ng) {
int o = blockIdx.x;
int s = blockIdx.y;
float sum = 0.0f;
size_t row = (size_t)o * rb;
const float *xs = x + (size_t)s * I;
- for (int i = threadIdx.x; i < I; i += blockDim.x)
- sum += xs[i] * weight_at(weights, fmt, row, i);
+ if (fmt == 4) {
+ /* Grouped int4: one f32 scale per gs elements along I (ng groups per row).
+ * Scale layout: scales[o*ng + g]. Each thread strides through I, applying
+ * the appropriate group scale as it crosses group boundaries. This matches
+ * the CPU matmul_i4_grouped accumulation exactly. */
+ const float *scl = scales + (size_t)o * ng;
+ for (int i = threadIdx.x; i < I; i += blockDim.x) {
+ int g = i / gs;
+ if (g >= ng) g = ng - 1; /* tail elements in the last (partial) group */
+ sum += xs[i] * weight_at(weights, fmt, row, i) * scl[g];
+ }
+ } else {
+ for (int i = threadIdx.x; i < I; i += blockDim.x)
+ sum += xs[i] * weight_at(weights, fmt, row, i);
+ }
__shared__ float partial[256];
partial[threadIdx.x] = sum;
@@ -127,7 +141,7 @@ __global__ static void quant_matmul(float *y, const float *x, const void *weight
__syncthreads();
}
if (!threadIdx.x)
- y[(size_t)s * O + o] = partial[0] * (fmt ? scales[o] : 1.0f);
+ y[(size_t)s * O + o] = (fmt && fmt != 4) ? partial[0] * scales[o] : partial[0];
}
__global__ static void silu_mul(float *gate, const float *up, size_t n) {
@@ -564,7 +578,7 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
* on such a slot failed here β€” the GPU tier silently never computed
* for host-released slab experts. */
ColiCudaTensor *t = *tensor;
- return t->fmt == fmt && t->I == I && t->O == O && t->device == device;
+ return t->fmt == fmt && t->I == I && t->O == O && t->device == device && t->gs == gs;
}
DeviceContext *ctx = find_ctx(device);
if (!weights || I < 1 || O < 1 || !select_ctx(ctx)) return 0;
@@ -574,7 +588,8 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
if (!t) return 0;
t->fmt = fmt; t->I = I; t->O = O; t->device = device; t->weight_bytes = rb * (size_t)O;
t->gs = (fmt==4 && g_upload_gs>0) ? g_upload_gs : 0;
- t->scale_count = t->gs ? (size_t)O * (size_t)((I + t->gs - 1) / t->gs) : (size_t)O;
+ t->ng = t->gs ? (I + t->gs - 1) / t->gs : 1;
+ t->scale_count = t->gs ? (size_t)O * (size_t)t->ng : (size_t)O;
if (!cuda_ok(cudaMalloc(&t->weights, t->weight_bytes), "tensor allocation") ||
!cuda_ok(cudaMemcpy(t->weights, weights, t->weight_bytes, cudaMemcpyHostToDevice), "tensor upload")) {
coli_cuda_tensor_free(t);
@@ -618,7 +633,9 @@ extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor,
(uint8_t*)tensor->weights,tensor->weight_bytes);
if(!cuda_ok(cudaGetLastError(),"int4 weight refresh")) return 0;
}
+ int ng = tensor->ng > 0 ? tensor->ng : 1;
return !tensor->fmt || cuda_ok(cudaMemcpy(tensor->scales,scales,
+<<<<<<< HEAD
(tensor->scale_count?tensor->scale_count:(size_t)tensor->O)*sizeof(float),
cudaMemcpyHostToDevice),"scale refresh");
}
@@ -631,14 +648,22 @@ static long g_gpu_calls;
static int fault_injected(void) {
const char *fa = std::getenv("COLI_GPU_FAIL_AFTER");
return fa && g_gpu_calls++ >= std::atol(fa);
+=======
+ (size_t)tensor->O*ng*sizeof(float),cudaMemcpyHostToDevice),"scale refresh");
+>>>>>>> 1a69113 (cuda+engine: full fmt=4 (grouped int4 gs=64) support + diagnostic harness)
}
extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor,
float *y, const float *x,
const void *weights, const float *scales,
+<<<<<<< HEAD
int fmt, int S, int I, int O, int device) {
if (fault_injected()) return 0;
if (S < 1 || !coli_cuda_tensor_upload(tensor, weights, scales, fmt, I, O, device)) return 0;
+=======
+ int fmt, int S, int I, int O, int device, int gs) {
+ if (S < 1 || !coli_cuda_tensor_upload(tensor, weights, scales, fmt, I, O, device, gs)) return 0;
+>>>>>>> 1a69113 (cuda+engine: full fmt=4 (grouped int4 gs=64) support + diagnostic harness)
ColiCudaTensor *t = *tensor;
DeviceContext *ctx = find_ctx(t->device);
if (!select_ctx(ctx)) return 0;
@@ -647,7 +672,7 @@ extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor,
if (!reserve(&ctx->x, &ctx->x_cap, xb) || !reserve(&ctx->y, &ctx->y_cap, yb)) return 0;
if (!cuda_ok(cudaMemcpy(ctx->x, x, xb, cudaMemcpyHostToDevice), "input upload")) return 0;
dim3 grid((unsigned)O, (unsigned)S);
- quant_matmul<<<grid, 256>>>(ctx->y, ctx->x, t->weights, t->scales, fmt, S, I, O, rb);
+ quant_matmul<<<grid, 256>>>(ctx->y, ctx->x, t->weights, t->scales, fmt, S, I, O, rb, t->gs, t->ng);
if (!cuda_ok(cudaGetLastError(), "matmul launch") ||
!cuda_ok(cudaMemcpy(y, ctx->y, yb, cudaMemcpyDeviceToHost), "output download")) return 0;
return 1;
@@ -671,13 +696,13 @@ extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,
if (!cuda_ok(cudaMemcpy(ctx->x,x,xb,cudaMemcpyHostToDevice),"expert input upload")) return 0;
dim3 hidden_grid((unsigned)I,(unsigned)S), output_grid((unsigned)D,(unsigned)S);
quant_matmul<<<hidden_grid,256>>>(ctx->gate,ctx->x,gate->weights,gate->scales,
- gate->fmt,S,D,I,row_bytes(gate->fmt,D));
+ gate->fmt,S,D,I,row_bytes(gate->fmt,D),gate->gs,gate->ng);
quant_matmul<<<hidden_grid,256>>>(ctx->up,ctx->x,up->weights,up->scales,
- up->fmt,S,D,I,row_bytes(up->fmt,D));
+ up->fmt,S,D,I,row_bytes(up->fmt,D),up->gs,up->ng);
size_t n=(size_t)S*I;
silu_mul<<<(unsigned)((n+255)/256),256>>>(ctx->gate,ctx->up,n);
quant_matmul<<<output_grid,256>>>(ctx->y,ctx->gate,down->weights,down->scales,
- down->fmt,S,I,D,row_bytes(down->fmt,I));
+ down->fmt,S,I,D,row_bytes(down->fmt,I),down->gs,down->ng);
if (!cuda_ok(cudaGetLastError(),"expert MLP launch") ||
!cuda_ok(cudaMemcpy(y,ctx->y,yb,cudaMemcpyDeviceToHost),"expert output download")) return 0;
return 1;
@@ -732,9 +757,16 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,
g->fmt,u->fmt,d->fmt,rows[c],total,
g->gs,u->gs,d->gs};
all_s4&=g->fmt==2&&u->fmt==2&&d->fmt==2;
+<<<<<<< HEAD
all_q4&=(g->fmt==2||g->fmt==4)&&(u->fmt==2||u->fmt==4)&&(d->fmt==2||d->fmt==4)&&
!(g->gs&1)&&!(u->gs&1)&&!(d->gs&1); /* even gs: a packed byte never straddles groups */
any_g4|=g->fmt==4||u->fmt==4||d->fmt==4;
+=======
+ /* fmt=4 (grouped int4) experts have per-group scales that the grouped_hidden/
+ * grouped_down kernels don't handle. Fall back to the per-expert path
+ * (coli_cuda_expert_mlp), which uses quant_matmul with correct gs/ng. */
+ if(g->fmt==4||u->fmt==4||d->fmt==4) return 0;
+>>>>>>> 1a69113 (cuda+engine: full fmt=4 (grouped int4 gs=64) support + diagnostic harness)
total+=rows[c]; if(rows[c]>max_rows) max_rows=rows[c];
}
DeviceContext *ctx=find_ctx(device); if(!select_ctx(ctx)) return 0;
@@ -805,12 +837,12 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,
/* piccoli batch: tile TC quasi vuoti + overhead di lancio β€” il
* kernel naive per-elemento resta piu' veloce (misurato in decode) */
quant_matmul<<<dim3((unsigned)I,(unsigned)r),256,0,ctx->stream>>>(g16,x16,
- host[c].g,host[c].gs,host[c].gf,r,D,I,row_bytes(host[c].gf,D));
+ host[c].g,host[c].gs,host[c].gf,r,D,I,row_bytes(host[c].gf,D),0,1);
quant_matmul<<<dim3((unsigned)I,(unsigned)r),256,0,ctx->stream>>>(u16,x16,
- host[c].u,host[c].us,host[c].uf,r,D,I,row_bytes(host[c].uf,D));
+ host[c].u,host[c].us,host[c].uf,r,D,I,row_bytes(host[c].uf,D),0,1);
silu_mul<<<(unsigned)(((size_t)r*I+255)/256),256,0,ctx->stream>>>(g16,u16,(size_t)r*I);
quant_matmul<<<dim3((unsigned)D,(unsigned)r),256,0,ctx->stream>>>(y16,g16,
- host[c].d,host[c].ds,host[c].df,r,I,D,row_bytes(host[c].df,I));
+ host[c].d,host[c].ds,host[c].df,r,I,D,row_bytes(host[c].df,I),0,1);
}
off16+=r;
}
@@ -976,7 +1008,7 @@ static int attention_absorb_batch_run(ColiCudaTensor *w,ColiCudaTensor *proj,flo
if(proj){
ob=(size_t)S*proj->O*sizeof(float);if(!reserve(&dc->y,&dc->y_cap,ob))return 0;
quant_matmul<<<dim3(proj->O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights,
- proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I));
+ proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng);
if(!cuda_ok(cudaGetLastError(),"attention o_proj launch"))return 0;src=dc->y;
}
if(!cuda_ok(cudaMemcpyAsync(out,src,ob,cudaMemcpyDeviceToHost,dc->stream),
@@ -1087,7 +1119,8 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {
DeviceContext *ctx = find_ctx(tensor->device);
if (ctx) select_ctx(ctx);
if (tensor->tracked && ctx) {
- size_t bytes = tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * sizeof(float) : 0);
+ int ng = tensor->ng > 0 ? tensor->ng : 1;
+ size_t bytes = tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * ng * sizeof(float) : 0);
if (ctx->tensor_count) ctx->tensor_count--;
if (ctx->tensor_bytes >= bytes) ctx->tensor_bytes -= bytes;
}
@@ -1101,7 +1134,9 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {
}
extern "C" size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor) {
- return tensor ? tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * sizeof(float) : 0) : 0;
+ if (!tensor) return 0;
+ int ng = tensor->ng > 0 ? tensor->ng : 1;
+ return tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * ng * sizeof(float) : 0);
}
extern "C" int coli_cuda_tensor_device(const ColiCudaTensor *tensor) {
@@ -1320,7 +1355,7 @@ extern "C" int coli_cuda_attention_project_batch_dev(ColiCudaTensor *w,ColiCudaT
size_t ob=(size_t)S*proj->O*sizeof(float);
if(!reserve(&dc->y,&dc->y_cap,ob))return 0;
quant_matmul<<<dim3(proj->O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights,
- proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I));
+ proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng);
if(!cuda_ok(cudaGetLastError(),"pipe o_proj launch"))return 0;
if(!cuda_ok(cudaMemcpyAsync(out,dc->y,ob,cudaMemcpyDeviceToHost,dc->stream),"pipe attention download")||
!cuda_ok(cudaStreamSynchronize(dc->stream),"pipe attention sync"))return 0;
@@ -1355,7 +1390,7 @@ extern "C" int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x
DeviceContext *ctx=find_ctx(t->device); if(!select_ctx(ctx)) return 0;
dim3 grid((unsigned)t->O,(unsigned)S);
quant_matmul<<<grid,256>>>(y_dev,x_dev,t->weights,t->scales,t->fmt,S,t->I,t->O,
- row_bytes(t->fmt,t->I));
+ row_bytes(t->fmt,t->I),t->gs,t->ng);
return cuda_ok(cudaGetLastError(),"pipe gemm");
}
/* copia diretta scheda->scheda (P2P se disponibile, altrimenti staging driver) */
@@ -1382,7 +1417,7 @@ extern "C" int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *w,ColiC
rope_dev,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale);
if(!cuda_ok(cudaGetLastError(),"pipe attention launch (dev out)"))return 0;
quant_matmul<<<dim3(proj->O,S),256,0,dc->stream>>>(out_dev,dc->ac,proj->weights,
- proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I));
+ proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I),proj->gs,proj->ng);
if(!cuda_ok(cudaGetLastError(),"pipe o_proj launch (dev out)"))return 0;
return cuda_ok(cudaStreamSynchronize(dc->stream),"pipe attention sync (dev out)");
}
diff --git a/c/backend_cuda.h b/c/backend_cuda.h
index dfe38091..3af966df 100644
--- a/c/backend_cuda.h
+++ b/c/backend_cuda.h
@@ -41,18 +41,19 @@ COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,
int fmt, int I, int O, int device, int gs);
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
const void *weights, const float *scales,
- int fmt, int I, int O, int device);
+ int fmt, int I, int O, int device, int gs);
/*
* y[S,O] = x[S,I] @ W[O,I]^T.
- * fmt matches QT in glm.c: 0=f32, 1=int8, 2=int4, 3=int2.
- * The first successful call uploads W and its row scales; later calls reuse it.
+ * fmt matches QT in glm.c: 0=f32, 1=int8, 2=int4, 3=int2, 4=grouped int4.
+ * gs is the group size for fmt=4 (0 for all other formats).
+ * The first successful call uploads W and its scales; later calls reuse it.
* Returns 1 on success and 0 when CUDA is not initialized or the format is invalid.
*/
COLI_CUDA_DLLEXPORT int coli_cuda_matmul(ColiCudaTensor **tensor,
float *y, const float *x,
const void *weights, const float *scales,
- int fmt, int S, int I, int O, int device);
+ int fmt, int S, int I, int O, int device, int gs);
/* Fused expert pipeline: y = down(silu(gate(x)) * up(x)). All three tensors
* must already be resident on one device. Activations cross PCIe once in
diff --git a/c/backend_loader.c b/c/backend_loader.c
index feb908e6..bf0dd9a0 100644
--- a/c/backend_loader.c
+++ b/c/backend_loader.c
@@ -54,7 +54,7 @@ typedef int (*fn_tensor_upload)(ColiCudaTensor **tensor, const void *
typedef int (*fn_tensor_upload_g)(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs);
typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const float *x,
const void *weights, const float *scales,
- int fmt, int S, int I, int O, int device);
+ int fmt, int S, int I, int O, int device, int gs);
typedef void (*fn_tensor_free)(ColiCudaTensor *tensor);
typedef size_t (*fn_tensor_bytes)(const ColiCudaTensor *tensor);
typedef int (*fn_tensor_device)(const ColiCudaTensor *tensor);
@@ -325,9 +325,9 @@ int coli_cuda_attention_absorb(ColiCudaTensor *kv_b, float *ctx, const float *q,
}
int coli_cuda_tensor_upload(ColiCudaTensor **tensor, const void *weights,
- const float *scales, int fmt, int I, int O, int device){
+ const float *scales, int fmt, int I, int O, int device, int gs){
if(!g_cuda.available) return 0;
- return g_cuda.tensor_upload(tensor, weights, scales, fmt, I, O, device);
+ return g_cuda.tensor_upload(tensor, weights, scales, fmt, I, O, device, gs);
}
int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs){
@@ -337,9 +337,9 @@ int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, const void *weights, cons
int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x,
const void *weights, const float *scales,
- int fmt, int S, int I, int O, int device){
+ int fmt, int S, int I, int O, int device, int gs){
if(!g_cuda.available) return 0;
- return g_cuda.matmul(tensor, y, x, weights, scales, fmt, S, I, O, device);
+ return g_cuda.matmul(tensor, y, x, weights, scales, fmt, S, I, O, device, gs);
}
void coli_cuda_tensor_free(ColiCudaTensor *tensor){
diff --git a/c/colibri.c b/c/colibri.c
index ef32b016..d8108be7 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -448,6 +448,713 @@ static float *falloc(int64_t n){
float *p=malloc((size_t)n*sizeof(float)); if(!p){fprintf(stderr,"OOM\n");exit(1);} return p; }
+/* y[S,O] = x[S,I] @ W^T, W[O,I] f32 */
+static void matmul(float *y, const float *x, const float *W, int S, int I, int O){
+ #pragma omp parallel for schedule(static)
+ for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }
+}
+/* y[S,O] = x[S,I] @ W^T con W quantizzato int8 per-riga + scala[O] (dequant-on-use) */
+static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int S, int I, int O){
+ #pragma omp parallel for schedule(static)
+ for (int o=0;o<O;o++){ const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
+#ifdef __AVX2__
+ __m256 acc=_mm256_setzero_ps();
+ for(;i+8<=I;i+=8){ __m256i wi=_mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*)(w+i)));
+ acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), _mm256_cvtepi32_ps(wi), acc); }
+ a=hsum256(acc);
+#elif defined(__ARM_NEON)
+ float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
+ for(;i+8<=I;i+=8){ int16x8_t w16=vmovl_s8(vld1_s8(w+i));
+ ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w16))));
+ ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w16)))); }
+ a=vaddvq_f32(vaddq_f32(ac0,ac1));
+#endif
+ for(;i<I;i++) a+=xs[i]*(float)w[i]; y[(int64_t)s*O+o]=a*sc; } }
+}
+/* y[S,O] = x[S,I] @ W^T con W int4 impacchettato (2 valori/byte) + scala[O]. */
+static void matmul_i4(float *y, const float *x, const uint8_t *q4, const float *scale, int S, int I, int O){
+ int rb=(I+1)/2;
+ #pragma omp parallel for schedule(static)
+ for (int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
+#if defined(__AVX512F__) && defined(__AVX512BW__)
+ if(g_i4_acc512){ a=dot_i4f_avx512(w,xs,I); i=I&~31; }
+ else {
+#endif
+#ifdef __AVX2__
+ const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
+ __m256 acc=_mm256_setzero_ps();
+ for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); /* 8 byte=16 nibble */
+ __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
+ __m128i nib=_mm_unpacklo_epi8(lo,hi); /* nibble in ordine */
+ __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
+ __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
+ acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
+ acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
+ a=hsum256(acc);
+#elif defined(__ARM_NEON)
+ const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
+ float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
+ for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); /* 8 byte=16 nibble */
+ uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); /* nibble in ordine */
+ int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8));
+ int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8));
+ ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
+ ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
+ ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
+ ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
+ a=vaddvq_f32(vaddq_f32(ac0,ac1));
+#endif
+#if defined(__AVX512F__) && defined(__AVX512BW__)
+ }
+#endif
+ for(;i+1<I;i+=2){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8;
+ a += xs[i]*(float)lo + xs[i+1]*(float)hi; }
+ if(i<I){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; }
+ y[(int64_t)s*O+o]=a*sc; } }
+}
+/* y[S,O] = x[S,I] @ W^T with W int4 packed (2/byte) + per-GROUP scales (fmt=4).
+ * Same nibble math as matmul_i4, but the scale changes every `gs` elements along I.
+ * The accumulator resets at each group boundary: dot(x[grp], w[grp]) * scale[grp].
+ * gs MUST be a multiple of 16 (the AVX2 vector width). */
+static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale,
+ int S, int I, int O, int gs){
+ int rb=(I+1)/2; int ng=(I+gs-1)/gs;
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o++){
+ const uint8_t *w=q4+(int64_t)o*rb;
+ const float *scl=scale+(int64_t)o*ng;
+ for(int s=0;s<S;s++){
+ const float *xs=x+(int64_t)s*I; float a=0;
+ for(int g=0; g*gs<I; g++){
+ int base=g*gs; int glen=gs; if(base+glen>I) glen=I-base;
+ float sc=scl[g];
+ int i=base;
+#ifdef __AVX2__
+ const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
+ __m256 acc=_mm256_setzero_ps();
+ for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
+ __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
+ __m128i nib=_mm_unpacklo_epi8(lo,hi);
+ __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
+ __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
+ acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
+ acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
+ a+=hsum256(acc)*sc;
+#endif
+ /* scalar tail for the group remainder */
+ for(; i<base+glen; i+=2){
+ if(i+1<base+glen){ uint8_t byte=w[i>>1];
+ a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; }
+ else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; }
+ }
+ }
+ y[(int64_t)s*O+o]=a;
+ }
+ }
+}
+/* Fused gate+up for grouped int4 (fmt=4): computes both yg[S,O] and yu[S,O] from
+ * the same x[S,I], with per-group scales. One OpenMP dispatch covers both matrices,
+ * reading x once instead of twice β€” saves ~33% of expert-matmul time at decode.
+ * The per-group scale logic matches matmul_i4_grouped exactly. */
+static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x,
+ const uint8_t *qg, const float *sg,
+ const uint8_t *qu, const float *su,
+ int S, int I, int O, int gs){
+ int rb=(I+1)/2; int ng=(I+gs-1)/gs;
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o++){
+ const uint8_t *wg=qg+(int64_t)o*rb; const uint8_t *wu2=qu+(int64_t)o*rb;
+ const float *sgl=sg+(int64_t)o*ng; const float *sul=su+(int64_t)o*ng;
+ for(int s=0;s<S;s++){
+ const float *xs=x+(int64_t)s*I;
+ float ag=0, au=0;
+ for(int g=0; g*gs<I; g++){
+ int base=g*gs; int glen=gs; if(base+glen>I) glen=I-base;
+ float scg=sgl[g], scu=sul[g];
+ int i=base;
+#ifdef __AVX2__
+ const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
+ __m256 accg=_mm256_setzero_ps(), accu=_mm256_setzero_ps();
+ for(; i+16<=base+glen; i+=16){
+ __m128i byg=_mm_loadl_epi64((const __m128i*)(wg+(i>>1)));
+ __m128i log=_mm_and_si128(byg,m4),hig=_mm_and_si128(_mm_srli_epi16(byg,4),m4);
+ __m128i nibg=_mm_unpacklo_epi8(log,hig);
+ __m256 w0g=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nibg),b8));
+ __m256 w1g=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nibg,8)),b8));
+ accg=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0g, accg);
+ accg=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1g, accg);
+ __m128i byu=_mm_loadl_epi64((const __m128i*)(wu2+(i>>1)));
+ __m128i lou=_mm_and_si128(byu,m4),hiu=_mm_and_si128(_mm_srli_epi16(byu,4),m4);
+ __m128i nibu=_mm_unpacklo_epi8(lou,hiu);
+ __m256 w0u=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nibu),b8));
+ __m256 w1u=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nibu,8)),b8));
+ accu=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0u, accu);
+ accu=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1u, accu);
+ }
+ ag+=hsum256(accg)*scg; au+=hsum256(accu)*scu;
+#endif
+ for(; i+1<base+glen; i+=2){
+ uint8_t bg=wg[i>>1], bu=wu2[i>>1];
+ ag+=(xs[i]*(float)((int)(bg&0xF)-8)+xs[i+1]*(float)((int)(bg>>4)-8))*scg;
+ au+=(xs[i]*(float)((int)(bu&0xF)-8)+xs[i+1]*(float)((int)(bu>>4)-8))*scu;
+ }
+ if(i<base+glen){
+ uint8_t bg=wg[i>>1], bu=wu2[i>>1];
+ ag+=xs[i]*(float)((int)(bg&0xF)-8)*scg;
+ au+=xs[i]*(float)((int)(bu&0xF)-8)*scu;
+ }
+ }
+ yg[(int64_t)s*O+o]=ag; yu[(int64_t)s*O+o]=au;
+ }
+ }
+}
+/* Decode hot path for gate+up: same exact q4 dot products as matmul_i4, but one
+ * OpenMP dispatch covers both matrices. KTransformers uses persistent pools;
+ * this keeps colibri dependency-free while removing one team launch/expert. */
+static void matmul_i4_pair(float *yg, float *yu, const float *x,
+ const uint8_t *qg, const float *sg,
+ const uint8_t *qu, const float *su, int I, int O){
+ int rb=(I+1)/2;
+ #pragma omp parallel for schedule(static)
+ for(int z=0;z<2*O;z++){
+ int o=z<O?z:z-O; const uint8_t *w=(z<O?qg:qu)+(int64_t)o*rb;
+ float a=0; int i=0;
+#if defined(__AVX512F__) && defined(__AVX512BW__)
+ if(g_i4_acc512){ a=dot_i4f_avx512(w,x,I); i=I&~31; }
+ else {
+#endif
+#ifdef __AVX2__
+ const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
+ __m256 acc=_mm256_setzero_ps();
+ for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
+ __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
+ __m128i nib=_mm_unpacklo_epi8(lo,hi);
+ __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
+ __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
+ acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc);
+ acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); }
+ a=hsum256(acc);
+#elif defined(__ARM_NEON)
+ const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
+ float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0);
+ for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1));
+ uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4));
+ int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8));
+ int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8));
+ ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
+ ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
+ ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
+ ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
+ a=vaddvq_f32(vaddq_f32(ac0,ac1));
+#endif
+#if defined(__AVX512F__) && defined(__AVX512BW__)
+ }
+#endif
+ for(;i+1<I;i+=2){ uint8_t b=w[i>>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); }
+ if(i<I) a+=x[i]*(float)((w[i>>1]&15)-8);
+ (z<O?yg:yu)[o]=a*(z<O?sg:su)[o];
+ }
+}
+
+static void matmul_qt(float *y,const float *x,QT *w,int S);
+static int g_no_fused_pair=0; /* COLI_NO_FUSED_PAIR=1: disable the gate+up kernel fusion
+ * that changes OMP scheduling vs separate matmul_qt calls β€” this
+ * shifts floating-point accumulation order and can collapse MTP
+ * draft acceptance by flipping near-ties (#163). */
+/* #163: l'acceptance MTP crolla quando il forward di draft (S=1) e quello di verifica
+ * (S>=2) non calcolano la STESSA funzione. Tre interruttori dipendono da S: il gate
+ * int4-IDOT (S>=g_i4s β€” asimmetrico proprio dove g_i4s>1), la fusione gate+up solo-S==1,
+ * e la soglia righe del GEMM Metal. Con SPEC_PIN=1 (default) ogni forward emesso mentre
+ * i draft del modello sono attivi resta sulla famiglia di kernel di S=1: draft e verifica
+ * coincidono per costruzione. Prefill e decode non speculativo sono intoccati.
+ * EN: MTP acceptance collapses when the draft (S=1) and verify (S>=2) forwards do not
+ * compute the SAME function. Three switches are S-dependent: the int4 IDOT gate
+ * (S>=g_i4s β€” asymmetric exactly on ISAs where g_i4s>1), the S==1-only gate+up fusion,
+ * and the Metal GEMM row threshold. SPEC_PIN=1 (default) pins every forward issued
+ * while model drafts are live to the platform's S=1 kernel family, so draft and verify
+ * agree by construction; prefill and non-speculative decode are untouched.
+ * SPEC_PIN=0 restores the S-dependent gates (A/B). */
+static int g_spec_pin=1;
+static int g_spec_live=0; /* set by spec_decode while drafts are live */
+static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; }
+static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){
+ if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O)
+ matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O);
+ else if(!g_no_fused_pair&&S==1&&wg->fmt==4&&wu->fmt==4&&wg->I==wu->I&&wg->O==wu->O&&wg->gs==wu->gs)
+ matmul_i4_grouped_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,S,wg->I,wg->O,wg->gs);
+ else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); }
+}
+/* y[S,O] = x[S,I] @ W^T con W int2 impacchettato (4 valori/byte) + scala[O]. nibble 2-bit -> [-2,1]. */
+static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){
+ int rb=(I+3)/4;
+ #pragma omp parallel for schedule(static)
+ for (int o=0;o<O;o++){ const uint8_t *w=q2+(int64_t)o*rb; float sc=scale[o];
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
+#ifdef __AVX2__
+ const __m128i m2=_mm_set1_epi8(0x03); const __m256i b2=_mm256_set1_epi32(2);
+ __m256 acc=_mm256_setzero_ps();
+ for(;i+16<=I;i+=16){ __m128i by=_mm_cvtsi32_si128(*(const int*)(w+(i>>2))); /* 4 byte=16 valori */
+ __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2);
+ __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2);
+ __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3);
+ __m128i nib=_mm_unpacklo_epi16(lo,hi); /* 16 valori in ordine */
+ __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2));
+ __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2));
+ acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
+ acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
+ a=hsum256(acc);
+#elif defined(__ARM_NEON)
+ const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2);
+ float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
+ for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); /* 4 byte=16 valori */
+ uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd));
+ uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v));
+ uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6));
+ uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0]));
+ int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); /* 16 valori in ordine */
+ int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v));
+ ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
+ ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
+ ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
+ ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
+ a=vaddvq_f32(vaddq_f32(ac0,ac1));
+#endif
+ for(;i<I;i++){ uint8_t byte=w[i>>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); }
+ y[(int64_t)s*O+o]=a*sc; } }
+}
+/* ---- KERNEL INTERI (IDOT): attivazioni quantizzate a int8 per riga (absmax/127,
+ * stile Q8_0), prodotto scalare INTERO via maddubs/madd AVX2 β€” niente conversione
+ * f32 dei pesi nel ciclo caldo. ~2-3x sui matmul quantizzati; errore aggiunto ~0.3%
+ * RMS per matmul (attivazione int8), IDOT=0 torna al percorso f32 esatto. */
+#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
+#define IDOT_KERNEL "avx512-vnni"
+#elif defined(__AVXVNNI__) && defined(__AVX2__)
+#define IDOT_KERNEL "avx-vnni"
+#elif defined(__AVX2__)
+#define IDOT_KERNEL "avx2"
+#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
+#define IDOT_KERNEL "neon-i8mm"
+#elif defined(__ARM_NEON)
+#define IDOT_KERNEL "neon"
+#elif defined(__VSX__)
+#define IDOT_KERNEL "vsx"
+#else
+#define IDOT_KERNEL "scalar"
+#endif
+static int g_idot=1;
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
+static int g_i4s=1; /* SDOT presente: int4 IDOT conviene anche a S=1 (decode). Misurato
+ * su Apple M-series: +14%%, expert-matmul -16%%. EN: with SDOT, int4
+ * IDOT pays even at S=1 (decode); measured on Apple M-series. */
+#elif defined(__VSX__)
+static int g_i4s=1; /* POWER8 vec_msum: qui il fallback f32 e' SCALARE, quindi l'IDOT
+ * int4 conviene anche a S=1. Misurato su POWER8 S824 (vedi PR).
+ * EN: on VSX the f32 fallback is plain scalar C, so int4 IDOT
+ * pays even at S=1. Measured on a POWER8 S824 (see PR). */
+#else
+static int g_i4s=2; /* senza SDOT / altrove: soglia originale (misura AVX2 dell'autore).
+ * EN: without SDOT / elsewhere: original threshold (author's AVX2). */
+#endif
+static inline float qrow_i8(const float *x, int8_t *q, int I){
+ float amax=0; for(int i=0;i<I;i++){ float a=fabsf(x[i]); if(a>amax)amax=a; }
+ float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s;
+ for(int i=0;i<I;i++) q[i]=(int8_t)lrintf(x[i]*inv);
+ return s;
+}
+#ifdef __AVX2__
+static inline int hsum256_i32(__m256i v){
+ __m128i lo=_mm256_castsi256_si128(v), hi=_mm256_extracti128_si256(v,1);
+ lo=_mm_add_epi32(lo,hi); lo=_mm_hadd_epi32(lo,lo); lo=_mm_hadd_epi32(lo,lo);
+ return _mm_cvtsi128_si32(lo);
+}
+#endif
+#if defined(__AVXVNNI__) && defined(__AVX2__)
+/* hsum di un __m128i a 4 lane s32 (l'AVX-VNNI 128-bit accumula su 4 lane). */
+static inline int hsum128_i32(__m128i v){
+ v=_mm_hadd_epi32(v,v); v=_mm_hadd_epi32(v,v); return _mm_cvtsi128_si32(v);
+}
+#endif
+/* dot int8Β·int8: trucco del segno (|w| unsigned Γ— xΒ·sign(w) signed). Sicuro:
+ * coppie <= 128*127*2 = 32512 < 32767, accumulo s32 fino a I=16384. */
+static inline int32_t dot_i8i8(const int8_t *w, const int8_t *x, int I){
+ int32_t sum=0; int i=0;
+#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
+ /* VNNI: vpdpbusd u8*s8 -> s32 directly, 64 bytes/iter, no 16-bit intermediate.
+ * AVX-512 has no vpsignb: |w| via abs, sign folded into x with a mask-negate
+ * (w==0 -> product 0 either way). |x|<=127 (qrow_i8), |w|<=128 as u8: each
+ * s32 lane adds <= 4*128*127, safe up to I=16384 like the AVX2 bound. */
+ __m512i acc=_mm512_setzero_si512();
+ for(;i+64<=I;i+=64){
+ __m512i wv=_mm512_loadu_si512((const void*)(w+i));
+ __m512i xv=_mm512_loadu_si512((const void*)(x+i));
+ __mmask64 neg=_mm512_movepi8_mask(wv);
+ __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv);
+ acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs);
+ }
+ sum=_mm512_reduce_add_epi32(acc);
+#elif defined(__AVXVNNI__) && defined(__AVX2__)
+ /* AVX-VNNI 128-bit: vpdpbusd u8*s8 -> s32, 16 byte/iter. Stesso trucco del
+ * segno della variante 512-bit: |w| via abs, segno piegato in x con maschera
+ * (w==0 -> product 0). __AVX2__ serve per _mm_sign_epi8 / abs. */
+ __m128i acc=_mm_setzero_si128();
+ for(;i+16<=I;i+=16){
+ __m128i wv=_mm_loadu_si128((const __m128i*)(w+i));
+ __m128i xv=_mm_loadu_si128((const __m128i*)(x+i));
+ __m128i xs=_mm_sign_epi8(xv,wv); /* x * sign(w); _mm_sign zona __AVX2__ */
+ acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),xs);
+ }
+ sum=hsum128_i32(acc);
+#elif defined(__AVX2__)
+ __m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1);
+ for(;i+32<=I;i+=32){
+ __m256i wv=_mm256_loadu_si256((const __m256i*)(w+i));
+ __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i));
+ __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv));
+ acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones));
+ }
+ sum=hsum256_i32(acc);
+#elif defined(__ARM_NEON)
+ /* ARM: SDOT nativo se disponibile (Apple Silicon: sempre); altrimenti vmull/vpadal.
+ * Stesso bound anti-overflow del trucco AVX2: coppie <= 128*127*2 = 32512 < 32767. */
+#if defined(__ARM_FEATURE_DOTPROD)
+ /* 4 accumulatori indipendenti: SDOT ha latenza ~3-4 cicli, con un solo acc la
+ * catena seriale strozza il core a ~26 GB/s di pesi; con 4 lane indipendenti il
+ * dot diventa memory-bound (misurato su M4: 26 -> 63 GB/s per core, 2.4x). */
+ int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0);
+ for(;i+64<=I;i+=64){
+ a0=vdotq_s32(a0,vld1q_s8(w+i), vld1q_s8(x+i));
+ a1=vdotq_s32(a1,vld1q_s8(w+i+16),vld1q_s8(x+i+16));
+ a2=vdotq_s32(a2,vld1q_s8(w+i+32),vld1q_s8(x+i+32));
+ a3=vdotq_s32(a3,vld1q_s8(w+i+48),vld1q_s8(x+i+48));
+ }
+ int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
+ for(;i+16<=I;i+=16) acc=vdotq_s32(acc,vld1q_s8(w+i),vld1q_s8(x+i));
+ sum=vaddvq_s32(acc);
+#else
+ int32x4_t acc=vdupq_n_s32(0);
+ for(;i+16<=I;i+=16){
+ int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i);
+ int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv));
+ p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv));
+ acc=vpadalq_s16(acc,p);
+ }
+ sum=vaddvq_s32(acc);
+#endif
+#elif defined(__VSX__)
+ /* POWER8: vec_msum (s8 x u8 -> s32) somma i prodotti byte DIRETTAMENTE in lane
+ * s32, 16 byte/iter: il bound anti-saturazione a 16 bit di maddubs qui non serve.
+ * Stesso trucco del segno (|w| u8 per x*sign(w) s8), ma |w| via select+sub MODULO
+ * e non vec_abs: -128 deve diventare 128 u8, non saturare a 127.
+ * EN: vec_msum accumulates byte products straight into s32 lanes; |w| is built
+ * with a modulo subtract select instead of vec_abs so w=-128 wraps to 128 (u8)
+ * rather than saturating to 127. |x|<=127 from qrow_i8, so x negation is safe. */
+ __vector signed int acc=vec_splats(0);
+ const __vector signed char vz=vec_splats((signed char)0);
+ for(;i+16<=I;i+=16){
+ __vector signed char wv=vec_xl(0,(const signed char*)(w+i));
+ __vector signed char xv=vec_xl(0,(const signed char*)(x+i));
+ __vector __bool char neg=vec_cmplt(wv,vz);
+ __vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg);
+ __vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg);
+ acc=vec_msum(xs,wa,acc);
+ }
+ sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3);
+#endif
+ for(;i<I;i++) sum+=(int32_t)w[i]*x[i];
+ return sum;
+}
+/* dot int4(packed)Β·int8: nibble -> int8 [-8,7] al volo, poi stesso trucco */
+static inline int32_t dot_i4i8(const uint8_t *w4, const int8_t *x, int I){
+ int32_t sum=0; int i=0;
+#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
+ /* 32 bytes = 64 nibbles -> int8 in [-8,7], one vpdpbusd per 64 values.
+ * 256-bit unpack leaves values in per-128-lane order [0-15][32-47]/[16-31][48-63];
+ * dot pairing is order-invariant, so permute x's 128-bit blocks to match
+ * instead of re-ordering w (one vpermq per iter, off the critical unpack path). */
+ const __m256i m4v=_mm256_set1_epi8(0x0F);
+ const __m512i b8v=_mm512_set1_epi8(8);
+ const __m512i xidx=_mm512_setr_epi64(0,1,4,5,2,3,6,7);
+ __m512i acc=_mm512_setzero_si512();
+ for(;i+64<=I;i+=64){
+ __m256i by=_mm256_loadu_si256((const __m256i*)(w4+(i>>1)));
+ __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v);
+ __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi);
+ __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v);
+ __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i)));
+ __mmask64 neg=_mm512_movepi8_mask(wv);
+ __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv);
+ acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs);
+ }
+ sum=_mm512_reduce_add_epi32(acc);
+#elif defined(__AVXVNNI__) && defined(__AVX2__)
+ /* AVX-VNNI 128-bit, int4: 16 byte = 32 nibble -> int8 [-8,7] in due half
+ * (n0/n1), ciascuno alimentato a un vpdpbusd da 16 byte. Stesso unpack
+ * 128-bit del ramo AVX2 sotto; 32 elementi/iter come li. */
+ const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8);
+ __m128i acc=_mm_setzero_si128();
+ for(;i+32<=I;i+=32){
+ __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */
+ __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
+ __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* nibble in ordine */
+ __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8);
+ __m128i x0=_mm_loadu_si128((const __m128i*)(x+i));
+ __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16));
+ acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0));
+ acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1));
+ }
+ sum=hsum128_i32(acc);
+#elif defined(__AVX2__)
+ const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8);
+ const __m256i ones=_mm256_set1_epi16(1);
+ __m256i acc=_mm256_setzero_si256();
+ for(;i+32<=I;i+=32){
+ __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */
+ __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
+ __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* in ordine */
+ __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8);
+ __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i));
+ __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv));
+ acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones));
+ }
+ sum=hsum256_i32(acc);
+#elif defined(__ARM_NEON)
+ const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8);
+#if defined(__ARM_FEATURE_DOTPROD)
+ /* 4 accumulatori indipendenti (vedi dot_i8i8): spezza la catena seriale su acc.
+ * Misurato su M4: 12.4 -> 29.9 GB/s di pesi per core (2.4x). */
+ int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0);
+ for(;i+64<=I;i+=64){
+ uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16);
+ uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); /* nibble in ordine */
+ uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4));
+ a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i));
+ a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16));
+ a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32));
+ a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48));
+ }
+ int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
+ for(;i+32<=I;i+=32){
+ uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */
+ uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */
+ acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i));
+ acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16));
+ }
+ sum=vaddvq_s32(acc);
+#else
+ int32x4_t acc=vdupq_n_s32(0);
+ for(;i+32<=I;i+=32){
+ uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */
+ uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */
+ int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q);
+ int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q);
+ int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16);
+ int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); /* |w|<=8: nessun overflow */
+ p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0));
+ acc=vpadalq_s16(acc,p);
+ p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1));
+ p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1));
+ acc=vpadalq_s16(acc,p);
+ }
+ sum=vaddvq_s32(acc);
+#endif
+#elif defined(__VSX__)
+ /* 16 byte = 32 nibble. vec_mergeh/vec_mergel su ppc64le (GCC) interallacciano come
+ * unpacklo/unpackhi x86 (verificato empiricamente su POWER8): i nibble escono in
+ * ordine di memoria. |w|<=8 dopo il -8, quindi stesso trucco del segno di dot_i8i8.
+ * EN: vec_mergeh/l on ppc64le interleave like x86 unpacklo/hi (verified on POWER8),
+ * so nibbles come out in memory order; then the same sign trick as dot_i8i8. */
+ const __vector unsigned char m4v=vec_splats((unsigned char)0x0F);
+ const __vector unsigned char sh4=vec_splats((unsigned char)4);
+ const __vector signed char b8v=vec_splats((signed char)8);
+ const __vector signed char vz=vec_splats((signed char)0);
+ __vector signed int acc=vec_splats(0);
+ for(;i+32<=I;i+=32){
+ __vector unsigned char by=vec_xl(0,w4+(i>>1)); /* 16 byte = 32 nibble */
+ __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4);
+ __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v);
+ __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v);
+ __vector signed char x0=vec_xl(0,(const signed char*)(x+i));
+ __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16));
+ __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz);
+ acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0),
+ (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc);
+ acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1),
+ (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc);
+ }
+ sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3);
+#endif
+ for(;i+1<I;i+=2){ uint8_t b=w4[i>>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; }
+ if(i<I){ uint8_t b=w4[i>>1]; sum+=((int)(b&0xF)-8)*x[i]; }
+ return sum;
+}
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
+/* SMMLA (i8mm): vmmlaq_s32 vede ogni int8x16_t come matrice 2x8 row-major (byte 0-7 =
+ * riga 0, byte 8-15 = riga 1) e accumula C += A*B^T nel 2x2 int32: lane0=a0.b0,
+ * lane1=a0.b1, lane2=a1.b0, lane3=a1.b1. vcombine di due mezze-righe costruisce la
+ * matrice: A = due righe di peso (o,o+1), B = due righe di attivazione (s,s+1), quindi
+ * meta' traffico pesi e doppio lavoro per istruzione a S>=2. EN: vmmlaq_s32 treats each
+ * int8x16_t as a 2x8 row-major matrix and does C += A*B^T on a 2x2 int32 tile; vcombine
+ * of vget_low/high halves builds the 2-row register from two weight/activation rows. */
+static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1,
+ int8x16_t xs, int8x16_t xs1){
+ acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)),
+ vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1)));
+ return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)),
+ vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1)));
+}
+static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q,
+ const float *scale, int S, int I, int O){
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<(O&~1);o+=2){
+ const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I;
+ float sc0=scale[o], sc1=scale[o+1];
+ for(int s=0;s<(S&~1);s+=2){
+ const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I;
+ /* 4 accumulatori indipendenti: una sola catena vmmla e' latency-bound.
+ * EN: 4 independent accumulators; a single vmmla chain is latency-bound. */
+ int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0;
+ for(;i+64<=I;i+=64){
+ a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i));
+ a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16));
+ a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32));
+ a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48));
+ }
+ for(;i+16<=I;i+=16)
+ a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i));
+ int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
+ int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1);
+ int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3);
+ for(;i<I;i++){ int a=wo[i],b=wo1[i],u=xs[i],v=xs1[i];
+ d00+=a*u; d01+=a*v; d10+=b*u; d11+=b*v; }
+ y[(int64_t)s*O+o] =(float)d00*sc0*sx[s];
+ y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s];
+ y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1];
+ y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1];
+ }
+ if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I;
+ y[(int64_t)s*O+o] =(float)dot_i8i8(wo, xs,I)*sc0*sx[s];
+ y[(int64_t)s*O+(o+1)]=(float)dot_i8i8(wo1,xs,I)*sc1*sx[s]; }
+ }
+ if(O&1){ int o=O-1; const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
+ #pragma omp parallel for schedule(static)
+ for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i8i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
+}
+static void matmul_i4_idot_mm(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
+ const float *scale, int S, int I, int O){
+ int rb=(I+1)/2;
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<(O&~1);o+=2){
+ const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8);
+ const uint8_t *wo=q4+(int64_t)o*rb, *wo1=q4+(int64_t)(o+1)*rb;
+ float sc0=scale[o], sc1=scale[o+1];
+ for(int s=0;s<(S&~1);s+=2){
+ const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I;
+ /* 4 accumulatori indipendenti (vedi matmul_q_idot_mm).
+ * EN: 4 independent accumulators, see matmul_q_idot_mm. */
+ int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0;
+ for(;i+64<=I;i+=64){
+ uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1));
+ uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16);
+ uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4));
+ uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4));
+ uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4));
+ uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4));
+ a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q),
+ vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q),
+ vld1q_s8(xs+i), vld1q_s8(xs1+i));
+ a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q),
+ vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q),
+ vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16));
+ a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q),
+ vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q),
+ vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32));
+ a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q),
+ vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q),
+ vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48));
+ }
+ for(;i+32<=I;i+=32){
+ uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1));
+ uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4));
+ uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4));
+ a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q),
+ vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q),
+ vld1q_s8(xs+i), vld1q_s8(xs1+i));
+ a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q),
+ vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q),
+ vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16));
+ }
+ int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
+ int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1);
+ int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3);
+ for(;i+1<I;i+=2){ uint8_t bo=wo[i>>1], bo1=wo1[i>>1];
+ int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8;
+ int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1];
+ d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; }
+ if(i<I){ uint8_t bo=wo[i>>1], bo1=wo1[i>>1];
+ int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8;
+ d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; }
+ y[(int64_t)s*O+o] =(float)d00*sc0*sx[s];
+ y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s];
+ y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1];
+ y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1];
+ }
+ if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I;
+ y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s];
+ y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; }
+ }
+ if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
+ #pragma omp parallel for schedule(static)
+ for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i4i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
+}
+#endif
+static void matmul_q_idot(float *y, const int8_t *xq, const float *sx, const int8_t *q,
+ const float *scale, int S, int I, int O){
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
+ if(S>=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; }
+#endif
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o++){ const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
+ for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i8i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
+}
+static void matmul_i4_idot(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
+ const float *scale, int S, int I, int O){
+ int rb=(I+1)/2;
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
+ if(S>=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; }
+#endif
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
+ for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i4i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
+}
+
+typedef struct { int8_t *xq; size_t xq_cap; float *sx; size_t sx_cap; } QScratch;
+static _Thread_local QScratch g_qscratch;
+static void quant_scratch(size_t xn, size_t sn, int8_t **xq, float **sx){
+ if(xn>g_qscratch.xq_cap){
+ int8_t *p=realloc(g_qscratch.xq,xn);
+ if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); }
+ g_qscratch.xq=p; g_qscratch.xq_cap=xn;
+ }
+ if(sn>g_qscratch.sx_cap){
+ float *p=realloc(g_qscratch.sx,sn*sizeof(float));
+ if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); }
+ g_qscratch.sx=p; g_qscratch.sx_cap=sn;
+ }
+ *xq=g_qscratch.xq; *sx=g_qscratch.sx;
+}
+
+/* allow_idot=0: forza il kernel int4/int8 ESATTO (attivazioni f32). Serve alle proiezioni di
+ * attenzione: sono sensibili alla quantizzazione int8 delle attivazioni dell'IDOT. Misurato su
+ * GLM-5.2 int4, 1023 token, log-lik -5040.33 (esatto) -> -5160.47 (IDOT) = +0.117 nat/token,
+ * ~+12% perplexity. Gli altri matmul del prefill (o_proj, kv_b, expert) tengono l'IDOT.
+ * EN: allow_idot=0 forces the EXACT int4/int8 kernel (f32 activations). The attention
+ * projections need it: IDOT's int8 activation quantization costs +0.117 nats/token there
+ * (~+12% perplexity), measured. Every other prefill matmul keeps IDOT as before. */
+static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot);
+static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); }
static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot){
#ifdef COLI_METAL
if(g_metal_enabled && S>=g_metal_gemm_min && !spec_pinned() && (w->fmt==1||w->fmt==2) && !omp_in_parallel()){
@@ -459,7 +1166,7 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot)
if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && w->fmt!=5 && !omp_in_parallel()){
const void *weights = w->fmt==0 ? (const void*)w->qf
: w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4;
- if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device)) return;
+ if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device,w->gs)) return;
w->cuda_failed=1;
fprintf(stderr,"[CUDA] tensor [%d,%d] on device %d disabled after an error; falling back to CPU\n",
w->O,w->I,w->cuda_device);
@@ -934,13 +1641,14 @@ static void qt_cuda_colocate(QT *dst,const QT *src){
}
static void layer_cuda_shard_kvb(Layer *l,int H,int Q,int V){
if(!g_cuda_enabled||!g_cuda_dense||g_cuda_ndev<2||l->kv_b.fmt==0)return;
- int rb=l->kv_b.fmt==1?l->kv_b.I:(l->kv_b.fmt==2?(l->kv_b.I+1)/2:(l->kv_b.I+3)/4);
+ int rb=l->kv_b.fmt==1?l->kv_b.I:
+ (l->kv_b.fmt==2||l->kv_b.fmt==4)?(l->kv_b.I+1)/2:(l->kv_b.I+3)/4;
const uint8_t *weights=l->kv_b.fmt==1?(const uint8_t*)l->kv_b.q8:l->kv_b.q4;
for(int d=0,h0=0;d<g_cuda_ndev;d++){
int hn=H/g_cuda_ndev+(d<H%g_cuda_ndev),rows=hn*(Q+V);
const void *part=weights+(int64_t)h0*(Q+V)*rb;
const float *scale=l->kv_b.s+(int64_t)h0*(Q+V);
- if(!coli_cuda_tensor_upload(&l->kv_b_shard[d],part,scale,l->kv_b.fmt,l->kv_b.I,rows,g_cuda_devices[d]))return;
+ if(!coli_cuda_tensor_upload(&l->kv_b_shard[d],part,scale,l->kv_b.fmt,l->kv_b.I,rows,g_cuda_devices[d],l->kv_b.gs))return;
l->shard_h0[d]=h0;l->shard_hn[d]=hn;l->n_kv_b_shard++;h0+=hn;
}
int old=-1;for(int i=0;i<g_cuda_ndev;i++)if(g_cuda_devices[i]==l->kv_b.cuda_device)old=i;
@@ -1122,6 +1830,14 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
static void embed_row(Model *m, int tok, float *x){
int D=m->c.hidden; QT *e=&m->embed;
if(e->fmt==0){ memcpy(x, e->qf+(int64_t)tok*D, D*sizeof(float)); return; }
+ if(e->fmt==4){ /* grouped int4: per-group scale (embed/lm_head at io_bits, usually fmt 0/1) */
+ const uint8_t *q=e->q4+(int64_t)tok*((D+1)/2); int gs=e->gs,ng=(D+gs-1)/gs;
+ const float *scl=e->s+(int64_t)tok*ng;
+ for(int g=0;g*gs<D;g++){ int base=g*gs,glen=gs; if(base+glen>D)glen=D-base; float s=scl[g];
+ for(int i=base;i+1<base+glen;i+=2){ uint8_t byte=q[i>>1]; x[i]=(float)((int)(byte&0xF)-8)*s;
+ x[i+1]=(float)((int)(byte>>4)-8)*s; }
+ if(glen&1){ uint8_t byte=q[(base+glen-1)>>1]; x[base+glen-1]=(float)((int)(byte&0xF)-8)*s; } }
+ return; }
if(e->fmt==1){ const int8_t *q=e->q8+(int64_t)tok*D; float s=e->s[tok];
for(int i=0;i<D;i++) x[i]=(float)q[i]*s; return; }
if(e->fmt==2){ const uint8_t *q=e->q4+(int64_t)tok*((D+1)/2); float s=e->s[tok]; /* int4 */
@@ -1885,6 +2601,13 @@ static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y)
int I=t->I;
for(int j=0;j<n;j++){ int row=r0+j; double a=0;
if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;i<I;i++) a+=(double)w[i]*x[i]; }
+ else if(t->fmt==4){ /* grouped int4: per-group scale */
+ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); int gs=t->gs,ng=(I+gs-1)/gs;
+ const float *scl=t->s+(int64_t)row*ng;
+ for(int g=0;g*gs<I;g++){ int base=g*gs,glen=gs; if(base+glen>I)glen=I-base; float sc=scl[g]; float acc=0;
+ for(int i=base;i+1<base+glen;i+=2){ uint8_t b=w[i>>1]; acc+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; }
+ if(glen&1){ uint8_t b=w[(base+glen-1)>>1]; acc+=((int)(b&0xF)-8)*x[base+glen-1]; }
+ a+=acc*sc; } }
else if(t->fmt==1){ const int8_t *w=t->q8+(int64_t)row*I; float s=t->s[row];
float acc=0; for(int i=0;i<I;i++) acc+=(float)w[i]*x[i]; a=acc*s; }
else if(t->fmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); float s=t->s[row]; float acc=0;
@@ -4479,9 +5202,21 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){
grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */
if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale β€” la coda della
* distribuzione int4 e' rumore di quantizzazione */
- int cap=(int)strlen(prompt)+16; int *pids=malloc(cap*sizeof(int));
+ int cap=(int)strlen(prompt)+16; int *pids=malloc((cap+2)*sizeof(int));
int np=tok_encode(&T,prompt,(int)strlen(prompt),pids,cap);
if(np<1){ fprintf(stderr,"prompt is empty after tokenization\n"); return; }
+ /* GLM prefix (#108): every GLM training sequence starts [gMASK]<sop>; without it the
+ * model is out-of-distribution and generates garbage. Serve/SCORE modes prepend it;
+ * run_text must too. CHAT_TEMPLATE=0 disables (raw prompt, for debugging/non-GLM).
+ * A prompt that already starts with the prefix passes through intact. */
+ int templ=getenv("CHAT_TEMPLATE")?atoi(getenv("CHAT_TEMPLATE")):1;
+ if(templ){
+ int gmask=tok_id_of(&T,"[gMASK]"), sop=tok_id_of(&T,"<sop>");
+ if(gmask>=0 && sop>=0 && (np<2 || pids[0]!=gmask || pids[1]!=sop)){
+ memmove(pids+2,pids,np*sizeof(int));
+ pids[0]=gmask; pids[1]=sop; np+=2;
+ }
+ }
printf("prompt: %d tokens | generating up to %d (EOS stop=%d) | n-gram draft=%d\n", np, ngen, eos, g_draft);
fputs(prompt,stdout); fflush(stdout);
kv_alloc(m, np+ngen+g_draft+2);
diff --git a/c/tests/bench_tensor_core.cu b/c/tests/bench_tensor_core.cu
index c8c43623..3666eb43 100644
--- a/c/tests/bench_tensor_core.cu
+++ b/c/tests/bench_tensor_core.cu
@@ -30,9 +30,9 @@ int main(){
for(int i=0;i<D;i++)ds[i]=0.006f+(i%7)*0.0002f;
for(size_t i=0;i<x.size();i++)x[i]=std::sin((float)(i+1)*0.013f)*2.f;
ColiCudaTensor *g=nullptr,*u=nullptr,*d=nullptr;
- if(!coli_cuda_tensor_upload(&g,hidden.data(),hs.data(),2,D,I,device)||
- !coli_cuda_tensor_upload(&u,hidden.data(),hs.data(),2,D,I,device)||
- !coli_cuda_tensor_upload(&d,down.data(),ds.data(),2,I,D,device))return 2;
+ if(!coli_cuda_tensor_upload(&g,hidden.data(),hs.data(),2,D,I,device,0)||
+ !coli_cuda_tensor_upload(&u,hidden.data(),hs.data(),2,D,I,device,0)||
+ !coli_cuda_tensor_upload(&d,down.data(),ds.data(),2,I,D,device,0))return 2;
for(int rows: {1,2,4,8}){
double scalar=run(g,u,d,x.data(),a.data(),rows,3,0);
double packed=run(g,u,d,x.data(),b.data(),rows,3,1);
diff --git a/c/tests/test_backend_cuda.cu b/c/tests/test_backend_cuda.cu
index 259fe6b5..b0b5a3e8 100644
--- a/c/tests/test_backend_cuda.cu
+++ b/c/tests/test_backend_cuda.cu
@@ -99,7 +99,7 @@ int main(int argc, char **argv) {
const int8_t q8b[8]={-1,-2,-3,-4, 1,-2,3,-4};
const float s8b[2]={1.f,.5f},want8b[4]={10.f,15.f,-3.f,-2.5f};
if(!coli_cuda_tensor_update(t8,q8b,s8b)||
- !coli_cuda_matmul(&t8,got,x,q8b,s8b,1,2,4,2,d0)||
+ !coli_cuda_matmul(&t8,got,x,q8b,s8b,1,2,4,2,d0,0)||
!close_enough(got,want8b,4))return 1;
/* Rows [-8,-1,0,7] and [1,2,3,4], packed low nibble first. */
@@ -107,26 +107,26 @@ int main(int argc, char **argv) {
const float s4[2] = {1.0f, 0.25f};
const float want4[2] = {-34.0f, -2.5f};
ColiCudaTensor *t4 = nullptr;
- if (!coli_cuda_matmul(&t4, got, x, q4, s4, 2, 1, 4, 2, d1) || !close_enough(got, want4, 2)) return 1;
+ if (!coli_cuda_matmul(&t4, got, x, q4, s4, 2, 1, 4, 2, d1, 0) || !close_enough(got, want4, 2)) return 1;
const uint8_t q2[2] = {0xe4, 0x1b};
const float s2[2] = {0.5f, 2.0f};
const float want2[2] = {-2.0f, 12.0f};
ColiCudaTensor *t2 = nullptr;
- if (!coli_cuda_matmul(&t2, got, x, q2, s2, 3, 1, 4, 2, d1) || !close_enough(got, want2, 2)) return 1;
+ if (!coli_cuda_matmul(&t2, got, x, q2, s2, 3, 1, 4, 2, d1, 0) || !close_enough(got, want2, 2)) return 1;
const float wf[8] = {1, 0, -1, 2, 0.5f, 0.5f, 0.5f, 0.5f};
const float wantf[2] = {-10.0f, -1.0f};
ColiCudaTensor *tf = nullptr;
- if (!coli_cuda_matmul(&tf, got, x, wf, nullptr, 0, 1, 4, 2, d0) || !close_enough(got, wantf, 2)) return 1;
+ if (!coli_cuda_matmul(&tf, got, x, wf, nullptr, 0, 1, 4, 2, d0, 0) || !close_enough(got, wantf, 2)) return 1;
const float eg[8] = {1,0,0,0, 0,1,0,0};
const float eu[8] = {1,0,0,0, 0,1,0,0};
const float ed[8] = {1,0, 0,1, 1,1, 1,-1};
ColiCudaTensor *tg=nullptr,*tu=nullptr,*td=nullptr;
- if (!coli_cuda_tensor_upload(&tg,eg,nullptr,0,4,2,d0) ||
- !coli_cuda_tensor_upload(&tu,eu,nullptr,0,4,2,d0) ||
- !coli_cuda_tensor_upload(&td,ed,nullptr,0,2,4,d0)) return 1;
+ if (!coli_cuda_tensor_upload(&tg,eg,nullptr,0,4,2,d0,0) ||
+ !coli_cuda_tensor_upload(&tu,eu,nullptr,0,4,2,d0,0) ||
+ !coli_cuda_tensor_upload(&td,ed,nullptr,0,2,4,d0,0)) return 1;
float expert[8], want_expert[8];
for(int s=0;s<2;s++){
float a=x[s*4], b=x[s*4+1];
@@ -144,7 +144,7 @@ int main(int argc, char **argv) {
const float aw[16]={1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1};
const float aq[4]={1,2,.5f,-.5f},al[12]={1,0,0,0, 0,1,0,0, 0,0,1,0};
const float ar[6]={1,0, 0,1, 1,1};float actx[2],aref[2];
- ColiCudaTensor *at=nullptr;if(!coli_cuda_tensor_upload(&at,aw,nullptr,0,4,4,d0))return 1;
+ ColiCudaTensor *at=nullptr;if(!coli_cuda_tensor_upload(&at,aw,nullptr,0,4,4,d0,0))return 1;
float score[3];for(int t=0;t<3;t++)score[t]=aq[0]*al[t*4]+aq[1]*al[t*4+1]+aq[2]*ar[t*2]+aq[3]*ar[t*2+1];
float mx=score[0],z=0;for(int t=1;t<3;t++)mx=score[t]>mx?score[t]:mx;
for(int t=0;t<3;t++){score[t]=std::exp(score[t]-mx);z+=score[t];}for(int t=0;t<3;t++)score[t]/=z;
@@ -163,9 +163,9 @@ int main(int argc, char **argv) {
for(int i=0;i<32;i++)ws4[i]=0.01f+(i%5)*0.002f;
for(int i=0;i<64;i++)gx4[i]=std::sin((float)(i+1)*0.17f)*2.f;
ColiCudaTensor *g4=nullptr,*u4=nullptr,*d4=nullptr;
- if(!coli_cuda_tensor_upload(&g4,w4,ws4,2,32,32,d0)||
- !coli_cuda_tensor_upload(&u4,w4,ws4,2,32,32,d0)||
- !coli_cuda_tensor_upload(&d4,w4,ws4,2,32,32,d0))return 1;
+ if(!coli_cuda_tensor_upload(&g4,w4,ws4,2,32,32,d0,0)||
+ !coli_cuda_tensor_upload(&u4,w4,ws4,2,32,32,d0,0)||
+ !coli_cuda_tensor_upload(&d4,w4,ws4,2,32,32,d0,0))return 1;
ColiCudaTensor *gg4[2]={g4,g4},*ug4[2]={u4,u4},*dg4[2]={d4,d4};
if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,scalar4,gx4))return 1;
setenv("COLI_CUDA_TC_INT4","1",1);
diff --git a/c/tests/test_pipe_cuda.cu b/c/tests/test_pipe_cuda.cu
index f20a9e5b..fc66dbdf 100644
--- a/c/tests/test_pipe_cuda.cu
+++ b/c/tests/test_pipe_cuda.cu
@@ -119,7 +119,7 @@ int main(void){
for(int i=0;i<O;i++) sc[i]=0.01f+0.001f*(i%7);
for(size_t i=0;i<(size_t)S*K;i++) x[i]=rndf();
ColiCudaTensor *t=NULL;
- ok&=coli_cuda_matmul(&t,ref,x,w4,sc,2,S,K,O,0); /* host path = reference */
+ ok&=coli_cuda_matmul(&t,ref,x,w4,sc,2,S,K,O,0,0); /* host path = reference */
float *xd=(float*)coli_cuda_pipe_alloc(0,(size_t)S*K*4);
float *yd=(float*)coli_cuda_pipe_alloc(0,(size_t)S*O*4);
ok&=coli_cuda_pipe_upload(0,xd,x,(size_t)S*K*4);
diff --git a/c/tools/diag_harness.py b/c/tools/diag_harness.py
new file mode 100644
index 00000000..56a1b225
--- /dev/null
+++ b/c/tools/diag_harness.py
@@ -0,0 +1,704 @@
+#!/usr/bin/env python3
+"""
+diag_harness.py β€” Comprehensive model diagnostic harness for the colibri GLM-5.2 engine.
+
+Runs a full campaign of tests against any model snapshot:
+ Phase 0 (system): startup telemetry β€” GPU, RAM, cache cap, load time, MTP, idot kernel
+ Phase 1 (smoke): correctness β€” curated prompts, coherence checks, corruption detection
+ Phase 2 (diagnostic): deep x-ray β€” full PROFILE breakdown, routing, MTP, disk-split, CUDA tier
+ Phase 3 (quality): benchmark accuracy β€” hellaswag/arc_challenge/mmlu via eval_glm.py SCORE
+ Phase 4 (throughput): tok/s with and without MTP speculation
+ Phase 5 (report): structured JSON + human-readable Markdown summary
+
+Usage:
+ python tools/diag_harness.py --snap /path/to/model --phase all
+ python tools/diag_harness.py --snap /path/to/model --phase smoke --ngen 64
+ python tools/diag_harness.py --snap /path/to/model --phase quality --quality-limit 200
+
+Output goes to --out (default ./diag_results/<timestamp>/). Each phase writes a raw log
+(<phase>_<run>.log) and all metrics are collected into report.json + report.md.
+
+Design notes:
+ - stdout and stderr are captured separately via subprocess.PIPE. The engine streams
+ generated text to stdout (interleaved with prompt + PROFILE stats), but TOKENS=1 dumps
+ clean token-id lists to stderr β€” that is the primary text-capture path.
+ - Every regex is anchored to the exact printf format strings in glm.c (verified against
+ profile_print line 3853, run_text line 3948, the banner line 5299, etc.).
+ - Subprocess calls have a hard timeout (default 600s) and are killed cleanly on expiry.
+ - A single phase can be run standalone; results accumulate in the output dir.
+"""
+import os, sys, re, json, time, argparse, subprocess, signal, traceback
+from datetime import datetime
+from pathlib import Path
+
+# ---------------------------------------------------------------------------
+# PROMPT SUITE β€” curated across categories. "expect" is a case-insensitive
+# substring checked against the generated text (None = coherence-only check).
+# ---------------------------------------------------------------------------
+PROMPTS = [
+ {"id":"fact_capital", "cat":"factual", "prompt":"The capital of France is",
+ "expect":"Paris", "note":"basic world knowledge"},
+ {"id":"fact_boiling", "cat":"factual", "prompt":"What is the boiling point of water in Celsius?",
+ "expect":"100", "note":"basic science"},
+ {"id":"fact_planet", "cat":"factual", "prompt":"What planet is closest to the Sun?",
+ "expect":"Mercury","note":"astronomy fact"},
+ {"id":"math_mult", "cat":"math", "prompt":"What is 15 times 12?",
+ "expect":"180", "note":"2-digit multiplication"},
+ {"id":"math_add", "cat":"math", "prompt":"What is 847 plus 153?",
+ "expect":"1000", "note":"3-digit addition"},
+ {"id":"reason_train", "cat":"reasoning", "prompt":"If a train travels 60 mph for 2.5 hours, how far does it go?",
+ "expect":"150", "note":"rate-time-distance"},
+ {"id":"code_factorial","cat":"code", "prompt":"Write a Python function that computes the factorial of a number.",
+ "expect":"def", "note":"code generation"},
+ {"id":"explain_nn", "cat":"explanation","prompt":"Explain what a neural network is in one sentence.",
+ "expect":None, "note":"coherence check"},
+ {"id":"creative_story","cat":"creative", "prompt":"Write a one-sentence story about a lighthouse.",
+ "expect":None, "note":"creative coherence"},
+ {"id":"edge_hello", "cat":"edge", "prompt":"Hello, how are you today?",
+ "expect":None, "note":"conversational opener"},
+ {"id":"edge_the", "cat":"edge", "prompt":"The weather today is",
+ "expect":None, "note":"simple continuation"},
+ {"id":"edge_repeat", "cat":"edge", "prompt":"The quick brown fox jumps over the lazy dog. The quick brown fox",
+ "expect":None, "note":"repetition-bait"},
+]
+
+# ---------------------------------------------------------------------------
+# METRIC EXTRACTION β€” each parser is (name, compiled_regex, group_index, cast).
+# Regexes match the EXACT printf format strings in glm.c.
+# ---------------------------------------------------------------------------
+def _f1(g): return float(g)
+def _i1(g): return int(g)
+
+# Build parsers as (regex, lambda(match)->value) for clarity
+RX = {
+ # stdout: run_text summary line (line 3948)
+ "decode_toks": (re.compile(r"decode (\d+) tokens in"), lambda m: int(m.group(1))),
+ "decode_secs": (re.compile(r"decode \d+ tokens in ([\d.]+)s"), lambda m: float(m.group(1))),
+ "decode_tps": (re.compile(r"decode \d+ tokens in [\d.]+s \(([\d.]+) tok/s\)"), lambda m: float(m.group(1))),
+ "prefill_toks": (re.compile(r"prefill (\d+) tokens in"), lambda m: int(m.group(1))),
+ "prefill_secs": (re.compile(r"prefill \d+ tokens in ([\d.]+)s"), lambda m: float(m.group(1))),
+ "hit_rate": (re.compile(r"expert hit rate ([\d.]+)%"), lambda m: float(m.group(1))),
+ "rss_gb": (re.compile(r"RSS ([\d.]+) GB"), lambda m: float(m.group(1))),
+ "experts_per_tok":(re.compile(r"experts loaded/token: ([\d.]+)"), lambda m: float(m.group(1))),
+ "mtp_accept": (re.compile(r"MTP acceptance ([\d.]+)%"), lambda m: float(m.group(1))),
+ "mtp_acc_cnt": (re.compile(r"MTP acceptance \d+% \((\d+)/(\d+)\)"), lambda m: (int(m.group(1)), int(m.group(2)))),
+ "spec_tok_per_fw":(re.compile(r"speculation: ([\d.]+) tokens/forward"),lambda m: float(m.group(1))),
+ # stdout: PROFILE lines (profile_print, line 3853-3864)
+ "prof_expert_disk": (re.compile(r"expert-disk ([\d.]+)s service"), lambda m: float(m.group(1))),
+ "prof_expert_wait": (re.compile(r"service / ([\d.]+)s wait"), lambda m: float(m.group(1))),
+ "prof_expert_mm": (re.compile(r"expert-matmul ([\d.]+)s"), lambda m: float(m.group(1))),
+ "prof_attention": (re.compile(r"\| attention ([\d.]+)s"), lambda m: float(m.group(1))),
+ "prof_kvb": (re.compile(r"including kvb ([\d.]+)s"), lambda m: float(m.group(1))),
+ "prof_lm_head": (re.compile(r"lm_head ([\d.]+)s"), lambda m: float(m.group(1))),
+ "prof_other": (re.compile(r"\| other ([\d.]+)s"), lambda m: float(m.group(1))),
+ # stdout: banner (line 5299)
+ "load_secs": (re.compile(r"loaded in ([\d.]+)s"), lambda m: float(m.group(1))),
+ "resident_mb": (re.compile(r"resident dense: ([\d.]+) MB"), lambda m: float(m.group(1))),
+ "mtp_status": (re.compile(r"MTP (ACTIVE|absent)"), lambda m: m.group(1)),
+ "n_layers": (re.compile(r"layers=(\d+) experts=(\d+)"), lambda m: int(m.group(1))),
+ "n_experts": (re.compile(r"layers=(\d+) experts=(\d+)"), lambda m: int(m.group(2))),
+ # stdout: banner idot kernel
+ "idot_kernel": (re.compile(r"idot: (\S+) =="), lambda m: m.group(1)),
+ "cache_cap": (re.compile(r"cache=(\d+) experts/layer"), lambda m: int(m.group(1))),
+ # stderr: RAM_GB (line 5086-5112)
+ "ram_budget": (re.compile(r"\[RAM_GB=([\d.]+)"), lambda m: float(m.group(1))),
+ "cap_lowered": (re.compile(r"cap lowered (\d+)->(\d+)"), lambda m: (int(m.group(1)), int(m.group(2)))),
+ "cap_raised": (re.compile(r"cap raised (\d+)->(\d+)"), lambda m: (int(m.group(1)), int(m.group(2)))),
+ "cap_ok": (re.compile(r"cap=(\d+) ok"), lambda m: int(m.group(1))),
+ # stderr: CUDA (backend_cuda.cu:389)
+ "cuda_device": (re.compile(r"\[CUDA\] device (\d+): (.*?), ([\d.]+) GB VRAM, sm_(\d)(\d)"),
+ lambda m: {"id":int(m.group(1)),"name":m.group(2).strip(),"vram_gb":float(m.group(3)),
+ "sm":f"{m.group(4)}.{m.group(5)}"}),
+ "cuda_mode": (re.compile(r"\[CUDA\] mode: (.+)"), lambda m: m.group(1)),
+ "cuda_tier": (re.compile(r"CUDA expert tier: (\d+) resident experts \(([\d.]+) GB\)"),
+ lambda m: {"resident":int(m.group(1)),"vram_gb":float(m.group(2))}),
+ # stderr: TOKENS dump (line 4010-4012)
+ "tokens_dump": (re.compile(r"^\[TOKENS\] (\d+) generated:(.*)$", re.MULTILINE),
+ lambda m: [int(x) for x in m.group(2).split()]),
+ # stderr: per-16-token progress (emit_stream line 3742)
+ "progress_tps": (re.compile(r"t=(\d+)\s+RSS ([\d.]+) GB\s+hit ([\d.]+)%\s+([\d.]+) tok/s\s+([\d.]+) tok/fw"),
+ lambda m: {"tok":int(m.group(1)),"rss":float(m.group(2)),"hit":float(m.group(3)),
+ "tps":float(m.group(4)),"tpf":float(m.group(5))}),
+ # stderr: DSA, USAGE, KV startup lines
+ "usage_loaded": (re.compile(r"\[USAGE\].*?(\d+) selections"), lambda m: int(m.group(1))),
+ "kv_slots": (re.compile(r"\[KV\].*?(\d+) context slots"), lambda m: int(m.group(1))),
+}
+
+def extract_metrics(stdout: str, stderr: str) -> dict:
+ """Parse all metrics from the engine's stdout+stderr output."""
+ text_out = stdout or ""
+ text_err = stderr or ""
+ metrics = {}
+ # For most parsers we search BOTH streams (engine is inconsistent about which
+ # channel a given line lands on). Some are stream-specific (noted below).
+ combined = text_out + "\n" + text_err
+ for name, (rx, fn) in RX.items():
+ m = rx.search(combined)
+ if m:
+ try: metrics[name] = fn(m)
+ except (ValueError, IndexError): pass
+ # TOKENS dump is stderr-only and may appear once; grab it explicitly
+ m = RX["tokens_dump"][0].search(text_err)
+ if m:
+ try: metrics["tokens_dump"] = RX["tokens_dump"][1](m)
+ except: pass
+ # Multiple CUDA devices β€” collect all
+ cuda_devs = []
+ for m in re.finditer(r"\[CUDA\] device (\d+): (.*?), ([\d.]+) GB VRAM, sm_(\d)(\d)", text_err):
+ cuda_devs.append({"id":int(m.group(1)),"name":m.group(2).strip(),
+ "vram_gb":float(m.group(3)),"sm":f"{m.group(4)}.{m.group(5)}"})
+ if cuda_devs: metrics["cuda_devices"] = cuda_devs
+ # Multiple progress checkpoints β€” collect the full curve
+ progress = []
+ for m in RX["progress_tps"][0].finditer(text_err):
+ try:
+ progress.append(RX["progress_tps"][1](m))
+ except: pass
+ if progress: metrics["progress_curve"] = progress
+ # Multiple PROFILE lines β€” there are two (prefill + decode). Keep both.
+ prof_lines = [(i, line) for i, line in enumerate(text_out.splitlines()) if line.startswith("PROFILE:")]
+ for idx, (line_no, line) in enumerate(prof_lines):
+ label = "prefill_profile" if idx == 0 else "decode_profile"
+ p = {}
+ for pname, (rx, fn) in RX.items():
+ if not pname.startswith("prof_"): continue
+ mm = rx.search(line)
+ if mm:
+ try: p[pname] = fn(mm)
+ except: pass
+ if p: metrics[label] = p
+ return metrics
+
+
+# ---------------------------------------------------------------------------
+# ENGINE RUNNER β€” subprocess wrapper with timeout, signal handling, logging.
+# ---------------------------------------------------------------------------
+class EngineRunner:
+ def __init__(self, glm_path, snap, out_dir, default_env=None, timeout=600):
+ self.glm = str(glm_path)
+ self.snap = str(snap)
+ self.out_dir = Path(out_dir)
+ self.out_dir.mkdir(parents=True, exist_ok=True)
+ self.default_env = default_env or {}
+ self.timeout = timeout
+ self.default_cap = 75 # production default (matches bench_full.sh)
+
+ def run_prompt(self, prompt, ngen=64, env_extra=None, log_name=None, cap=None):
+ """Run the engine in PROMPT mode and return (stdout, stderr, returncode, elapsed)."""
+ env = dict(os.environ, SNAP=self.snap, PROMPT=prompt, NGEN=str(ngen))
+ env.update(self.default_env)
+ if env_extra: env.update(env_extra)
+ env["TOKENS"] = "1" # always capture token ids for reliable text extraction
+ cmd = [self.glm, str(cap if cap is not None else self.default_cap)]
+ return self._exec(cmd, env, log_name)
+
+ def run_score(self, score_file, cap=None, env_extra=None, log_name=None):
+ """Run the engine in SCORE (log-likelihood) mode."""
+ env = dict(os.environ, SNAP=self.snap, SCORE=score_file)
+ env.update(self.default_env)
+ if env_extra: env.update(env_extra)
+ cmd = [self.glm, str(cap if cap is not None else self.default_cap)]
+ return self._exec(cmd, env, log_name)
+
+ def _exec(self, cmd, env, log_name):
+ t0 = time.time()
+ log_path = self.out_dir / (log_name or f"run_{int(t0)}.log")
+ try:
+ proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ text=True, encoding="utf-8", errors="replace")
+ try:
+ stdout, stderr = proc.communicate(timeout=self.timeout)
+ rc = proc.returncode
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ stdout, stderr = proc.communicate()
+ rc = -1
+ stderr = (stderr or "") + f"\n[TIMEOUT after {self.timeout}s]\n"
+ except Exception as e:
+ stdout, stderr, rc = "", f"[EXCEPTION] {e}\n{traceback.format_exc()}", -2
+ elapsed = time.time() - t0
+ # Write raw log (both streams, clearly delimited)
+ with open(log_path, "w", encoding="utf-8") as f:
+ f.write(f"=== CMD: {' '.join(cmd)}\n=== ELAPSED: {elapsed:.1f}s\n=== RC: {rc}\n\n")
+ f.write("--- STDOUT ---\n"); f.write(stdout or ""); f.write("\n")
+ f.write("--- STDERR ---\n"); f.write(stderr or ""); f.write("\n")
+ return stdout or "", stderr or "", rc, elapsed
+
+
+# ---------------------------------------------------------------------------
+# TEXT RECOVERY β€” decode the [TOKENS] ids back to text using the tokenizer.
+# Falls back to stdout extraction if tokenizer/tokens unavailable.
+# ---------------------------------------------------------------------------
+def recover_text(stdout, stderr, prompt, tokenizer=None):
+ """Extract generated text. Primary: TOKENS dump decoded. Fallback: stdout parse."""
+ # Method 1: decode TOKENS ids via tokenizer
+ if tokenizer:
+ m = re.search(r"^\[TOKENS\] \d+ generated:(.*)$", stderr, re.MULTILINE)
+ if m:
+ ids = [int(x) for x in m.group(1).split()]
+ if ids:
+ try:
+ return tokenizer.decode(ids), "tokens"
+ except Exception: pass
+ # Method 2: stdout β€” text is between the prompt string and "PROFILO" or "\n---"
+ text = stdout or ""
+ # Find the prompt in stdout, take everything after it up to PROFILO
+ idx = text.find(prompt)
+ if idx >= 0:
+ after = text[idx + len(prompt):]
+ cut = after.find("PROFILO")
+ if cut >= 0:
+ return after[:cut].strip(), "stdout"
+ cut = after.find("\n---")
+ if cut >= 0:
+ return after[:cut].strip(), "stdout"
+ return "", "none"
+
+
+# ---------------------------------------------------------------------------
+# CORRUPTION / COHERENCE CHECKS
+# ---------------------------------------------------------------------------
+def check_repetition(token_ids):
+ """Detect degenerate repetition: same token 3+ consecutive times."""
+ if not token_ids or len(token_ids) < 6:
+ return False, 0
+ max_run = 1; cur = 1
+ for i in range(1, len(token_ids)):
+ if token_ids[i] == token_ids[i-1]: cur += 1; max_run = max(max_run, cur)
+ else: cur = 1
+ return max_run >= 3, max_run
+
+def check_expected(text, expect):
+ """Check if expected substring appears case-insensitively in the first 200 chars."""
+ if not expect or not text: return None
+ return expect.lower() in text[:200].lower()
+
+
+# ---------------------------------------------------------------------------
+# PHASES
+# ---------------------------------------------------------------------------
+class DiagnosticHarness:
+ def __init__(self, args):
+ self.args = args
+ self.snap = args.snap
+ self.glm = args.glm
+ ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+ self.out_dir = Path(args.out or f"./diag_results/{ts}")
+ self.out_dir.mkdir(parents=True, exist_ok=True)
+ # Base env for all runs
+ base_env = {"TEMP": "0", "PIPE": "1", "PIPE_WORKERS": "8", "DIRECT": "1"}
+ if args.ram: base_env["RAM_GB"] = str(args.ram)
+ if args.cuda:
+ base_env.update({
+ "COLI_CUDA": "1",
+ "COLI_GPU": str(args.gpu) if args.gpu is not None else "0",
+ "CUDA_DENSE": "1",
+ "COLI_CUDA_ATTN": "1",
+ "COLI_CUDA_PIPE": "2",
+ "COLI_CUDA_PIPE_S_MIN": "1",
+ })
+ self.runner = EngineRunner(self.glm, self.snap, self.out_dir, base_env, timeout=args.timeout)
+ self.runner.default_cap = args.cap
+ # Load tokenizer for text recovery
+ self.tokenizer = None
+ tok_path = os.path.join(self.snap, "tokenizer.json")
+ if os.path.exists(tok_path):
+ try:
+ from tokenizers import Tokenizer
+ self.tokenizer = Tokenizer.from_file(tok_path)
+ except Exception as e:
+ print(f"[warn] could not load tokenizer: {e}", file=sys.stderr)
+ self.results = {"meta": {"snap": self.snap, "glm": self.glm,
+ "timestamp": ts, "args": vars(args)},
+ "phases": {}}
+
+ def phase_system(self):
+ """Phase 0: minimal run to capture startup telemetry."""
+ print("\n" + "="*60 + "\nPHASE 0: SYSTEM PROBE\n" + "="*60)
+ stdout, stderr, rc, elapsed = self.runner.run_prompt(
+ "Hello", ngen=1, log_name="system_probe.log")
+ metrics = extract_metrics(stdout, stderr)
+ result = {
+ "rc": rc, "elapsed": elapsed,
+ "load_secs": metrics.get("load_secs"),
+ "resident_mb": metrics.get("resident_mb"),
+ "n_layers": metrics.get("n_layers"),
+ "n_experts": metrics.get("n_experts"),
+ "mtp_status": metrics.get("mtp_status"),
+ "idot_kernel": metrics.get("idot_kernel"),
+ "cache_cap_requested": metrics.get("cache_cap"),
+ "ram_budget_gb": metrics.get("ram_budget"),
+ "cap_lowered": metrics.get("cap_lowered"),
+ "cap_raised": metrics.get("cap_raised"),
+ "cap_final": metrics.get("cap_ok"),
+ "cuda_devices": metrics.get("cuda_devices", []),
+ "cuda_mode": metrics.get("cuda_mode"),
+ }
+ # Print summary
+ print(f" load time: {result['load_secs']:.2f}s" if result['load_secs'] else " load time: FAILED")
+ print(f" resident: {result['resident_mb']:.0f} MB" if result['resident_mb'] else "")
+ print(f" layers/exp: {result['n_layers']}/{result['n_experts']}" if result['n_layers'] else "")
+ print(f" MTP: {result['mtp_status']}")
+ print(f" idot kernel: {result['idot_kernel']}")
+ print(f" RAM budget: {result['ram_budget_gb']} GB" if result['ram_budget_gb'] else " RAM budget: auto")
+ if result['cap_lowered']:
+ print(f" cache cap: {result['cap_lowered'][0]} -> {result['cap_lowered'][1]} (RAM-lowered)")
+ elif result['cap_final']:
+ print(f" cache cap: {result['cap_final']} (ok)")
+ else:
+ print(f" cache cap: {result['cache_cap_requested']}")
+ for d in result['cuda_devices']:
+ print(f" GPU {d['id']}: {d['name']}, {d['vram_gb']:.1f} GB, sm_{d['sm']}")
+ if rc != 0 and rc != -1:
+ print(f" WARNING: engine returned rc={rc}")
+ self.results["phases"]["system"] = result
+ return result
+
+ def phase_smoke(self):
+ """Phase 1: correctness smoke test across curated prompts."""
+ print("\n" + "="*60 + "\nPHASE 1: CORRECTNESS SMOKE TEST\n" + "="*60)
+ ngen = self.args.ngen
+ prompt_results = []
+ pass_count = 0; total = len(PROMPTS)
+ for p in PROMPTS:
+ stdout, stderr, rc, elapsed = self.runner.run_prompt(
+ p["prompt"], ngen=ngen, log_name=f"smoke_{p['id']}.log")
+ metrics = extract_metrics(stdout, stderr)
+ token_ids = metrics.get("tokens_dump", [])
+ text, method = recover_text(stdout, stderr, p["prompt"], self.tokenizer)
+ is_rep, max_run = check_repetition(token_ids)
+ expect_ok = check_expected(text, p.get("expect"))
+ # Verdict: pass if text is non-empty, no severe repetition, and (if factual) expected found
+ has_text = bool(text.strip())
+ ok = has_text and not is_rep
+ if p.get("expect") and expect_ok is False: ok = False
+ if ok: pass_count += 1
+ # Diagnose failure mode for reporting
+ if not has_text:
+ fail_reason = "no tokens generated (immediate EOS)"
+ elif is_rep:
+ fail_reason = f"repetition loop (max run={max_run})"
+ elif p.get("expect") and expect_ok is False:
+ fail_reason = f"expected '{p['expect']}' not found"
+ else:
+ fail_reason = ""
+ prompt_results.append({
+ "id": p["id"], "cat": p["cat"], "prompt": p["prompt"],
+ "expect": p.get("expect"), "generated": text[:300],
+ "expect_match": expect_ok, "repetition": is_rep, "max_run": max_run,
+ "toks_generated": len(token_ids), "decode_tps": metrics.get("decode_tps"),
+ "hit_rate": metrics.get("hit_rate"), "rc": rc, "elapsed": elapsed,
+ "text_method": method, "pass": ok, "fail_reason": fail_reason,
+ })
+ status = "PASS" if ok else "FAIL"
+ extra = f" expect={'Y' if expect_ok else ('N' if expect_ok is False else '-')}" if p.get("expect") else ""
+ tps = f" {metrics.get('decode_tps',0):.2f}t/s" if metrics.get("decode_tps") else ""
+ print(f" [{status}] {p['id']:<22} ({p['cat']:<11}) rep={'Y' if is_rep else 'N'}{extra}{tps}")
+ if text:
+ preview = text.replace("\n", " ")[:80]
+ print(f" -> {preview}")
+ elif rc != 0:
+ print(f" -> [engine rc={rc}]")
+ else:
+ print(f" -> [{fail_reason}]")
+ result = {"prompts": prompt_results, "pass": pass_count, "total": total,
+ "pass_rate": 100.0 * pass_count / total if total else 0}
+ print(f"\n SMOKE SUMMARY: {pass_count}/{total} passed ({result['pass_rate']:.0f}%)")
+ self.results["phases"]["smoke"] = result
+ return result
+
+ def phase_diagnostic(self):
+ """Phase 2: single deep-instrumented run β€” full PROFILE + routing + MTP."""
+ print("\n" + "="*60 + "\nPHASE 2: FULL SYSTEM DIAGNOSTIC\n" + "="*60)
+ diag_env = {
+ "LOOKA": "1", "DISK_SPLIT": "1", "ROUTE_AGREE": "1",
+ }
+ if self.args.cuda:
+ diag_env["COLI_CUDA_PROFILE"] = "1"
+ prompt = ("Write a short paragraph explaining how photosynthesis works. "
+ "Include the roles of sunlight, water, and carbon dioxide.")
+ stdout, stderr, rc, elapsed = self.runner.run_prompt(
+ prompt, ngen=self.args.ngen, env_extra=diag_env, log_name="diagnostic.log")
+ metrics = extract_metrics(stdout, stderr)
+ text, _ = recover_text(stdout, stderr, prompt, self.tokenizer)
+ result = {
+ "rc": rc, "elapsed": elapsed,
+ "generated_text": text[:500],
+ "prefill_profile": metrics.get("prefill_profile", {}),
+ "decode_profile": metrics.get("decode_profile", {}),
+ "decode_tps": metrics.get("decode_tps"),
+ "prefill_secs": metrics.get("prefill_secs"),
+ "hit_rate": metrics.get("hit_rate"),
+ "rss_gb": metrics.get("rss_gb"),
+ "experts_per_tok": metrics.get("experts_per_tok"),
+ "mtp_accept": metrics.get("mtp_accept"),
+ "mtp_counts": metrics.get("mtp_acc_cnt"),
+ "spec_tok_per_fw": metrics.get("spec_tok_per_fw"),
+ "cuda_tier": metrics.get("cuda_tier"),
+ "progress_curve": metrics.get("progress_curve", []),
+ }
+ # Print the PROFILE breakdown
+ dp = result["decode_profile"]
+ print(f"\n DECODE TIMING BREAKDOWN (per-bucket, seconds):")
+ print(f" expert-disk: {dp.get('prof_expert_disk', '?'):>8}")
+ print(f" expert-matmul: {dp.get('prof_expert_mm', '?'):>8}")
+ print(f" attention: {dp.get('prof_attention', '?'):>8}")
+ print(f" lm_head: {dp.get('prof_lm_head', '?'):>8}")
+ print(f" other: {dp.get('prof_other', '?'):>8}")
+ print(f"\n PERFORMANCE:")
+ print(f" prefill: {result['prefill_secs']:.2f}s" if result['prefill_secs'] else " prefill: ?")
+ print(f" decode: {result['decode_tps']:.3f} tok/s" if result['decode_tps'] else " decode: ?")
+ print(f" hit rate: {result['hit_rate']:.1f}%" if result.get('hit_rate') is not None else " hit rate: ?")
+ print(f" RSS: {result['rss_gb']:.1f} GB" if result.get('rss_gb') else " RSS: ?")
+ print(f" exp/tok: {result['experts_per_tok']:.1f}" if result.get('experts_per_tok') else " exp/tok: ?")
+ print(f" MTP: {result['mtp_accept']:.0f}% accept" if result.get('mtp_accept') is not None else " MTP: ?")
+ if result.get('cuda_tier'):
+ ct = result['cuda_tier']
+ print(f" CUDA tier: {ct['resident']} experts, {ct['vram_gb']:.1f} GB VRAM")
+ # Show generated text preview
+ if text:
+ print(f"\n GENERATED TEXT (first 200 chars):")
+ print(f" {text[:200].replace(chr(10), ' ')}")
+ else:
+ print(f"\n GENERATED TEXT: [none recovered]")
+ self.results["phases"]["diagnostic"] = result
+ return result
+
+ def phase_quality(self):
+ """Phase 3: benchmark accuracy via eval_glm.py SCORE mode."""
+ print("\n" + "="*60 + "\nPHASE 3: QUALITY BENCHMARKS\n" + "="*60)
+ eval_script = os.path.join(os.path.dirname(__file__), "eval_glm.py")
+ bench_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "bench")
+ if not os.path.exists(eval_script):
+ print(f" [SKIP] eval_glm.py not found at {eval_script}")
+ self.results["phases"]["quality"] = {"error": "eval_glm.py not found"}
+ return None
+ tasks = ["hellaswag", "arc_challenge", "mmlu"]
+ missing = [t for t in tasks if not os.path.exists(os.path.join(bench_dir, f"{t}.jsonl"))]
+ if missing:
+ print(f" [SKIP] benchmark data missing: {missing}")
+ print(f" run: python tools/fetch_benchmarks.py --out {bench_dir}")
+ self.results["phases"]["quality"] = {"error": f"missing benchmark data: {missing}"}
+ return None
+ limit = self.args.quality_limit
+ py = sys.executable
+ cmd = [py, eval_script, "--snap", self.snap, "--glm", self.glm,
+ "--data", bench_dir, "--tasks", ",".join(tasks), "--limit", str(limit)]
+ env = dict(os.environ)
+ if self.args.ram: env["RAM_GB"] = str(self.args.ram)
+ print(f" Running eval_glm.py (tasks={tasks}, n={limit})...")
+ print(f" This takes ~{limit*3*4/0.05:.0f}s at 0.05 tok/s (worst case)...")
+ t0 = time.time()
+ log_path = self.out_dir / "quality_eval.log"
+ try:
+ with open(log_path, "w", encoding="utf-8") as logf:
+ proc = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=self.args.timeout*3,
+ encoding="utf-8", errors="replace")
+ logf.write(proc.stdout); logf.write("\n--- STDERR ---\n"); logf.write(proc.stderr)
+ elapsed = time.time() - t0
+ # Parse the acc/acc_norm table from eval_glm.py output
+ scores = {}
+ for line in proc.stdout.splitlines():
+ # lines like: "hellaswag 40 45.0% 50.0%"
+ m = re.match(r"(\w+)\s+(\d+)\s+([\d.]+)%\s+([\d.]+)%", line.strip())
+ if m:
+ scores[m.group(1)] = {"n": int(m.group(2)), "acc": float(m.group(3)),
+ "acc_norm": float(m.group(4))}
+ mean_m = re.search(r"MEAN acc_norm:\s*([\d.]+)%", proc.stdout)
+ result = {"scores": scores, "mean_acc_norm": float(mean_m.group(1)) if mean_m else None,
+ "elapsed": elapsed, "rc": proc.returncode}
+ print(f" Completed in {elapsed:.0f}s\n")
+ print(f" {'task':<18} {'n':>4} {'acc':>7} {'acc_norm':>9}")
+ for t, s in scores.items():
+ print(f" {t:<18} {s['n']:>4} {s['acc']:>6.1f}% {s['acc_norm']:>8.1f}%")
+ if result["mean_acc_norm"] is not None:
+ print(f"\n MEAN acc_norm: {result['mean_acc_norm']:.1f}%")
+ except subprocess.TimeoutExpired:
+ result = {"error": f"eval timed out after {self.args.timeout*3}s"}
+ print(f" [TIMEOUT] eval_glm.py exceeded {self.args.timeout*3}s")
+ except Exception as e:
+ result = {"error": str(e)}
+ print(f" [ERROR] {e}")
+ self.results["phases"]["quality"] = result
+ return result
+
+ def phase_throughput(self):
+ """Phase 4: tok/s comparison β€” MTP on vs off."""
+ print("\n" + "="*60 + "\nPHASE 4: THROUGHPUT BENCHMARK\n" + "="*60)
+ prompt = "Summarize the plot of Romeo and Juliet in three sentences."
+ ngen = self.args.ngen
+ results = {}
+ # Run 1: with MTP (default)
+ print(f" [1/2] MTP ON (draft=3)...")
+ stdout, stderr, rc, t = self.runner.run_prompt(
+ prompt, ngen=ngen, log_name="throughput_mtp_on.log")
+ m_on = extract_metrics(stdout, stderr)
+ results["mtp_on"] = {"tps": m_on.get("decode_tps"), "hit": m_on.get("hit_rate"),
+ "mtp_accept": m_on.get("mtp_accept"), "elapsed": t}
+ print(f" {m_on.get('decode_tps',0):.3f} tok/s | hit {m_on.get('hit_rate',0):.1f}% | "
+ f"MTP {m_on.get('mtp_accept',0):.0f}%")
+ # Run 2: MTP off
+ print(f" [2/2] MTP OFF (MTP=0)...")
+ stdout, stderr, rc, t = self.runner.run_prompt(
+ prompt, ngen=ngen, env_extra={"MTP": "0"}, log_name="throughput_mtp_off.log")
+ m_off = extract_metrics(stdout, stderr)
+ results["mtp_off"] = {"tps": m_off.get("decode_tps"), "hit": m_off.get("hit_rate"),
+ "elapsed": t}
+ print(f" {m_off.get('decode_tps',0):.3f} tok/s | hit {m_off.get('hit_rate',0):.1f}%")
+ # Compute speedup
+ if results["mtp_on"]["tps"] and results["mtp_off"]["tps"] and results["mtp_off"]["tps"] > 0:
+ sp = results["mtp_on"]["tps"] / results["mtp_off"]["tps"]
+ results["mtp_speedup"] = sp
+ print(f"\n MTP speedup: {sp:.2f}x ({'MTP helps' if sp > 1.05 else 'MTP hurts' if sp < 0.95 else 'no effect'})")
+ else:
+ print(f"\n MTP speedup: (insufficient data)")
+ self.results["phases"]["throughput"] = results
+ return results
+
+ def write_report(self):
+ """Write report.json and report.md."""
+ ts = self.results["meta"]["timestamp"]
+ json_path = self.out_dir / "report.json"
+ md_path = self.out_dir / "report.md"
+ # JSON
+ with open(json_path, "w", encoding="utf-8") as f:
+ json.dump(self.results, f, indent=2, default=str)
+ # Markdown
+ lines = [
+ f"# Diagnostic Report β€” {ts}",
+ f"",
+ f"**Model:** `{self.snap}`",
+ f"**Engine:** `{self.glm}`",
+ f"**Date:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
+ f"",
+ ]
+ phases = self.results["phases"]
+ # System
+ if "system" in phases:
+ s = phases["system"]
+ lines += ["## Phase 0: System Probe", ""]
+ lines += [f"- Load time: **{s.get('load_secs','?')}s**"]
+ lines += [f"- Layers/experts: {s.get('n_layers','?')}/{s.get('n_experts','?')}"]
+ lines += [f"- MTP: {s.get('mtp_status','?')}"]
+ lines += [f"- idot kernel: `{s.get('idot_kernel','?')}`"]
+ lines += [f"- RAM budget: {s.get('ram_budget_gb','auto')} GB"]
+ if s.get("cap_lowered"):
+ lines += [f"- Cache cap: {s['cap_lowered'][0]}β†’{s['cap_lowered'][1]} (RAM-lowered)"]
+ elif s.get("cap_final"):
+ lines += [f"- Cache cap: {s['cap_final']}"]
+ for d in s.get("cuda_devices", []):
+ lines += [f"- GPU {d['id']}: {d['name']}, {d['vram_gb']:.1f} GB, sm_{d['sm']}"]
+ lines.append("")
+ # Smoke
+ if "smoke" in phases:
+ sm = phases["smoke"]
+ lines += [f"## Phase 1: Correctness Smoke", "",
+ f"**{sm['pass']}/{sm['total']} prompts passed ({sm['pass_rate']:.0f}%)**", "",
+ "| ID | Category | Pass | Expect | Repetition | tok/s | Generated (first 80 chars) |",
+ "|---|---|---|---|---|---|---|"]
+ for p in sm["prompts"]:
+ gen = p["generated"][:80].replace("|", "\\|").replace("\n", " ") if p["generated"] else ""
+ exp = "Y" if p.get("expect_match") else ("N" if p.get("expect_match") is False else "-")
+ tps = f"{p.get('decode_tps',0):.2f}" if p.get("decode_tps") else "-"
+ lines.append(f"| {p['id']} | {p['cat']} | {'βœ…' if p['pass'] else '❌'} | {exp} | "
+ f"{'⚠️' if p['repetition'] else '-'} (run={p.get('max_run',0)}) | {tps} | {gen} |")
+ lines.append("")
+ # Diagnostic
+ if "diagnostic" in phases:
+ d = phases["diagnostic"]
+ lines += ["## Phase 2: Full System Diagnostic", ""]
+ dp = d.get("decode_profile", {})
+ lines += ["### Decode Timing Breakdown", "",
+ "| Bucket | Seconds |", "|---|---|"]
+ for k, label in [("prof_expert_disk","expert-disk"),("prof_expert_mm","expert-matmul"),
+ ("prof_attention","attention"),("prof_lm_head","lm_head"),
+ ("prof_other","other")]:
+ v = dp.get(k, "?")
+ lines.append(f"| {label} | {v} |")
+ lines += [f"", f"### Performance", f"- Decode: **{d.get('decode_tps','?')} tok/s**",
+ f"- Prefill: {d.get('prefill_secs','?')}s",
+ f"- Expert hit rate: {d.get('hit_rate','?')}%",
+ f"- RSS: {d.get('rss_gb','?')} GB",
+ f"- Experts/token: {d.get('experts_per_tok','?')}",
+ f"- MTP acceptance: {d.get('mtp_accept','?')}%", ""]
+ if d.get("generated_text"):
+ lines += [f"### Generated Text", f"```\n{d['generated_text'][:300]}\n```", ""]
+ # Quality
+ if "quality" in phases:
+ q = phases["quality"]
+ lines += ["## Phase 3: Quality Benchmarks", ""]
+ if "error" in q:
+ lines += [f"⚠️ {q['error']}", ""]
+ else:
+ lines += [f"**Mean acc_norm: {q.get('mean_acc_norm','?')}%**", "",
+ "| Task | n | acc | acc_norm |", "|---|---|---|---|"]
+ for t, s in q.get("scores", {}).items():
+ lines.append(f"| {t} | {s['n']} | {s['acc']:.1f}% | {s['acc_norm']:.1f}% |")
+ lines.append("")
+ # Throughput
+ if "throughput" in phases:
+ th = phases["throughput"]
+ lines += ["## Phase 4: Throughput", "",
+ "| Mode | tok/s | hit% | MTP accept |", "|---|---|---|---|"]
+ on = th.get("mtp_on", {}); off = th.get("mtp_off", {})
+ lines.append(f"| MTP ON | {on.get('tps','?')} | {on.get('hit','?')}% | {on.get('mtp_accept','?')}% |")
+ lines.append(f"| MTP OFF | {off.get('tps','?')} | {off.get('hit','?')}% | β€” |")
+ if th.get("mtp_speedup"):
+ lines.append(f"\n**MTP speedup: {th['mtp_speedup']:.2f}x**")
+ lines.append("")
+ with open(md_path, "w", encoding="utf-8") as f:
+ f.write("\n".join(lines))
+ print(f"\n{'='*60}")
+ print(f"Report written:")
+ print(f" JSON: {json_path}")
+ print(f" MD: {md_path}")
+ print(f" Logs: {self.out_dir}/*.log")
+ print(f"{'='*60}")
+
+ def run(self):
+ phase = self.args.phase
+ if phase == "all":
+ self.phase_system()
+ self.phase_smoke()
+ self.phase_diagnostic()
+ self.phase_quality()
+ self.phase_throughput()
+ elif phase == "system": self.phase_system()
+ elif phase == "smoke": self.phase_smoke()
+ elif phase == "diagnostic": self.phase_diagnostic()
+ elif phase == "quality": self.phase_quality()
+ elif phase == "throughput": self.phase_throughput()
+ else:
+ print(f"Unknown phase: {phase}", file=sys.stderr); sys.exit(1)
+ self.write_report()
+
+
+def main():
+ ap = argparse.ArgumentParser(description="Comprehensive model diagnostic harness for colibri GLM-5.2")
+ ap.add_argument("--snap", required=True, help="model snapshot directory")
+ ap.add_argument("--glm", default=None, help="engine binary path (default: ./glm.exe or ./glm)")
+ ap.add_argument("--phase", default="all",
+ choices=["all","system","smoke","diagnostic","quality","throughput"],
+ help="which test phase to run")
+ ap.add_argument("--out", default=None, help="output directory (default: ./diag_results/<timestamp>)")
+ ap.add_argument("--ngen", type=int, default=64, help="generation length for smoke/throughput (default 64)")
+ ap.add_argument("--quality-limit", type=int, default=40, help="questions per benchmark task (default 40)")
+ ap.add_argument("--ram", type=float, default=0, help="RAM_GB override (0=auto)")
+ ap.add_argument("--cuda", action="store_true", help="enable COLI_CUDA GPU tier")
+ ap.add_argument("--gpu", type=int, default=None, help="GPU device ordinal (with --cuda)")
+ ap.add_argument("--cap", type=int, default=75, help="experts-per-layer cache cap (default 75)")
+ ap.add_argument("--timeout", type=int, default=600, help="per-run timeout in seconds (default 600)")
+ a = ap.parse_args()
+ # Auto-detect engine binary
+ if a.glm is None:
+ for cand in ["./glm.exe", "./glm", "../glm.exe", os.path.join(os.path.dirname(__file__), "..", "glm.exe")]:
+ if os.path.exists(cand): a.glm = os.path.abspath(cand); break
+ if a.glm is None:
+ print("ERROR: could not find glm/glm.exe. Specify with --glm", file=sys.stderr); sys.exit(1)
+ if not os.path.isdir(a.snap):
+ print(f"ERROR: snapshot dir does not exist: {a.snap}", file=sys.stderr); sys.exit(1)
+ harness = DiagnosticHarness(a)
+ harness.run()
+
+if __name__ == "__main__":
+ main()