Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
/* idot.h -- integer dot-product kernels shared by every engine.
*
* The activation is quantized to int8 once (one scale per row, amax/127,
* qrow_i8) and the weights, int8 rows or int4 planar blocks with per-row or
* per-group scales, are multiplied with integer instructions: maddubs on
* AVX2, vpdpbusd on AVX-VNNI and AVX-512 VNNI, sdot/smmla on NEON, and the
* AMX tile kernel where the CPU has it. Exact int32 sums, scaled once per
* output, so every ISA path is bit-identical to the scalar reference.
*
* Lived inside quant.h; moved here so an engine that has its own dense
* kernels (qwen36.c uses qgemv.h/gsgemv.h, whose matmul_q would collide
* with quant.h's) can still take the integer path for its dense trunk.
* quant.h includes this file at the top, so its other consumers see the
* same declarations in the same place as before. */
#ifndef COLIBRI_IDOT_H
#define COLIBRI_IDOT_H
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdio.h>
/* ---- SIMD includes -------------------------------------------------------- */
#ifdef __AVX2__
#include <immintrin.h>
static inline float hsum256(__m256 v){
__m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1);
lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh);
sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo);
}
static inline int hsum256_i32(__m256i v){
__m128i lo=_mm256_castsi256_si128(v), hi=_mm256_extracti128_si256(v,1);
lo=_mm_add_epi32(lo,hi); lo=_mm_hadd_epi32(lo,lo); lo=_mm_hadd_epi32(lo,lo);
return _mm_cvtsi128_si32(lo);
}
#endif
#if defined(__AVXVNNI__) && defined(__AVX2__)
static inline int hsum128_i32(__m128i v){
v=_mm_hadd_epi32(v,v); v=_mm_hadd_epi32(v,v); return _mm_cvtsi128_si32(v);
}
#endif
#ifdef __ARM_NEON
#include <arm_neon.h>
#endif
#ifdef __VSX__
#include <altivec.h>
#undef vector
#undef pixel
#undef bool
#endif
/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
#define IDOT_KERNEL "avx512-vnni"
#elif defined(__AVXVNNI__) && defined(__AVX2__)
#define IDOT_KERNEL "avx-vnni"
#elif defined(__AVX2__)
#define IDOT_KERNEL "avx2"
#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
#define IDOT_KERNEL "neon-i8mm"
#elif defined(__ARM_NEON)
#define IDOT_KERNEL "neon"
#elif defined(__VSX__)
#define IDOT_KERNEL "vsx"
#else
#define IDOT_KERNEL "scalar"
#endif
static inline float qrow_i8(const float *x, int8_t *q, int I){
float amax=0; for(int i=0;i<I;i++){ float a=fabsf(x[i]); if(a>amax)amax=a; }
float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s;
for(int i=0;i<I;i++) q[i]=(int8_t)lrintf(x[i]*inv);
return s;
}
/* Activation -> int8 with one scale (shared by every engine whose trunk
* meets the integer kernels below), plus the int32 sum of every block of 64
* (the K1b kernel subtracts 8*sum per group because its nibbles are unsigned).
* Vectorized: the scalar lrintf loop was measured at ~7 us for 4096 values,
* which times ~150 GEMVs per token is a millisecond thrown away. Rounding is
* to nearest even in both paths, so the vector path equals qrow_i8 bit for bit. */
static float dense_act_i8(const float *x, int I, int8_t *xq, int32_t *xsg){
float amax = 0.f;
int i = 0;
#ifdef __AVX2__
{
__m256 am = _mm256_setzero_ps();
const __m256 sign = _mm256_set1_ps(-0.0f);
for (; i + 8 <= I; i += 8) am = _mm256_max_ps(am, _mm256_andnot_ps(sign, _mm256_loadu_ps(x + i)));
float tmp[8]; _mm256_storeu_ps(tmp, am);
for (int k = 0; k < 8; k++) if (tmp[k] > amax) amax = tmp[k];
}
#endif
for (; i < I; i++) { float a = fabsf(x[i]); if (a > amax) amax = a; }
float s = amax / 127.f; if (s < 1e-12f) s = 1e-12f;
float inv = 1.f / s;
i = 0;
#ifdef __AVX2__
{
const __m256 vinv = _mm256_set1_ps(inv);
for (; i + 32 <= I; i += 32) {
__m256i a = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i), vinv));
__m256i b = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i + 8), vinv));
__m256i c = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i + 16), vinv));
__m256i d = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + i + 24), vinv));
/* packs interleave 128-bit lanes: fix the order with one permute */
__m256i ab = _mm256_packs_epi32(a, b), cd = _mm256_packs_epi32(c, d);
__m256i abcd = _mm256_packs_epi16(ab, cd);
abcd = _mm256_permutevar8x32_epi32(abcd, _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7));
_mm256_storeu_si256((__m256i *)(xq + i), abcd);
}
}
#endif
for (; i < I; i++) xq[i] = (int8_t)lrintf(x[i] * inv);
if (xsg) {
int ng = I / 64;
for (int g = 0; g < ng; g++) {
int32_t sum = 0;
for (int k = 0; k < 64; k++) sum += xq[g * 64 + k];
xsg[g] = sum;
}
}
return s;
}
/* dot int8*int8 */
static inline int32_t dot_i8i8(const int8_t *w, const int8_t *x, int I){
int32_t sum=0; int i=0;
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
__m512i acc=_mm512_setzero_si512();
for(;i+64<=I;i+=64){
__m512i wv=_mm512_loadu_si512((const void*)(w+i));
__m512i xv=_mm512_loadu_si512((const void*)(x+i));
__mmask64 neg=_mm512_movepi8_mask(wv);
__m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv);
acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs);
}
sum=_mm512_reduce_add_epi32(acc);
#elif defined(__AVXVNNI__) && defined(__AVX2__)
/* 4 accumulatori indipendenti (64 byte/iter): un solo acc incatena i vpdpbusd
* (latenza-bound ~5c). Somme intere associative -> bit-identico. Stessa struttura
* dei 4 accumulatori del ramo NEON piu' sotto.
* EN: four independent accumulators break the serial vpdpbusd->acc chain; integer
* adds are associative, so the result is bit-identical (mirrors the NEON path). */
__m128i a0=_mm_setzero_si128(),a1=_mm_setzero_si128(),a2=_mm_setzero_si128(),a3=_mm_setzero_si128();
for(;i+64<=I;i+=64){
__m128i w0=_mm_loadu_si128((const __m128i*)(w+i)), x0=_mm_loadu_si128((const __m128i*)(x+i));
__m128i w1=_mm_loadu_si128((const __m128i*)(w+i+16)), x1=_mm_loadu_si128((const __m128i*)(x+i+16));
__m128i w2=_mm_loadu_si128((const __m128i*)(w+i+32)), x2=_mm_loadu_si128((const __m128i*)(x+i+32));
__m128i w3=_mm_loadu_si128((const __m128i*)(w+i+48)), x3=_mm_loadu_si128((const __m128i*)(x+i+48));
a0=_mm_dpbusd_epi32(a0,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0));
a1=_mm_dpbusd_epi32(a1,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1));
a2=_mm_dpbusd_epi32(a2,_mm_abs_epi8(w2),_mm_sign_epi8(x2,w2));
a3=_mm_dpbusd_epi32(a3,_mm_abs_epi8(w3),_mm_sign_epi8(x3,w3));
}
__m128i acc=_mm_add_epi32(_mm_add_epi32(a0,a1),_mm_add_epi32(a2,a3));
for(;i+16<=I;i+=16){
__m128i wv=_mm_loadu_si128((const __m128i*)(w+i));
__m128i xv=_mm_loadu_si128((const __m128i*)(x+i));
acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),_mm_sign_epi8(xv,wv));
}
sum=hsum128_i32(acc);
#elif defined(__AVX2__)
__m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1);
for(;i+32<=I;i+=32){
__m256i wv=_mm256_loadu_si256((const __m256i*)(w+i));
__m256i xv=_mm256_loadu_si256((const __m256i*)(x+i));
__m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv));
acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones));
}
sum=hsum256_i32(acc);
#elif defined(__ARM_NEON)
#if defined(__ARM_FEATURE_DOTPROD)
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0);
for(;i+64<=I;i+=64){
a0=vdotq_s32(a0,vld1q_s8(w+i), vld1q_s8(x+i));
a1=vdotq_s32(a1,vld1q_s8(w+i+16),vld1q_s8(x+i+16));
a2=vdotq_s32(a2,vld1q_s8(w+i+32),vld1q_s8(x+i+32));
a3=vdotq_s32(a3,vld1q_s8(w+i+48),vld1q_s8(x+i+48));
}
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
for(;i+16<=I;i+=16) acc=vdotq_s32(acc,vld1q_s8(w+i),vld1q_s8(x+i));
sum=vaddvq_s32(acc);
#else
int32x4_t acc=vdupq_n_s32(0);
for(;i+16<=I;i+=16){
int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i);
int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv));
p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv));
acc=vpadalq_s16(acc,p);
}
sum=vaddvq_s32(acc);
#endif
#elif defined(__VSX__)
__vector signed int acc=vec_splats(0);
const __vector signed char vz=vec_splats((signed char)0);
for(;i+16<=I;i+=16){
__vector signed char wv=vec_xl(0,(const signed char*)(w+i));
__vector signed char xv=vec_xl(0,(const signed char*)(x+i));
__vector __bool char neg=vec_cmplt(wv,vz);
__vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg);
__vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg);
acc=vec_msum(xs,wa,acc);
}
sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3);
#endif
for(;i<I;i++) sum+=(int32_t)w[i]*x[i];
return sum;
}
/* dot int4(packed)*int8 */
static inline int32_t dot_i4i8(const uint8_t *w4, const int8_t *x, int I){
int32_t sum=0; int i=0;
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
const __m256i m4v=_mm256_set1_epi8(0x0F);
const __m512i b8v=_mm512_set1_epi8(8);
const __m512i xidx=_mm512_setr_epi64(0,1,4,5,2,3,6,7);
__m512i acc=_mm512_setzero_si512();
for(;i+64<=I;i+=64){
__m256i by=_mm256_loadu_si256((const __m256i*)(w4+(i>>1)));
__m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v);
__m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi);
__m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v);
__m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i)));
__mmask64 neg=_mm512_movepi8_mask(wv);
__m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv);
acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs);
}
sum=_mm512_reduce_add_epi32(acc);
#elif defined(__AVXVNNI__) && defined(__AVX2__)
/* 4 accumulatori indipendenti (64 elementi = 32 byte packed/iter): un solo acc
* incatena i vpdpbusd (latenza-bound ~5c). Somme intere associative -> bit-identico.
* Stessa struttura dei 4 accumulatori del ramo NEON piu' sotto.
* EN: four independent accumulators break the serial vpdpbusd->acc chain; integer
* adds are associative, so the result is bit-identical (mirrors the NEON path). */
const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8);
__m128i a0=_mm_setzero_si128(),a1=_mm_setzero_si128(),a2=_mm_setzero_si128(),a3=_mm_setzero_si128();
for(;i+64<=I;i+=64){
__m128i by0=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* elem i..i+31 */
__m128i by1=_mm_loadu_si128((const __m128i*)(w4+(i>>1)+16)); /* elem i+32..i+63 */
__m128i lo0=_mm_and_si128(by0,m4), hi0=_mm_and_si128(_mm_srli_epi16(by0,4),m4);
__m128i lo1=_mm_and_si128(by1,m4), hi1=_mm_and_si128(_mm_srli_epi16(by1,4),m4);
__m128i w0=_mm_sub_epi8(_mm_unpacklo_epi8(lo0,hi0),b8), w1=_mm_sub_epi8(_mm_unpackhi_epi8(lo0,hi0),b8);
__m128i w2=_mm_sub_epi8(_mm_unpacklo_epi8(lo1,hi1),b8), w3=_mm_sub_epi8(_mm_unpackhi_epi8(lo1,hi1),b8);
__m128i x0=_mm_loadu_si128((const __m128i*)(x+i)), x1=_mm_loadu_si128((const __m128i*)(x+i+16));
__m128i x2=_mm_loadu_si128((const __m128i*)(x+i+32)), x3=_mm_loadu_si128((const __m128i*)(x+i+48));
a0=_mm_dpbusd_epi32(a0,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0));
a1=_mm_dpbusd_epi32(a1,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1));
a2=_mm_dpbusd_epi32(a2,_mm_abs_epi8(w2),_mm_sign_epi8(x2,w2));
a3=_mm_dpbusd_epi32(a3,_mm_abs_epi8(w3),_mm_sign_epi8(x3,w3));
}
__m128i acc=_mm_add_epi32(_mm_add_epi32(a0,a1),_mm_add_epi32(a2,a3));
for(;i+32<=I;i+=32){ /* 32-nibble remainder: 2 dpbusd, same unpack */
__m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1)));
__m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i w0=_mm_sub_epi8(_mm_unpacklo_epi8(lo,hi),b8), w1=_mm_sub_epi8(_mm_unpackhi_epi8(lo,hi),b8);
__m128i x0=_mm_loadu_si128((const __m128i*)(x+i));
__m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16));
acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0));
acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1));
}
sum=hsum128_i32(acc);
#elif defined(__AVX2__)
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8);
const __m256i ones=_mm256_set1_epi16(1);
__m256i acc=_mm256_setzero_si256();
for(;i+32<=I;i+=32){
__m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1)));
__m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi);
__m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8);
__m256i xv=_mm256_loadu_si256((const __m256i*)(x+i));
__m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv));
acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones));
}
sum=hsum256_i32(acc);
#elif defined(__ARM_NEON)
const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8);
#if defined(__ARM_FEATURE_DOTPROD)
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0);
for(;i+64<=I;i+=64){
uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16);
uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4));
uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4));
a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i));
a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16));
a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32));
a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48));
}
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
for(;i+32<=I;i+=32){
uint8x16_t by=vld1q_u8(w4+(i>>1));
uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4));
acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i));
acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16));
}
sum=vaddvq_s32(acc);
#else
int32x4_t acc=vdupq_n_s32(0);
for(;i+32<=I;i+=32){
uint8x16_t by=vld1q_u8(w4+(i>>1));
uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4));
int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q);
int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q);
int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16);
int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0));
p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0));
acc=vpadalq_s16(acc,p);
p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1));
p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1));
acc=vpadalq_s16(acc,p);
}
sum=vaddvq_s32(acc);
#endif
#elif defined(__VSX__)
const __vector unsigned char m4v=vec_splats((unsigned char)0x0F);
const __vector unsigned char sh4=vec_splats((unsigned char)4);
const __vector signed char b8v=vec_splats((signed char)8);
const __vector signed char vz=vec_splats((signed char)0);
__vector signed int acc=vec_splats(0);
for(;i+32<=I;i+=32){
__vector unsigned char by=vec_xl(0,w4+(i>>1));
__vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4);
__vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v);
__vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v);
__vector signed char x0=vec_xl(0,(const signed char*)(x+i));
__vector signed char x1=vec_xl(0,(const signed char*)(x+i+16));
__vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz);
acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0),
(__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc);
acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1),
(__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc);
}
sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3);
#endif
for(;i+1<I;i+=2){ uint8_t b=w4[i>>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; }
if(i<I){ uint8_t b=w4[i>>1]; sum+=((int)(b&0xF)-8)*x[i]; }
return sum;
}
/* ---- ARM i8mm SMMLA tiled kernels ---------------------------------------- */
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1,
int8x16_t xs, int8x16_t xs1){
acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)),
vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1)));
return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)),
vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1)));
}
static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q,
const float *scale, int S, int I, int O){
#pragma omp parallel for schedule(static)
for(int o=0;o<(O&~1);o+=2){
const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I;
float sc0=scale[o], sc1=scale[o+1];
for(int s=0;s<(S&~1);s+=2){
const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I;
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0;
for(;i+64<=I;i+=64){
a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i));
a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16));
a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32));
a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48));
}
for(;i+16<=I;i+=16)
a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i));
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1);
int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3);
for(;i<I;i++){ int a=wo[i],b=wo1[i],u=xs[i],v=xs1[i];
d00+=a*u; d01+=a*v; d10+=b*u; d11+=b*v; }
y[(int64_t)s*O+o] =(float)d00*sc0*sx[s];
y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s];
y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1];
y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1];
}
if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I;
y[(int64_t)s*O+o] =(float)dot_i8i8(wo, xs,I)*sc0*sx[s];
y[(int64_t)s*O+(o+1)]=(float)dot_i8i8(wo1,xs,I)*sc1*sx[s]; }
}
if(O&1){ int o=O-1; const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
#pragma omp parallel for schedule(static)
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i8i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
}
static void matmul_i4_idot_mm(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
const float *scale, int S, int I, int O){
int rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for(int o=0;o<(O&~1);o+=2){
const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8);
const uint8_t *wo=q4+(int64_t)o*rb, *wo1=q4+(int64_t)(o+1)*rb;
float sc0=scale[o], sc1=scale[o+1];
for(int s=0;s<(S&~1);s+=2){
const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I;
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0;
for(;i+64<=I;i+=64){
uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1));
uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16);
uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4));
uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4));
uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4));
uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4));
a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q),
vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q),
vld1q_s8(xs+i), vld1q_s8(xs1+i));
a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q),
vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q),
vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16));
a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q),
vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q),
vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32));
a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q),
vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q),
vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48));
}
for(;i+32<=I;i+=32){
uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1));
uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4));
uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4));
a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q),
vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q),
vld1q_s8(xs+i), vld1q_s8(xs1+i));
a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q),
vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q),
vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16));
}
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1);
int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3);
for(;i+1<I;i+=2){ uint8_t bo=wo[i>>1], bo1=wo1[i>>1];
int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8;
int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1];
d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; }
if(i<I){ uint8_t bo=wo[i>>1], bo1=wo1[i>>1];
int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8;
d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; }
y[(int64_t)s*O+o] =(float)d00*sc0*sx[s];
y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s];
y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1];
y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1];
}
if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I;
y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s];
y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; }
}
if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
#pragma omp parallel for schedule(static)
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i4i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
}
#endif
/* ---- IDOT dispatch (int8-quantized activations) --------------------------- */
static void matmul_q_idot(float *y, const int8_t *xq, const float *sx, const int8_t *q,
const float *scale, int S, int I, int O){
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
if(S>=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; }
#endif
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i8i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
}
static void matmul_i4_idot(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
const float *scale, int S, int I, int O){
int rb=(I+1)/2;
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
if(S>=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; }
#endif
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i4i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
}
/* ================= K1: layout int4 A PIANI (fmt 2/4, opzionale) ==============
* Layout classico ("a coppie"): byte k = elementi (2k, 2k+1). Ogni kernel paga
* unpacklo/unpackhi per riordinare i nibble, e l'IDOT paga sub+abs+sign per
* forzare l'operando con segno dentro vpdpbusd (che e' nativamente u8 x s8).
*
* Layout A PIANI, per blocco di 64 elementi consecutivi (32 byte):
* byte k del blocco = (elem k + 8) | ((elem k+32 + 8) << 4) k = 0..31
* `and 0x0F` produce gli elementi 0..31 GIA' in ordine; `srli 4 + and` produce
* 32..63 in ordine: l'unpack sparisce. La coda (I mod 64) resta a coppie.
*
* L'algebra dell'IDOT: i nibble memorizzati sono u = v+8 (unsigned 0..15), e
* dot(v, x) = dot(u, x) - 8 * sum(x)
* dove sum(x) e' UN int32 per riga di attivazione, condiviso da ogni riga di
* output. vpdpbusd consuma (u8, s8) direttamente: via sub+abs+sign (3 op
* vettoriali per dpbusd). Misurato su questa famiglia: 1.87x L3-resident,
* 25.3 GB/s DRAM (al tetto), 0 differenze di bit.
* NB: allargare a 256 bit SENZA cambiare layout e' stato misurato 0.79-0.81x
* (il fixup di lane costa piu' della larghezza): il layout e' la precondizione
* della larghezza, non un'ottimizzazione indipendente.
*
* I kernel f32 planari preservano ESATTAMENTE l'ordine di accumulazione dei
* gemelli a coppie (stesse lane, stessa sequenza 0..63): bit-identici anche
* loro. EN: plane-nibble int4 layout. Low nibble of block byte k = element k,
* high = element k+32; the unpack disappears, and the stored-unsigned nibbles
* feed vpdpbusd natively via dot(v,x) = dot(u,x) - 8*sum(x). The f32 planar
* kernels keep the pair kernels' exact accumulation order: bit-identical. */
/* riordina in place una riga (o un tensore [O,rb] riga per riga) da coppie a
* piani; la coda I%64 resta a coppie. EN: in-place pair->planar repack. */
static void planarize_i4_row(uint8_t *row, int I){
uint8_t tmp[32];
int nb=I/64;
for(int b=0;b<nb;b++){
uint8_t *blk=row+b*32;
/* pair: byte j ha (elem 2j, elem 2j+1). planar: byte k = (elem k, elem k+32) */
for(int k=0;k<32;k++){
int src_lo=k, src_hi=k+32; /* elementi voluti */
uint8_t nib_lo=(blk[src_lo>>1]>>((src_lo&1)*4))&0xF;
uint8_t nib_hi=(blk[src_hi>>1]>>((src_hi&1)*4))&0xF;
tmp[k]=(uint8_t)(nib_lo|(nib_hi<<4));
}
memcpy(blk,tmp,32);
}
}
static void planarize_i4(uint8_t *q4, int O, int I){
int rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++) planarize_i4_row(q4+(int64_t)o*rb, I);
}
/* dot unsigned-nibble planare * int8: ritorna dot(u, x) SENZA la correzione
* -8*sum(x) (la applica il chiamante, una volta per riga di output).
* Coda I%64: nibble a coppie letti come unsigned, stessa identita'. */
#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))
#if defined(__AVXVNNI__) && !defined(__AVX512VNNI__)
#define coli_dpbusd256 _mm256_dpbusd_avx_epi32
#else
#define coli_dpbusd256 _mm256_dpbusd_epi32
#endif
#endif
static inline int32_t dot_i4p_u(const uint8_t *w4, const int8_t *x, int I){
int32_t sum=0; int i=0;
#if defined(coli_dpbusd256)
const __m256i m4=_mm256_set1_epi8(0x0F);
__m256i a0=_mm256_setzero_si256(),a1=_mm256_setzero_si256();
__m256i a2=_mm256_setzero_si256(),a3=_mm256_setzero_si256();
for(;i+128<=I;i+=128){
__m256i b0=_mm256_loadu_si256((const __m256i*)(w4+(i>>1)));
__m256i b1=_mm256_loadu_si256((const __m256i*)(w4+(i>>1)+32));
a0=coli_dpbusd256(a0,_mm256_and_si256(b0,m4),
_mm256_loadu_si256((const __m256i*)(x+i)));
a1=coli_dpbusd256(a1,_mm256_and_si256(_mm256_srli_epi16(b0,4),m4),
_mm256_loadu_si256((const __m256i*)(x+i+32)));
a2=coli_dpbusd256(a2,_mm256_and_si256(b1,m4),
_mm256_loadu_si256((const __m256i*)(x+i+64)));
a3=coli_dpbusd256(a3,_mm256_and_si256(_mm256_srli_epi16(b1,4),m4),
_mm256_loadu_si256((const __m256i*)(x+i+96)));
}
__m256i acc=_mm256_add_epi32(_mm256_add_epi32(a0,a1),_mm256_add_epi32(a2,a3));
for(;i+64<=I;i+=64){
__m256i b0=_mm256_loadu_si256((const __m256i*)(w4+(i>>1)));
acc=coli_dpbusd256(acc,_mm256_and_si256(b0,m4),
_mm256_loadu_si256((const __m256i*)(x+i)));
acc=coli_dpbusd256(acc,_mm256_and_si256(_mm256_srli_epi16(b0,4),m4),
_mm256_loadu_si256((const __m256i*)(x+i+32)));
}
sum=hsum256_i32(acc);
#elif defined(__AVX2__)
const __m256i m4=_mm256_set1_epi8(0x0F);
const __m256i ones=_mm256_set1_epi16(1);
__m256i acc=_mm256_setzero_si256();
for(;i+64<=I;i+=64){
__m256i b0=_mm256_loadu_si256((const __m256i*)(w4+(i>>1)));
/* maddubs(u8, s8): u<=15, |x|<=127 -> coppia <= 3810, int16 sicuro */
__m256i p0=_mm256_maddubs_epi16(_mm256_and_si256(b0,m4),
_mm256_loadu_si256((const __m256i*)(x+i)));
__m256i p1=_mm256_maddubs_epi16(_mm256_and_si256(_mm256_srli_epi16(b0,4),m4),
_mm256_loadu_si256((const __m256i*)(x+i+32)));
acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p0,ones));
acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p1,ones));
}
sum=hsum256_i32(acc);
#elif defined(__ARM_NEON)
int32x4_t acc=vdupq_n_s32(0);
for(;i+64<=I;i+=64){
uint8x16_t b0=vld1q_u8(w4+(i>>1)), b1=vld1q_u8(w4+(i>>1)+16);
int8x16_t lo0=vreinterpretq_s8_u8(vandq_u8(b0,vdupq_n_u8(0x0F)));
int8x16_t lo1=vreinterpretq_s8_u8(vandq_u8(b1,vdupq_n_u8(0x0F)));
int8x16_t hi0=vreinterpretq_s8_u8(vshrq_n_u8(b0,4));
int8x16_t hi1=vreinterpretq_s8_u8(vshrq_n_u8(b1,4));
#if defined(__ARM_FEATURE_DOTPROD)
acc=vdotq_s32(acc,lo0,vld1q_s8(x+i));
acc=vdotq_s32(acc,lo1,vld1q_s8(x+i+16));
acc=vdotq_s32(acc,hi0,vld1q_s8(x+i+32));
acc=vdotq_s32(acc,hi1,vld1q_s8(x+i+48));
#else
int8x16_t xs0=vld1q_s8(x+i), xs1=vld1q_s8(x+i+16);
int8x16_t xs2=vld1q_s8(x+i+32), xs3=vld1q_s8(x+i+48);
int16x8_t m;
m=vmull_s8(vget_low_s8(lo0),vget_low_s8(xs0)); acc=vpadalq_s16(acc,m);
m=vmull_s8(vget_high_s8(lo0),vget_high_s8(xs0)); acc=vpadalq_s16(acc,m);
m=vmull_s8(vget_low_s8(lo1),vget_low_s8(xs1)); acc=vpadalq_s16(acc,m);
m=vmull_s8(vget_high_s8(lo1),vget_high_s8(xs1)); acc=vpadalq_s16(acc,m);
m=vmull_s8(vget_low_s8(hi0),vget_low_s8(xs2)); acc=vpadalq_s16(acc,m);
m=vmull_s8(vget_high_s8(hi0),vget_high_s8(xs2)); acc=vpadalq_s16(acc,m);
m=vmull_s8(vget_low_s8(hi1),vget_low_s8(xs3)); acc=vpadalq_s16(acc,m);
m=vmull_s8(vget_high_s8(hi1),vget_high_s8(xs3)); acc=vpadalq_s16(acc,m);
#endif
}
sum=vaddvq_s32(acc);
#endif
for(;i+64<=I;i+=64){ /* fallback scalare sui blocchi planari */
const uint8_t *blk=w4+(i>>1);
for(int k=0;k<32;k++){
sum+=(int32_t)(blk[k]&0xF)*x[i+k];
sum+=(int32_t)(blk[k]>>4)*x[i+k+32];
}
}
for(;i<I;i+=2){ /* coda a coppie, unsigned (u=v+8 memorizzato) */
uint8_t byte=w4[i>>1];
sum+=(int32_t)(byte&0xF)*x[i];
if(i+1<I) sum+=(int32_t)(byte>>4)*x[i+1];
}
return sum;
}
/* ---- K1b (OPT-IN, IDOT_GS=1): IDOT planare A GRUPPI (fmt=4, gs%64==0) -----
* Con gs=64 il gruppo di scala COINCIDE col blocco-piano da 64 elementi: il
* dot unsigned del blocco (2 dpbusd) -> int32 di gruppo, meno 8*somma(x) del
* gruppo, per la scala f32 del gruppo. Attivazioni int8 (stessa famiglia
* qrow_i8 del resto dell'IDOT): NON bit-identico al kernel f32 a gruppi --
* per questo e' dietro flag, in attesa dell'ablazione. xsg = somme int32
* per (riga, gruppo), calcolate dal chiamante in una passata esatta.
* EN: grouped planar IDOT, opt-in. With gs=64 the scale group IS the plane
* block; per-group unsigned dot minus 8*group-sum, times the group scale.
* int8 activations: not bit-identical to the f32 grouped kernel, hence the
* flag until the ablation blesses a default.
*
* Three execution shapes below, one contract: per (row, output) the float
* accumulation is the SAME sequence of fmaf((float)group_int, scale[g], a)
* in ascending g, and every group_int is an exact int32 β so the per-row
* path, the 1x4 row tile, and the AMX tile are bit-identical to each other
* and to the pure-C reference on every ISA. */
/* one planar 64-element block, unpacked once and shared across a row tile:
* lo nibbles = elements base..base+31 in order, hi = base+32..base+63. */
#if defined(coli_dpbusd256)
static inline void i4p_blk256(const uint8_t *blk, __m256i *lo, __m256i *hi){
const __m256i m4=_mm256_set1_epi8(0x0F);
__m256i b=_mm256_loadu_si256((const __m256i*)blk);
*lo=_mm256_and_si256(b,m4);
*hi=_mm256_and_si256(_mm256_srli_epi16(b,4),m4);
}
#endif
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
/* whole block in one zmm: lanes 0..31 = lo elements, 32..63 = hi β matches a
* straight 64-byte load of the activation block, so ONE dpbusd per block. */
static inline __m512i i4p_blk512(const uint8_t *blk){
const __m256i m4=_mm256_set1_epi8(0x0F);
__m256i b=_mm256_loadu_si256((const __m256i*)blk);
return _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_and_si256(b,m4)),
_mm256_and_si256(_mm256_srli_epi16(b,4),m4),1);
}
#endif
/* ---- K1c: AMX int8 tile kernel (Sapphire Rapids+, opt-in via the same
* IDOT_GS=1 family gate; AMX=0 kills it, AMX_S_MIN sets the row threshold).
* With gs a multiple of 64, K=64 tile-multiplies cover a scale group exactly:
* B tile = 16 output rows' int4 block unpacked once to SIGNED int8 (v-8, so
* tdpbssd returns d - 8*sum(x_group) directly β the same int32 the vector
* path computes as d_unsigned - 8*xsg), A tile = up to 16 activation rows.
* The unpack cost is paid once per (output tile, group) and amortized over
* every activation row β the multi-row reuse the pair-layout kernels lack.
* Linux-only arming: tile data needs an ARCH_REQ_XCOMP_PERM handshake. */
#if defined(__AMX_INT8__) && defined(__AMX_TILE__) && defined(__AVX512F__)
#define COLI_HAVE_AMX_I4P 1
#if defined(__linux__)
#include <unistd.h>
#include <sys/syscall.h>
#elif defined(_WIN32)
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
#endif
static int coli_amx_state=-1;
static int coli_amx_s_min=8;
static int coli_amx_ok(void){
if(coli_amx_state<0){
const char *e=getenv("AMX");
const char *sm=getenv("AMX_S_MIN"); if(sm&&atoi(sm)>0) coli_amx_s_min=atoi(sm);
#if defined(__linux__)
/* ARCH_REQ_XCOMP_PERM(XFEATURE_XTILEDATA): kernel >= 5.16 grants tile
* state per-process; without it the first tile op SIGILLs. */
coli_amx_state=(e&&*e=='0')?0:(syscall(SYS_arch_prctl,0x1023,18)==0);
#elif defined(_WIN32)
/* Windows 11: tile data is an opt-in per-process XState feature.
* Dynamic lookup keeps older kernels building/running (arming just
* fails closed there). XSTATE_AMX_TILE_DATA = 18. */
if(e&&*e=='0') coli_amx_state=0;
else {
HMODULE k32=GetModuleHandleA("kernel32.dll");
typedef BOOL (WINAPI *coli_xstate_fn)(ULONG64);
coli_xstate_fn f=k32?(coli_xstate_fn)(void*)GetProcAddress(k32,"EnableProcessOptionalXStateFeatures"):NULL;
coli_amx_state=(f && f(1ull<<18))?1:0;
}
#else
(void)e; coli_amx_state=0;
#endif
if(coli_amx_state)
fprintf(stderr,"[K1c] AMX int8 tile kernel armed for gs64 tensors "
"(AMX=0 disables, engages at S>=%d)\n",coli_amx_s_min);
}
return coli_amx_state;
}
/* ldtilecfg layout (palette 1): tmm0=C [rows x 16 i32], tmm1=A [rows x 64 i8],
* tmm2=B [16 x 64 i8 VNNI]. rows<16 reconfigures for the last partial s-tile. */
struct coli_tilecfg { uint8_t palette,start_row,rsvd[14]; uint16_t colsb[16]; uint8_t rows[16]; };
static void coli_amx_cfg(int arows){
struct coli_tilecfg c; memset(&c,0,sizeof c); c.palette=1;
c.rows[0]=(uint8_t)arows; c.colsb[0]=64;
c.rows[1]=(uint8_t)arows; c.colsb[1]=64;
c.rows[2]=16; c.colsb[2]=64;
_tile_loadconfig(&c);
}
static void matmul_i4p_gidot_amx(float *y, const int8_t *xq, const float *sx,
const uint8_t *q4, const float *scale,
int S, int I, int O16, int O, int gs){
int rb=(I+1)/2, ng=(I+gs-1)/gs, bpg=gs/64;
#pragma omp parallel
{
float *acc=malloc((size_t)S*16*sizeof(float));
if(!acc){ fprintf(stderr,"OOM: amx acc\n"); exit(1); }
int8_t bstage[4*1024] __attribute__((aligned(64))); /* bpg<=4 gated below */
int32_t cbuf[16*16] __attribute__((aligned(64)));
float sclT[16];
int cur=16; coli_amx_cfg(16);
#pragma omp for schedule(static)
for(int ot=0; ot<O16; ot+=16){
memset(acc,0,(size_t)S*16*sizeof(float));
for(int g=0; g<ng; g++){
for(int j=0;j<16;j++) sclT[j]=scale[(int64_t)(ot+j)*ng+g];
for(int b=0;b<bpg;b++){
/* B tile, VNNI [k/4][n][4]: element k of output row ot+n
* lands at row k/4, byte n*4+k%4 β signed (v-8) at unpack. */
int8_t *dst=bstage+(size_t)b*1024;
int64_t boff=((int64_t)g*gs+b*64)>>1;
for(int n=0;n<16;n++){
const uint8_t *blk=q4+(int64_t)(ot+n)*rb+boff;
for(int k=0;k<32;k++){
dst[(k>>2)*64+n*4+(k&3)] =(int8_t)((blk[k]&0xF)-8);
dst[((k+32)>>2)*64+n*4+((k+32)&3)]=(int8_t)((blk[k]>>4)-8);
}
}
}
for(int st=0; st<S; st+=16){
int rows=S-st<16?S-st:16;
if(rows!=cur){ coli_amx_cfg(rows); cur=rows; }
_tile_zero(0);
for(int b=0;b<bpg;b++){
_tile_loadd(1, xq+(int64_t)st*I+(int64_t)g*gs+b*64, (size_t)I);
_tile_loadd(2, bstage+(size_t)b*1024, 64);
_tile_dpbssd(0,1,2);
}
_tile_stored(0, cbuf, 64);
/* per-lane fmadd == the vector path's per-(s,o) scalar fmaf:
* same values, same ascending-g order, single rounding. */
for(int r=0;r<rows;r++){
__m512 cv=_mm512_cvtepi32_ps(_mm512_load_si512((const void*)(cbuf+(size_t)r*16)));
__m512 av=_mm512_loadu_ps(acc+(int64_t)(st+r)*16);
_mm512_storeu_ps(acc+(int64_t)(st+r)*16,
_mm512_fmadd_ps(cv,_mm512_loadu_ps(sclT),av));
}
}
}
for(int s=0;s<S;s++)
_mm512_storeu_ps(y+(int64_t)s*O+ot,
_mm512_mul_ps(_mm512_loadu_ps(acc+(int64_t)s*16),
_mm512_set1_ps(sx[s])));
}
_tile_release();
free(acc);
}
}
#endif
/* vector body over an output range [o0,o1): the 1x4 row tile pays the weight
* block's load+mask once per 4 activation rows (the fmt=2 K2 tile's idea,
* brought to the grouped family β the prefill batch-union and the serve mux
* deliver exactly these multi-row calls). Integer group dots in any lane
* order are exact, and each row keeps its own ascending-g fmaf chain. */
static void i4p_gidot_rows(float *y, const int8_t *xq, const float *sx,
const int32_t *xsg, const uint8_t *q4,
const float *scale, int S, int I, int O, int gs,
int o0, int o1){
int rb=(I+1)/2, ng=(I+gs-1)/gs, bpg=gs/64; /* blocchi-piano per gruppo */
#pragma omp parallel for schedule(static)
for(int o=o0;o<o1;o++){
const uint8_t *w=q4+(int64_t)o*rb;
const float *scl=scale+(int64_t)o*ng;
int s=0;
for(; s+4<=S; s+=4){
const int8_t *x0=xq +(int64_t)s*I, *x1=x0+I, *x2=x1+I, *x3=x2+I;
const int32_t *g0=xsg+(int64_t)s*ng, *g1=g0+ng, *g2=g1+ng, *g3=g2+ng;
float a0=0,a1=0,a2=0,a3=0;
int g=0;
for(; (g+1)*gs<=I; g++){
int32_t d0,d1,d2,d3;
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
__m512i A0=_mm512_setzero_si512(),A1=_mm512_setzero_si512();
__m512i A2=_mm512_setzero_si512(),A3=_mm512_setzero_si512();
for(int b=0;b<bpg;b++){
int base=g*gs+b*64;
__m512i wz=i4p_blk512(w+(base>>1));
A0=_mm512_dpbusd_epi32(A0,wz,_mm512_loadu_si512((const void*)(x0+base)));
A1=_mm512_dpbusd_epi32(A1,wz,_mm512_loadu_si512((const void*)(x1+base)));
A2=_mm512_dpbusd_epi32(A2,wz,_mm512_loadu_si512((const void*)(x2+base)));
A3=_mm512_dpbusd_epi32(A3,wz,_mm512_loadu_si512((const void*)(x3+base)));
}
d0=_mm512_reduce_add_epi32(A0); d1=_mm512_reduce_add_epi32(A1);
d2=_mm512_reduce_add_epi32(A2); d3=_mm512_reduce_add_epi32(A3);
#elif defined(coli_dpbusd256)
__m256i A0=_mm256_setzero_si256(),A1=_mm256_setzero_si256();
__m256i A2=_mm256_setzero_si256(),A3=_mm256_setzero_si256();
for(int b=0;b<bpg;b++){
int base=g*gs+b*64; __m256i lo,hi;
i4p_blk256(w+(base>>1),&lo,&hi);
A0=coli_dpbusd256(A0,lo,_mm256_loadu_si256((const __m256i*)(x0+base)));
A0=coli_dpbusd256(A0,hi,_mm256_loadu_si256((const __m256i*)(x0+base+32)));
A1=coli_dpbusd256(A1,lo,_mm256_loadu_si256((const __m256i*)(x1+base)));
A1=coli_dpbusd256(A1,hi,_mm256_loadu_si256((const __m256i*)(x1+base+32)));
A2=coli_dpbusd256(A2,lo,_mm256_loadu_si256((const __m256i*)(x2+base)));
A2=coli_dpbusd256(A2,hi,_mm256_loadu_si256((const __m256i*)(x2+base+32)));
A3=coli_dpbusd256(A3,lo,_mm256_loadu_si256((const __m256i*)(x3+base)));
A3=coli_dpbusd256(A3,hi,_mm256_loadu_si256((const __m256i*)(x3+base+32)));
}
d0=hsum256_i32(A0); d1=hsum256_i32(A1);
d2=hsum256_i32(A2); d3=hsum256_i32(A3);
#elif defined(__AVX2__)
const __m256i ones=_mm256_set1_epi16(1);
const __m256i m4=_mm256_set1_epi8(0x0F);
__m256i A0=_mm256_setzero_si256(),A1=_mm256_setzero_si256();
__m256i A2=_mm256_setzero_si256(),A3=_mm256_setzero_si256();
for(int b=0;b<bpg;b++){
int base=g*gs+b*64;
__m256i bb=_mm256_loadu_si256((const __m256i*)(w+(base>>1)));
__m256i lo=_mm256_and_si256(bb,m4);
__m256i hi=_mm256_and_si256(_mm256_srli_epi16(bb,4),m4);
/* maddubs(u8,s8): u<=15, |x|<=127 -> pair <= 3810, int16-safe */
A0=_mm256_add_epi32(A0,_mm256_madd_epi16(_mm256_maddubs_epi16(lo,_mm256_loadu_si256((const __m256i*)(x0+base))),ones));
A0=_mm256_add_epi32(A0,_mm256_madd_epi16(_mm256_maddubs_epi16(hi,_mm256_loadu_si256((const __m256i*)(x0+base+32))),ones));
A1=_mm256_add_epi32(A1,_mm256_madd_epi16(_mm256_maddubs_epi16(lo,_mm256_loadu_si256((const __m256i*)(x1+base))),ones));
A1=_mm256_add_epi32(A1,_mm256_madd_epi16(_mm256_maddubs_epi16(hi,_mm256_loadu_si256((const __m256i*)(x1+base+32))),ones));
A2=_mm256_add_epi32(A2,_mm256_madd_epi16(_mm256_maddubs_epi16(lo,_mm256_loadu_si256((const __m256i*)(x2+base))),ones));
A2=_mm256_add_epi32(A2,_mm256_madd_epi16(_mm256_maddubs_epi16(hi,_mm256_loadu_si256((const __m256i*)(x2+base+32))),ones));
A3=_mm256_add_epi32(A3,_mm256_madd_epi16(_mm256_maddubs_epi16(lo,_mm256_loadu_si256((const __m256i*)(x3+base))),ones));
A3=_mm256_add_epi32(A3,_mm256_madd_epi16(_mm256_maddubs_epi16(hi,_mm256_loadu_si256((const __m256i*)(x3+base+32))),ones));
}
d0=hsum256_i32(A0); d1=hsum256_i32(A1);
d2=hsum256_i32(A2); d3=hsum256_i32(A3);
#else
d0=d1=d2=d3=0;
for(int b=0;b<bpg;b++){
int base=g*gs+b*64;
const uint8_t *blk=w+(base>>1);
for(int k=0;k<32;k++){
int32_t ul=(int32_t)(blk[k]&0xF), uh=(int32_t)(blk[k]>>4);
d0+=ul*x0[base+k]+uh*x0[base+k+32];
d1+=ul*x1[base+k]+uh*x1[base+k+32];
d2+=ul*x2[base+k]+uh*x2[base+k+32];
d3+=ul*x3[base+k]+uh*x3[base+k+32];
}
}
#endif
a0=fmaf((float)(d0-8*g0[g]),scl[g],a0);
a1=fmaf((float)(d1-8*g1[g]),scl[g],a1);
a2=fmaf((float)(d2-8*g2[g]),scl[g],a2);
a3=fmaf((float)(d3-8*g3[g]),scl[g],a3);
}
if(g*gs<I){ /* coda: gruppo parziale, nibble a coppie */
int32_t d0=0,d1=0,d2=0,d3=0;
for(int i=g*gs;i<I;i++){
uint8_t byte=w[i>>1];
int32_t u=(int32_t)((i&1)?(byte>>4):(byte&0xF));
d0+=u*x0[i]; d1+=u*x1[i]; d2+=u*x2[i]; d3+=u*x3[i];
}
a0=fmaf((float)(d0-8*g0[g]),scl[g],a0);
a1=fmaf((float)(d1-8*g1[g]),scl[g],a1);
a2=fmaf((float)(d2-8*g2[g]),scl[g],a2);
a3=fmaf((float)(d3-8*g3[g]),scl[g],a3);
}
y[(int64_t)s*O+o] =a0*sx[s];
y[(int64_t)(s+1)*O+o]=a1*sx[s+1];
y[(int64_t)(s+2)*O+o]=a2*sx[s+2];
y[(int64_t)(s+3)*O+o]=a3*sx[s+3];
}
for(; s<S; s++){
const int8_t *xr=xq+(int64_t)s*I;
const int32_t *xg=xsg+(int64_t)s*ng;
float a=0; int g=0;
for(; (g+1)*gs<=I; g++){ /* gruppi interi */
int32_t d=0;
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
__m512i acc=_mm512_setzero_si512();
for(int b=0;b<bpg;b++){
int base=g*gs+b*64;
acc=_mm512_dpbusd_epi32(acc,i4p_blk512(w+(base>>1)),
_mm512_loadu_si512((const void*)(xr+base)));
}
d=_mm512_reduce_add_epi32(acc);
#else
for(int b=0;b<bpg;b++){
int base=g*gs+b*64;
const uint8_t *blk=w+(base>>1);
const int8_t *xb=xr+base;
#if defined(coli_dpbusd256)
__m256i lo,hi; i4p_blk256(blk,&lo,&hi);
__m256i acc=_mm256_setzero_si256();
acc=coli_dpbusd256(acc,lo,_mm256_loadu_si256((const __m256i*)xb));
acc=coli_dpbusd256(acc,hi,_mm256_loadu_si256((const __m256i*)(xb+32)));
d+=hsum256_i32(acc);
#elif defined(__AVX2__)
const __m256i m4=_mm256_set1_epi8(0x0F);
const __m256i ones=_mm256_set1_epi16(1);
__m256i bb=_mm256_loadu_si256((const __m256i*)blk);
__m256i p0=_mm256_maddubs_epi16(_mm256_and_si256(bb,m4),
_mm256_loadu_si256((const __m256i*)xb));
__m256i p1=_mm256_maddubs_epi16(_mm256_and_si256(_mm256_srli_epi16(bb,4),m4),
_mm256_loadu_si256((const __m256i*)(xb+32)));
__m256i acc=_mm256_add_epi32(_mm256_madd_epi16(p0,ones),
_mm256_madd_epi16(p1,ones));
d+=hsum256_i32(acc);
#else
for(int k=0;k<32;k++){
d+=(int32_t)(blk[k]&0xF)*xb[k];
d+=(int32_t)(blk[k]>>4)*xb[k+32];
}
#endif
}
#endif
a=fmaf((float)(d-8*xg[g]),scl[g],a);
}
if(g*gs<I){ /* coda: gruppo parziale, nibble a coppie */
int32_t d=0;
for(int i=g*gs;i<I;i++){
uint8_t byte=w[i>>1];
d+=(int32_t)((i&1)?(byte>>4):(byte&0xF))*xr[i];
}
a=fmaf((float)(d-8*xg[g]),scl[g],a);
}
y[(int64_t)s*O+o]=a*sx[s];
}
}
}
static void matmul_i4p_grouped_idot(float *y, const int8_t *xq, const float *sx,
const int32_t *xsg, const uint8_t *q4,
const float *scale, int S, int I, int O, int gs){
#if defined(COLI_HAVE_AMX_I4P)
/* AMX takes the aligned bulk (full 16-output tiles, full groups only β
* the real gs64 checkpoints have I%gs==0); the vector path finishes any
* output remainder. Below the row threshold the B-tile unpack does not
* amortize and the vector tile is the better kernel. */
if(coli_amx_ok() && S>=coli_amx_s_min && gs%64==0 && gs<=256 && I%gs==0 && O>=16){
int O16=O&~15;
matmul_i4p_gidot_amx(y,xq,sx,q4,scale,S,I,O16,O,gs);
if(O16<O) i4p_gidot_rows(y,xq,sx,xsg,q4,scale,S,I,O,gs,O16,O);
return;
}
#endif
i4p_gidot_rows(y,xq,sx,xsg,q4,scale,S,I,O,gs,0,O);
}
/* matmul IDOT planare (fmt=2): y = (dot_u - 8*xsum[s]) * scale[o] * sx[s].
* Bit-identico a matmul_i4_idot: somme intere, identita' esatta. */
static void matmul_i4p_idot(float *y, const int8_t *xq, const float *sx, const int32_t *xsum,
const uint8_t *q4, const float *scale, int S, int I, int O){
int rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
int s=0;
#if defined(coli_dpbusd256)
/* K2: tile 1x4 sulla union β il blocco pesi (load + 2 and + srli) si
* paga UNA volta per 4 righe di attivazione invece che per riga. La
* union del prefill consegna nr=2..16 righe per expert: e' esattamente
* la finestra che il per-riga serviva peggio. Somme intere ->
* bit-identico al path per-riga per associativita'.
* EN: 1x4 register tile β the weight block's load+mask cost is paid
* once per 4 activation rows. Integer sums: bit-identical to the
* per-row path by associativity. */
const __m256i m4t=_mm256_set1_epi8(0x0F);
for(;s+4<=S;s+=4){
const int8_t *x0=xq+(int64_t)s*I, *x1=x0+I, *x2=x1+I, *x3=x2+I;
__m256i a0=_mm256_setzero_si256(), a1=_mm256_setzero_si256();
__m256i a2=_mm256_setzero_si256(), a3=_mm256_setzero_si256();
int i=0;
for(;i+64<=I;i+=64){
__m256i b =_mm256_loadu_si256((const __m256i*)(w+(i>>1)));
__m256i lo=_mm256_and_si256(b,m4t);
__m256i hi=_mm256_and_si256(_mm256_srli_epi16(b,4),m4t);
a0=coli_dpbusd256(a0,lo,_mm256_loadu_si256((const __m256i*)(x0+i)));
a0=coli_dpbusd256(a0,hi,_mm256_loadu_si256((const __m256i*)(x0+i+32)));
a1=coli_dpbusd256(a1,lo,_mm256_loadu_si256((const __m256i*)(x1+i)));
a1=coli_dpbusd256(a1,hi,_mm256_loadu_si256((const __m256i*)(x1+i+32)));
a2=coli_dpbusd256(a2,lo,_mm256_loadu_si256((const __m256i*)(x2+i)));
a2=coli_dpbusd256(a2,hi,_mm256_loadu_si256((const __m256i*)(x2+i+32)));
a3=coli_dpbusd256(a3,lo,_mm256_loadu_si256((const __m256i*)(x3+i)));
a3=coli_dpbusd256(a3,hi,_mm256_loadu_si256((const __m256i*)(x3+i+32)));
}
int32_t d0=hsum256_i32(a0), d1=hsum256_i32(a1);
int32_t d2=hsum256_i32(a2), d3=hsum256_i32(a3);
/* coda a coppie, unsigned: stessa identita' -8*xsum del per-riga */
for(;i<I;i+=2){
uint8_t byte=w[i>>1];
d0+=(int32_t)(byte&0xF)*x0[i]; d1+=(int32_t)(byte&0xF)*x1[i];
d2+=(int32_t)(byte&0xF)*x2[i]; d3+=(int32_t)(byte&0xF)*x3[i];
if(i+1<I){
d0+=(int32_t)(byte>>4)*x0[i+1]; d1+=(int32_t)(byte>>4)*x1[i+1];
d2+=(int32_t)(byte>>4)*x2[i+1]; d3+=(int32_t)(byte>>4)*x3[i+1];
}
}
y[(int64_t)(s+0)*O+o]=(float)(d0-8*xsum[s+0])*sc*sx[s+0];
y[(int64_t)(s+1)*O+o]=(float)(d1-8*xsum[s+1])*sc*sx[s+1];
y[(int64_t)(s+2)*O+o]=(float)(d2-8*xsum[s+2])*sc*sx[s+2];
y[(int64_t)(s+3)*O+o]=(float)(d3-8*xsum[s+3])*sc*sx[s+3];
}
#endif
for(;s<S;s++){
int32_t d=dot_i4p_u(w,xq+(int64_t)s*I,I)-8*xsum[s];
y[(int64_t)s*O+o]=(float)d*sc*sx[s];
}
}
}
#endif /* COLIBRI_IDOT_H */