37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
Release 2026-09-27: finer activations for weights in blocks of 32
camauri728f80eac3dc
diff --git a/ChangeLog.md b/ChangeLog.mdindex ff9b826..3ec7e7c 100644--- a/ChangeLog.md+++ b/ChangeLog.md@@ -2,6 +2,20 @@> Curated, user-facing summary of completed work, newest first.+## [2026-09-27] - Finer activations for weights in blocks of 32++- **Weights stored in blocks of 32 (Q8_0, Q4_0, Q4_1, Q5_1, IQ4_NL) now+ meet activations with a scale per 32 values**, as llama.cpp gives them,+ instead of one per 256: a large activation no longer flattens the small+ ones beside it. The activations are quantized once into both forms; the+ K types keep theirs. Checked against an exact float64 forward pass of+ Gemma-4-26B-A4B (all Q8_0 and Q5_1 where it is most sensitive), 200+ tokens: the likeliest token at 156 positions instead of 147, mean logit+ error 0.87 instead of 1.39 (llama.cpp: 160, 0.90). Against llama.cpp,+ mean differences fall by 15-40% on Qwen3.5 in IQ4_NL, Q4_0, Q5_1 and+ UD-Q4_K_XL (Q5_1: 376 -> 389 of 400). Models made only of K types give+ the same bits as before.+## [2026-09-27] - The threads chosen per machine- **The time left to read a long prompt no longer jumps at its start.**diff --git a/README.md b/README.mdindex 0400815..0cfa518 100644--- a/README.md+++ b/README.md@@ -96,8 +96,11 @@ report from `janas-bench` is the single most useful thing you can send back**to 12.9 and the 26B from 16.9 to 24.5 on the development laptop, thesame tokens with and without it. Against llama.cpp on 200 tokens of prose the most likely tokenagrees at 160 positions on the 12B (mean logit difference 0.84) and at- 145 on the 26B (1.43); on another 200-token text at 191 on E2B (0.45) and- 195 on E4B (0.28). The larger two turn small numerical differences into+ 160 on the 26B (0.97); on another 200-token text at 191 on E2B (0.45) and+ 195 on E4B (0.28). Against an exact float64 computation over the same+ weights (a NumPy forward pass written to check them), on the prose: the 12B+ agrees at 167 positions (llama.cpp 173), the 26B at 156 (llama.cpp+ 160), with mean errors of 0.75 and 0.87 (llama.cpp 0.67 and 0.90). The larger two turn small numerical differences intolarge ones, and llama.cpp is not a steady reference for them: againstitself with an eight-bit cache it agrees at 153 positions on the 12B, andwith a 32-bit cache and no flash attention, at 112. Thediff --git a/src/llm/quant.c b/src/llm/quant.cindex 7cd6ae8..a3a9abb 100644--- a/src/llm/quant.c+++ b/src/llm/quant.c@@ -77,6 +77,32 @@ uint16_t janas_fp32_to_fp16(float f)static int use_avx2_fma; /* set by float_helpers_init, further down */+/* The eight blocks of 32 (d32, qs32, bsums32) of one block, as the 256+ are made: det, the division-free scale of the _det variant */+static void q8_32_ref(const float *x, struct janas_block_q8k *y, int det)+{+ for (int k = 0; k < 8; k++) {+ const float *xk = x + 32 * k;+ float amax = 0.0f;+ for (int i = 0; i < 32; i++)+ amax = fmaxf(amax, fabsf(xk[i]));+ float d = det ? amax * 0.007874015718698502f : amax / 127.0f;+ float id = d > 0.0f ? (det ? janas_recip_det(d) : 1.0f / d) : 0.0f;+ y->d32[k] = d;+ for (int i = 0; i < 32; i++) {+ float q = nearbyintf(xk[i] * id);+ y->qs32[32 * k + i] =+ (int8_t)(q > 127.0f ? 127 : (q < -127.0f ? -127 : q));+ }+ }+ for (int g = 0; g < JANAS_QK / 16; g++) {+ int s = 0;+ for (int i = 0; i < 16; i++)+ s += y->qs32[g * 16 + i];+ y->bsums32[g] = (int16_t)s;+ }+}+static void q8k_quantize_ref(const float *x, struct janas_block_q8k *y,size_t n){@@ -98,6 +124,7 @@ static void q8k_quantize_ref(const float *x, struct janas_block_q8k *y,s += y[b].qs[g * 16 + i];y[b].bsums[g] = (int16_t)s;}+ q8_32_ref(x, &y[b], 0);}}@@ -122,6 +149,7 @@ static void q8k_quantize_det_ref(const float *x, struct janas_block_q8k *y,s += y[b].qs[g * 16 + i];y[b].bsums[g] = (int16_t)s;}+ q8_32_ref(x, &y[b], 1);}}@@ -149,13 +177,14 @@ __attribute__((target("avx2,fma"))) static float q8k_amax(const float *x)return _mm_cvtss_f32(m);}-/* One block, given the scale and its reciprocal already computed. */+/* groups of 16 values to 8 bits with the reciprocal id of their scale,+ and their sums */__attribute__((target("avx2,fma"))) static void-q8k_block_avx2(const float *x, struct janas_block_q8k *y, float id)+q8_groups_avx2(const float *x, int8_t *qs, int16_t *bsums, int groups, float id){const __m256i lo = _mm256_set1_epi32(-127), hi = _mm256_set1_epi32(127);const __m256 vid = _mm256_set1_ps(id);- for (int g = 0; g < JANAS_QK / 16; g++) {+ for (int g = 0; g < groups; g++) {__m256i i0 =_mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(x + g * 16), vid));__m256i i1 = _mm256_cvtps_epi32(@@ -167,15 +196,44 @@ q8k_block_avx2(const float *x, struct janas_block_q8k *y, float id)_mm256_extracti128_si256(s, 1));h = _mm_add_epi32(h, _mm_shuffle_epi32(h, 0x4e));h = _mm_add_epi32(h, _mm_shuffle_epi32(h, 0xb1));- y->bsums[g] = (int16_t)_mm_cvtsi128_si32(h);+ bsums[g] = (int16_t)_mm_cvtsi128_si32(h);/* packs leaves the two halves lane by lane: 0,2,1,3 puts them back */__m256i p = _mm256_permute4x64_epi64(_mm256_packs_epi32(i0, i1), 0xd8);- _mm_storeu_si128((__m128i *)(y->qs + g * 16),+ _mm_storeu_si128((__m128i *)(qs + g * 16),_mm_packs_epi16(_mm256_castsi256_si128(p),_mm256_extracti128_si256(p, 1)));}}+/* the largest absolute value of 32 */+__attribute__((target("avx2,fma"))) static float q8_amax32(const float *x)+{+ const __m256 mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff));+ __m256 a = _mm256_and_ps(_mm256_loadu_ps(x), mask);+ for (int i = 8; i < 32; i += 8)+ a = _mm256_max_ps(a, _mm256_and_ps(_mm256_loadu_ps(x + i), mask));+ __m128 m =+ _mm_max_ps(_mm256_castps256_ps128(a), _mm256_extractf128_ps(a, 1));+ m = _mm_max_ps(m, _mm_movehl_ps(m, m));+ m = _mm_max_ss(m, _mm_shuffle_ps(m, m, 1));+ return _mm_cvtss_f32(m);+}++/* One block: the 256 with the scale whose reciprocal is id, and the+ eight blocks of 32, as q8_32_ref */+__attribute__((target("avx2,fma"))) static void+q8k_block_avx2(const float *x, struct janas_block_q8k *y, float id, int det)+{+ q8_groups_avx2(x, y->qs, y->bsums, JANAS_QK / 16, id);+ for (int k = 0; k < 8; k++) {+ float amax = q8_amax32(x + 32 * k);+ float d = det ? amax * 0.007874015718698502f : amax / 127.0f;+ y->d32[k] = d;+ q8_groups_avx2(x + 32 * k, y->qs32 + 32 * k, y->bsums32 + 2 * k, 2,+ d > 0.0f ? (det ? janas_recip_det(d) : 1.0f / d) : 0.0f);+ }+}+__attribute__((target("avx2,fma"))) static voidq8k_quantize_avx2(const float *x, struct janas_block_q8k *y, size_t n){@@ -183,7 +241,7 @@ q8k_quantize_avx2(const float *x, struct janas_block_q8k *y, size_t n)float amax = q8k_amax(x);float d = amax / 127.0f;y[b].d = d;- q8k_block_avx2(x, &y[b], d > 0.0f ? 1.0f / d : 0.0f);+ q8k_block_avx2(x, &y[b], d > 0.0f ? 1.0f / d : 0.0f, 0);}}@@ -193,7 +251,7 @@ q8k_quantize_det_avx2(const float *x, struct janas_block_q8k *y, size_t n)for (size_t b = 0; b < n / JANAS_QK; b++, x += JANAS_QK) {float d = q8k_amax(x) * 0.007874015718698502f;y[b].d = d;- q8k_block_avx2(x, &y[b], d > 0.0f ? janas_recip_det(d) : 0.0f);+ q8k_block_avx2(x, &y[b], d > 0.0f ? janas_recip_det(d) : 0.0f, 1);}}#endif@@ -635,9 +693,9 @@ float janas_q8_0_dot_ref(const struct janas_block_q8_0 *w,const struct janas_block_q8_0 *wk = w + 8 * b + k;int32_t dot = 0;for (int l = 0; l < 32; l++)- dot += wk->qs[l] * x[b].qs[32 * k + l];- acc[k] =- fmaf(janas_fp16_to_fp32(wk->d) * x[b].d, (float)dot, acc[k]);+ dot += wk->qs[l] * x[b].qs32[32 * k + l];+ acc[k] = fmaf(janas_fp16_to_fp32(wk->d) * x[b].d32[k], (float)dot,+ acc[k]);}return ((acc[0] + acc[4]) + (acc[2] + acc[6])) +((acc[1] + acc[5]) + (acc[3] + acc[7]));@@ -686,9 +744,9 @@ static float nib_dot_ref(const struct janas_block_iq4nl *w,for (size_t b = 0; b < nb; b++)for (int k = 0; k < 8; k++) {const struct janas_block_iq4nl *wk = w + 8 * b + k;- int32_t dot = nib_sum32(wk->qs, x[b].qs + 32 * k, levels);- acc[k] =- fmaf(janas_fp16_to_fp32(wk->d) * x[b].d, (float)dot, acc[k]);+ int32_t dot = nib_sum32(wk->qs, x[b].qs32 + 32 * k, levels);+ acc[k] = fmaf(janas_fp16_to_fp32(wk->d) * x[b].d32[k], (float)dot,+ acc[k]);}return ((acc[0] + acc[4]) + (acc[2] + acc[6])) +((acc[1] + acc[5]) + (acc[3] + acc[7]));@@ -772,11 +830,11 @@ static float dm_dot_ref(const uint8_t *w, size_t size, int five,const uint8_t *wk = w + (8 * b + k) * size;int32_t dot = 0;for (int l = 0; l < 32; l++)- dot += dm_value(wk, five, l) * x[b].qs[32 * k + l];- int32_t asum = x[b].bsums[2 * k] + x[b].bsums[2 * k + 1];- acc[k] = fmaf(janas_fp16_to_fp32(dm_u16(wk, 0)) * x[b].d,+ dot += dm_value(wk, five, l) * x[b].qs32[32 * k + l];+ int32_t asum = x[b].bsums32[2 * k] + x[b].bsums32[2 * k + 1];+ acc[k] = fmaf(janas_fp16_to_fp32(dm_u16(wk, 0)) * x[b].d32[k],(float)dot, acc[k]);- accm[k] = fmaf(janas_fp16_to_fp32(dm_u16(wk, 2)) * x[b].d,+ accm[k] = fmaf(janas_fp16_to_fp32(dm_u16(wk, 2)) * x[b].d32[k],(float)asum, accm[k]);}return (((acc[0] + acc[4]) + (acc[2] + acc[6])) +diff --git a/src/llm/quant.h b/src/llm/quant.hindex 4ef7369..67b4783 100644--- a/src/llm/quant.h+++ b/src/llm/quant.h@@ -156,6 +156,16 @@ struct janas_block_q8k {float d;int8_t qs[JANAS_QK];int16_t bsums[JANAS_QK / 16];+ /* the same 256 activations in eight blocks of 32, each with a scale of+ its own, for the weights stored in blocks of 32 (Q8_0, Q4_0, Q4_1,+ Q5_1, IQ4_NL), as llama.cpp gives them: one scale for 256 lost the+ small activations beside a large one, and Gemma-4-26B, all Q8_0 and+ Q5_1 where it is most sensitive, strayed from the exact result half+ again as far as llama.cpp did (27 Sep 2026: the likeliest token at+ 147 of 200 positions against 160) */+ float d32[8];+ int8_t qs32[JANAS_QK];+ int16_t bsums32[JANAS_QK / 16];};_Static_assert(sizeof(struct janas_block_q4k) == 144, "q4k block size");@@ -170,7 +180,7 @@ _Static_assert(sizeof(struct janas_block_q5_1) == 24, "q5_1 block size");_Static_assert(sizeof(struct janas_block_q4_1) == 20, "q4_1 block size");_Static_assert(sizeof(struct janas_block_q3k) == 110, "q3k block size");_Static_assert(sizeof(struct janas_block_iq3s) == 110, "iq3_s block size");-_Static_assert(sizeof(struct janas_block_q8k) == 292, "q8k block size");+_Static_assert(sizeof(struct janas_block_q8k) == 612, "q8k block size");/* Quantization types, numbered as in GGUF (ggml_type). */enum janas_qtype {diff --git a/src/llm/quant_dotn.h b/src/llm/quant_dotn.hindex 7f753b7..6312488 100644--- a/src/llm/quant_dotn.h+++ b/src/llm/quant_dotn.h@@ -317,7 +317,7 @@ DOTN(q8_0_dotn_body)(const struct janas_block_q8_0 *w,{__m256i q = _mm256_loadu_si256((const __m256i *)wb[k].qs);__m256i a =- _mm256_loadu_si256((const __m256i *)(xb->qs + 32 * k));+ _mm256_loadu_si256((const __m256i *)(xb->qs32 + 32 * k));/* |w| . (a with the sign of w): maddubs wants unsigned */__m256i p = _mm256_maddubs_epi16(_mm256_abs_epi8(q),_mm256_sign_epi8(a, q));@@ -332,8 +332,9 @@ DOTN(q8_0_dotn_body)(const struct janas_block_q8_0 *w,__m256i is8 =_mm256_add_epi32(_mm256_permute2x128_si256(t0, t1, 0x20),_mm256_permute2x128_si256(t0, t1, 0x31));- acc[v] = _mm256_fmadd_ps(_mm256_mul_ps(dk, _mm256_set1_ps(xb->d)),- _mm256_cvtepi32_ps(is8), acc[v]);+ acc[v] =+ _mm256_fmadd_ps(_mm256_mul_ps(dk, _mm256_loadu_ps(xb->d32)),+ _mm256_cvtepi32_ps(is8), acc[v]);}}_Pragma("GCC unroll 8") for (int v = 0; v < nv; v++)@@ -690,7 +691,7 @@ DOTN(nib_dotn_body)(const struct janas_block_iq4nl *w,_Pragma("GCC unroll 8") for (int k = 0; k < 8; k++){__m256i a =- _mm256_loadu_si256((const __m256i *)(xb->qs + 32 * k));+ _mm256_loadu_si256((const __m256i *)(xb->qs32 + 32 * k));__m256i p =_mm256_maddubs_epi16(aq[k], _mm256_sign_epi8(a, q[k]));s[k] = _mm256_madd_epi16(p, ones);@@ -704,8 +705,9 @@ DOTN(nib_dotn_body)(const struct janas_block_iq4nl *w,__m256i is8 =_mm256_add_epi32(_mm256_permute2x128_si256(t0, t1, 0x20),_mm256_permute2x128_si256(t0, t1, 0x31));- acc[v] = _mm256_fmadd_ps(_mm256_mul_ps(dk, _mm256_set1_ps(xb->d)),- _mm256_cvtepi32_ps(is8), acc[v]);+ acc[v] =+ _mm256_fmadd_ps(_mm256_mul_ps(dk, _mm256_loadu_ps(xb->d32)),+ _mm256_cvtepi32_ps(is8), acc[v]);}}_Pragma("GCC unroll 8") for (int v = 0; v < nv; v++)@@ -1157,14 +1159,14 @@ DOTN(dm_dotn_body)(const uint8_t *w, const size_t size, const int five,_Pragma("GCC unroll 8") for (int k = 0; k < 8; k++){__m256i a =- _mm256_loadu_si256((const __m256i *)(xb->qs + 32 * k));+ _mm256_loadu_si256((const __m256i *)(xb->qs32 + 32 * k));s[k] = _mm256_madd_epi16(_mm256_maddubs_epi16(q[k], a), ones);}__m256i is8 = DOTN(sum8x8)(s);/* bsums 2k and 2k + 1 are block k's 32 activations */__m256i as8 = _mm256_madd_epi16(- _mm256_loadu_si256((const __m256i *)xb->bsums), ones);- __m256 dx = _mm256_set1_ps(xb->d);+ _mm256_loadu_si256((const __m256i *)xb->bsums32), ones);+ __m256 dx = _mm256_loadu_ps(xb->d32);acc[v] = _mm256_fmadd_ps(_mm256_mul_ps(dk, dx),_mm256_cvtepi32_ps(is8), acc[v]);accm[v] = _mm256_fmadd_ps(_mm256_mul_ps(mk, dx),diff --git a/tests/test_quant.c b/tests/test_quant.cindex 8f1ebfb..c72c2a6 100644--- a/tests/test_quant.c+++ b/tests/test_quant.c@@ -30,6 +30,17 @@ static int failures;} \} while (0)+/* An activation as the product of weight type t sees it: weights in+ blocks of 32 take the eight blocks of 32 (d32, qs32), the others the+ 256 with one scale. */+static double act(const struct janas_block_q8k *x, int b, int i, int t)+{+ int b32 = t == JANAS_Q8_0 || t == JANAS_Q4_0 || t == JANAS_Q4_1 ||+ t == JANAS_Q5_1 || t == JANAS_IQ4_NL;+ return b32 ? (double)x[b].d32[i / 32] * x[b].qs32[i]+ : (double)x[b].d * x[b].qs[i];+}+static uint64_t rng_state = 0x9E3779B97F4A7C15ull;static uint32_t rnd(void)@@ -540,8 +551,7 @@ static void test_dot_q5k_q8_0(void)double expect = 0.0, mag = 0.0;for (int b = 0; b < NB; b++)for (int i = 0; i < JANAS_QK; i++) {- double e =- (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];+ double e = (double)wf[b * JANAS_QK + i] * act(x, b, i, t);expect += e;mag += fabs(e);}@@ -607,8 +617,7 @@ static void test_dot_iq4(void)double expect = 0.0, mag = 0.0;for (int b = 0; b < NB; b++)for (int i = 0; i < JANAS_QK; i++) {- double e =- (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];+ double e = (double)wf[b * JANAS_QK + i] * act(x, b, i, t);expect += e;mag += fabs(e);}@@ -695,8 +704,7 @@ static void test_dot_q4_0_q5_1(void)double expect = 0.0, mag = 0.0;for (int b = 0; b < NB; b++)for (int i = 0; i < JANAS_QK; i++) {- double e =- (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];+ double e = (double)wf[b * JANAS_QK + i] * act(x, b, i, t);expect += e;mag += fabs(e);}@@ -895,6 +903,26 @@ static void q8k_ref(const float *x, struct janas_block_q8k *y, size_t n,s += y[b].qs[g * 16 + i];y[b].bsums[g] = (int16_t)s;}+ for (int k = 0; k < 8; k++) {+ float m = 0.0f;+ for (int i = 0; i < 32; i++)+ m = fmaxf(m, fabsf(x[32 * k + i]));+ float dk = det ? m * 0.007874015718698502f : m / 127.0f;+ float ik =+ dk > 0.0f ? (det ? janas_recip_det(dk) : 1.0f / dk) : 0.0f;+ y[b].d32[k] = dk;+ for (int i = 0; i < 32; i++) {+ float q = nearbyintf(x[32 * k + i] * ik);+ y[b].qs32[32 * k + i] =+ (int8_t)(q > 127.0f ? 127 : (q < -127.0f ? -127 : q));+ }+ }+ for (int g = 0; g < JANAS_QK / 16; g++) {+ int s = 0;+ for (int i = 0; i < 16; i++)+ s += y[b].qs32[g * 16 + i];+ y[b].bsums32[g] = (int16_t)s;+ }}}