Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
CPU lane IQ1_M AVX-512 rows: the 64-weight step's scale words by one vpermw and the delta signs by a masked subtract (bit-identical to ggml's dot: 0 of 2.7M rows differ at 1 and 2 tokens on Zen 4 and Sapphire Rapids); +9% on the rows, the 1-token policy is unchanged
Niko122106f313b019da
diff --git a/src/kernels/cpu/iq_avx512.cpp b/src/kernels/cpu/iq_avx512.cppindex 14330f6ec..5437ef584 100644--- a/src/kernels/cpu/iq_avx512.cpp+++ b/src/kernels/cpu/iq_avx512.cpp@@ -451,6 +451,12 @@ void rows_m(const uint8_t* w, size_t rb, int n, const block_q8_K* const* y, consconst __m256i mask = _mm256_set1_epi16(0x7), mone = _mm256_set1_epi16(1), mtwo8 = _mm256_set1_epi8(2);const __m256i scales_shift = _mm256_set_epi64x(9, 3, 6, 0);const __m512i zero = _mm512_setzero_si512();+ // the 32 scale words a 64-weight step (8 each of chunks 4 it .. 4 it + 3) out of the 16 decoded scales (their order: 4 e + j for+ // shift e, word j): one vpermw a step instead of two vpshufb + adds + an insert+ __m512i sidx[4];+ for (int it = 0; it < 4; ++it)+ sidx[it] = _mm512_set_epi16(12 + it, 12 + it, 12 + it, 12 + it, 12 + it, 12 + it, 12 + it, 12 + it, 4 + it, 4 + it, 4 + it, 4 + it, 4 + it, 4 + it,+ 4 + it, 4 + it, 8 + it, 8 + it, 8 + it, 8 + it, 8 + it, 8 + it, 8 + it, 8 + it, it, it, it, it, it, it, it, it);for (int r = r0; r < r1; ++r) {__m256 a1[NT], a2[NT];for (int t = 0; t < NT; ++t) a1[t] = a2[t] = _mm256_setzero_ps();@@ -469,8 +475,7 @@ void rows_m(const uint8_t* w, size_t rb, int n, const block_q8_K* const* y, cons__m256i scales = _mm256_set1_epi64x((long long) u64(scb));scales = _mm256_srlv_epi64(scales, scales_shift);scales = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scales, mask), 1), mone);- __m256i sidx1 = _mm256_set1_epi16(0x0100);- __m256i sidx2 = _mm256_add_epi8(sidx1, _mm256_set1_epi8(8));+ const __m512i sctab = _mm512_castsi256_si512(scales);__m512i s1[NT], s2[NT];for (int t = 0; t < NT; ++t) s1[t] = s2[t] = zero;for (int it = 0; it < 4; ++it) {@@ -483,13 +488,10 @@ void rows_m(const uint8_t* w, size_t rb, int n, const block_q8_K* const* y, consconst __m512i g = _mm512_set_epi64((long long) iq1s_grid[ix[7]], (long long) iq1s_grid[ix[6]], (long long) iq1s_grid[ix[5]],(long long) iq1s_grid[ix[4]], (long long) iq1s_grid[ix[3]], (long long) iq1s_grid[ix[2]],(long long) iq1s_grid[ix[1]], (long long) iq1s_grid[ix[0]]);- const __m512i negv = _mm512_maskz_set1_epi64((__mmask8) _pext_u32(qh32, 0x88888888u), -1);- const __m256i sca = _mm256_shuffle_epi8(scales, sidx1);- const __m256i scb2 = _mm256_shuffle_epi8(scales, sidx2);- sidx1 = _mm256_add_epi8(sidx1, mtwo8);- sidx2 = _mm256_add_epi8(sidx2, mtwo8);- const __m512i sc = _mm512_inserti64x4(_mm512_castsi256_si512(sca), scb2, 1);- const __m512i ssc = _mm512_sub_epi16(_mm512_xor_si512(sc, negv), negv);+ const __m512i sc = _mm512_permutexvar_epi16(sidx[it], sctab);+ // the delta sign of each 8-weight group (bit 3 of its qh nibble) negates that group's 4 scale words+ const __mmask32 dneg = (__mmask32) (_pdep_u32(_pext_u32(qh32, 0x88888888u), 0x11111111u) * 15u);+ const __m512i ssc = _mm512_mask_sub_epi16(sc, dneg, zero, sc);const __m512i ag = _mm512_abs_epi8(g);const __mmask64 gneg = _mm512_movepi8_mask(g);for (int t = 0; t < NT; ++t) {