Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// src/kernels/cpu/q2_avx1.cpp - the Q2_0 expert rows and the activation quantizer for CPUs that have AVX but
// no AVX2 (Sandy Bridge through Ivy Bridge Xeons; issue #623, #1375), the third rung of q2_rows_any /
// act_quant_any below the AVX-2 one (issue #1699: those dispatchers used to hand AVX2 code to an AVX1 CPU).
//
// The arithmetic is q2_avx2.cpp's: codes 0..3 against the int8 activation per 32-value chunk, times the weight
// scale and the chunk scale, minus the weight scale times the chunk's `hx`. 128-bit integer lanes (maddubs and
// madd are SSSE3/SSE2), 256-bit float accumulation with mul+add (no FMA), fp16 scales converted in software (no
// F16C). The legacy row kernel keeps q2_avx2.cpp's lane pairing and reduction tree exactly, so the only
// difference from an AVX2 CPU is its FMA becoming mul+add (last bits, unavoidable without FMA); the bit-plane
// kernel's float summation order is its own (last bits, as documented for the AVX-2 one), and every token's
// operations are the same whatever the window width (#152).
//
// Compiled with -msse4.2 -mavx -mno-avx2 -mno-fma -mno-f16c (CMakeLists.txt, the iq_avx1.cpp pattern) and called
// only behind cpu_avx1_ok() && !cpu_avx2_ok(), so no CPU that lacks those features can reach it - and no CPU that
// has AVX2 changes what it runs.
#include "strata/kernels/cpu/expert.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
#include <immintrin.h>
#include <cmath>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#if defined(__AVX2__) || defined(__FMA__) || defined(__F16C__)
#error "q2_avx1.cpp must be compiled without AVX2, FMA and F16C (see the per-source flags in CMakeLists.txt)"
#endif
namespace strata::kernels::cpu {
namespace {
// fp16 -> fp32 without F16C (the written-out conversion, subnormals included; iq_avx1.cpp's copy).
inline float h2f(uint16_t h) {
const uint32_t sign = (uint32_t) (h >> 15) & 1u;
uint32_t exp = (h >> 10) & 0x1Fu, man = h & 0x3FFu, f;
if (exp == 0) {
if (man == 0) {
f = sign << 31;
} else {
exp = 127 - 15 + 1;
while (!(man & 0x400u)) { man <<= 1; --exp; }
man &= 0x3FFu;
f = (sign << 31) | (exp << 23) | (man << 13);
}
} else if (exp == 31) {
f = (sign << 31) | 0x7F800000u | (man << 13);
} else {
f = (sign << 31) | ((exp - 15 + 127) << 23) | (man << 13);
}
float out;
std::memcpy(&out, &f, 4);
return out;
}
// 16 bytes of 2-bit codes -> 64 codes in value order, four 16-byte vectors: v0 values 0..15, v1 16..31,
// v2 32..47, v3 48..63 (q2_avx2.cpp's unpack64 cut to 128-bit lanes; the same interleave).
inline void unpack64(const uint8_t* codes, __m128i& v0, __m128i& v1, __m128i& v2, __m128i& v3) {
const __m128i b = _mm_loadu_si128((const __m128i*) codes);
const __m128i m3 = _mm_set1_epi8(3);
const __m128i c0 = _mm_and_si128(b, m3);
const __m128i c1 = _mm_and_si128(_mm_srli_epi16(b, 2), m3);
const __m128i c2 = _mm_and_si128(_mm_srli_epi16(b, 4), m3);
const __m128i c3 = _mm_and_si128(_mm_srli_epi16(b, 6), m3);
const __m128i a0 = _mm_unpacklo_epi8(c0, c1), a1 = _mm_unpacklo_epi8(c2, c3); // bytes 0..7
const __m128i b0 = _mm_unpackhi_epi8(c0, c1), b1 = _mm_unpackhi_epi8(c2, c3); // bytes 8..15
v0 = _mm_unpacklo_epi16(a0, a1); // values 0..15
v1 = _mm_unpackhi_epi16(a0, a1); // values 16..31
v2 = _mm_unpacklo_epi16(b0, b1); // values 32..47
v3 = _mm_unpackhi_epi16(b0, b1); // values 48..63
}
// 16 codes against 16 activations, four products per int32 lane (q2_avx2_rows.inl's dot4 at 128-bit;
// maddubs cannot saturate: a pair is at most 2 * 3 * 128).
inline __m128i dot4(__m128i codes, __m128i act) {
return _mm_madd_epi16(_mm_maddubs_epi16(codes, act), _mm_set1_epi16(1));
}
// The legacy row kernel. The accumulator pairing matches q2_avx2.cpp's lane pairing: each 256-bit lane takes
// the s0 contribution (values 0..31) and the s1 contribution (values 32..63) into the SAME lanes, block by
// block, and the reduction tree is the same one ((L0+L4)+(L2+L6)) + ((L1+L5)+(L3+L7)) - the only difference
// from an AVX2 CPU is that kernel's FMA being a mul+add here (last bits).
template <int NT>
inline void row_multi(const uint8_t* row, const ActQ* const* a, int nblocks, float* res) {
__m256 acc[NT];
float corr[NT];
for (int t = 0; t < NT; ++t) { acc[t] = _mm256_setzero_ps(); corr[t] = 0.f; }
for (int b = 0; b < nblocks; ++b) {
const uint8_t* blk = row + (size_t) b * 18;
const float d = h2f((uint16_t) (blk[0] | (blk[1] << 8)));
__m128i v0, v1, v2, v3;
unpack64(blk + 2, v0, v1, v2, v3);
for (int t = 0; t < NT; ++t) {
const int8_t* q = a[t]->q + b * 64;
// s0 = values 0..31: lanes 0..3 of accA's 128-bit halves; s1 = values 32..63 into the SAME lanes
const __m128i s0a = dot4(v0, _mm_loadu_si128((const __m128i*) q)); // values 0..15
const __m128i s0b = dot4(v1, _mm_loadu_si128((const __m128i*) (q + 16))); // values 16..31
const __m128i s1a = dot4(v2, _mm_loadu_si128((const __m128i*) (q + 32))); // values 32..47
const __m128i s1b = dot4(v3, _mm_loadu_si128((const __m128i*) (q + 48))); // values 48..63
const __m256 s0 = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm_cvtepi32_ps(s0a)), _mm_cvtepi32_ps(s0b), 1);
const __m256 s1 = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm_cvtepi32_ps(s1a)), _mm_cvtepi32_ps(s1b), 1);
// exactly the AVX-2 kernel's two fmadds: the whole s0 (values 0..31) by scale[2b], the whole s1
// (values 32..63) by scale[2b+1], into the same lanes - lane pairing and reduce tree follow suit
acc[t] = _mm256_add_ps(acc[t], _mm256_mul_ps(_mm256_set1_ps(d * a[t]->scale[2 * b]), s0));
acc[t] = _mm256_add_ps(acc[t], _mm256_mul_ps(_mm256_set1_ps(d * a[t]->scale[2 * b + 1]), s1));
corr[t] += d * (a[t]->hx[2 * b] + a[t]->hx[2 * b + 1]);
}
}
for (int t = 0; t < NT; ++t) {
const __m128 h = _mm_add_ps(_mm256_castps256_ps128(acc[t]), _mm256_extractf128_ps(acc[t], 1));
const __m128 s = _mm_add_ps(h, _mm_movehl_ps(h, h));
res[t] = _mm_cvtss_f32(_mm_add_ss(s, _mm_movehdup_ps(s))) - corr[t];
}
}
template <int NT>
void rows(const uint8_t* w, size_t row_bytes, int nblocks, const ActQ* const* a, float* const* out, int r0, int r1) {
float res[NT];
for (int r = r0; r < r1; ++r) {
row_multi<NT>(w + (size_t) r * row_bytes, a, nblocks, res);
for (int t = 0; t < NT; ++t) out[t][r] = res[t];
}
}
// ---- the bit-plane row kernel (STRATA_Q2_BITPLANE=1), 128-bit lanes: one 64-value block at a time instead of
// a pair per 256-bit register. The integer part is exact; the float summation order is this kernel's own (the
// same statement as for the AVX-2 one), and every row reduces with the same tree alone or in a group (#152).
template <int NT>
inline void row_bp_acc(const uint8_t* row, const ActQ* const* a, int npairs, __m128* acc) {
for (int t = 0; t < NT; ++t) { acc[2 * t] = _mm_setzero_ps(); acc[2 * t + 1] = _mm_setzero_ps(); }
const __m128i m3 = _mm_set1_epi8(3);
const __m128i ones = _mm_set1_epi16(1);
for (int p = 0; p < npairs; ++p) {
const uint8_t* blk = row + (size_t) p * 36;
for (int h = 0; h < 2; ++h) {
uint16_t dh;
std::memcpy(&dh, blk + h * 18, 2); // the block's fp16 scale: bytes 0..1 (block 1 at +18)
const __m128 d = _mm_set1_ps(h2f(dh));
const __m128i c = _mm_loadu_si128((const __m128i*) (blk + 2 + h * 18)); // its 16 code bytes
const __m128i c0 = _mm_and_si128(c, m3);
const __m128i c1 = _mm_and_si128(_mm_srli_epi16(c, 2), m3);
const __m128i c2 = _mm_and_si128(_mm_srli_epi16(c, 4), m3);
const __m128i c3 = _mm_and_si128(_mm_srli_epi16(c, 6), m3);
for (int t = 0; t < NT; ++t) {
const int8_t* q = a[t]->qp + (size_t) p * 128 + h * 16;
__m128i s = _mm_maddubs_epi16(c0, _mm_load_si128((const __m128i*) q));
s = _mm_add_epi16(s, _mm_maddubs_epi16(c1, _mm_load_si128((const __m128i*) (q + 32))));
s = _mm_add_epi16(s, _mm_maddubs_epi16(c2, _mm_load_si128((const __m128i*) (q + 64))));
s = _mm_add_epi16(s, _mm_maddubs_epi16(c3, _mm_load_si128((const __m128i*) (q + 96))));
const __m128i i32 = _mm_sub_epi32(_mm_madd_epi16(s, ones),
_mm_load_si128((const __m128i*) (a[t]->psum + p * 8 + h * 4)));
const __m128 v = _mm_mul_ps(_mm_cvtepi32_ps(i32),
_mm_load_ps(a[t]->pscale + p * 8 + h * 4));
__m128& x = acc[2 * t + h];
x = _mm_add_ps(x, _mm_mul_ps(d, v));
}
}
}
}
inline float reduce1(__m128 a, __m128 b) { // ((v0+v1)+(v2+v3)) + ((v4+v5)+(v6+v7)), the AVX-2 kernel's tree
__m128 ha = _mm_hadd_ps(a, a); // [a0+a1, a2+a3, a0+a1, a2+a3]
ha = _mm_add_ss(ha, _mm_movehdup_ps(ha)); // (v0+v1)+(v2+v3)
__m128 hb = _mm_hadd_ps(b, b);
hb = _mm_add_ss(hb, _mm_movehdup_ps(hb)); // (v4+v5)+(v6+v7)
return _mm_cvtss_f32(_mm_add_ss(ha, hb));
}
template <int NT>
void rows_bp(const uint8_t* w, size_t row_bytes, int npairs, const ActQ* const* a, float* const* out, int r0, int r1) {
for (int r = r0; r < r1; ++r) {
__m128 acc[2 * NT];
row_bp_acc<NT>(w + (size_t) r * row_bytes, a, npairs, acc);
for (int t = 0; t < NT; ++t) out[t][r] = reduce1(acc[2 * t], acc[2 * t + 1]);
}
}
const bool kBitplane = [] {
const char* e = std::getenv("STRATA_Q2_BITPLANE");
return e != nullptr && e[0] == '1';
}();
// ActQ::qp / psum / pscale from q and scale: q2_avx2.cpp's bitplane_image, which is already pure SSE2/SSSE3
// (shuffle_epi8, sad, unpack) - copied rather than shared so q2_avx2.cpp stays untouched.
void bitplane_image(ActQ& a) {
if (a.nchunks % 4 != 0) { a.bp_pairs = 0; return; }
const int nblk = a.nchunks / 2;
const __m128i shuf = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
const __m128i bias = _mm_set1_epi8((char) 0x80);
for (int b = 0; b < nblk; ++b) {
const int8_t* q = a.q + b * 64;
__m128i s[4];
for (int g = 0; g < 4; ++g) {
const __m128i v = _mm_loadu_si128((const __m128i*) (q + 16 * g));
s[g] = _mm_shuffle_epi8(v, shuf);
const __m128i sad = _mm_sad_epu8(_mm_xor_si128(v, bias), _mm_setzero_si128());
a.psum[(b >> 1) * 8 + (b & 1) * 4 + g] =
_mm_cvtsi128_si32(sad) + _mm_extract_epi16(sad, 4) - 16 * 128;
}
const __m128i t0 = _mm_unpacklo_epi32(s[0], s[1]), t1 = _mm_unpacklo_epi32(s[2], s[3]);
const __m128i t2 = _mm_unpackhi_epi32(s[0], s[1]), t3 = _mm_unpackhi_epi32(s[2], s[3]);
int8_t* dst = a.qp + (b >> 1) * 128 + (b & 1) * 16;
_mm_store_si128((__m128i*) (dst), _mm_unpacklo_epi64(t0, t1));
_mm_store_si128((__m128i*) (dst + 32), _mm_unpackhi_epi64(t0, t1));
_mm_store_si128((__m128i*) (dst + 64), _mm_unpacklo_epi64(t2, t3));
_mm_store_si128((__m128i*) (dst + 96), _mm_unpackhi_epi64(t2, t3));
}
for (int c = 0; c < a.nchunks; ++c) {
a.pscale[2 * c] = a.scale[c];
a.pscale[2 * c + 1] = a.scale[c];
}
a.bp_pairs = nblk / 2;
}
} // namespace
void q2_0_gguf_rows_multi_avx1_legacy(const uint8_t* w, size_t row_bytes, int nblocks, const ActQ* const* a, int nt,
float* const* out, int r0, int r1) {
switch (nt) {
case 1: rows<1>(w, row_bytes, nblocks, a, out, r0, r1); break;
case 2: rows<2>(w, row_bytes, nblocks, a, out, r0, r1); break;
case 3: rows<3>(w, row_bytes, nblocks, a, out, r0, r1); break;
case 4: rows<4>(w, row_bytes, nblocks, a, out, r0, r1); break;
default:
for (int t0 = 0; t0 < nt; t0 += 4) {
const int k = nt - t0 < 4 ? nt - t0 : 4;
q2_0_gguf_rows_multi_avx1_legacy(w, row_bytes, nblocks, a + t0, k, out + t0, r0, r1);
}
}
}
void q2_0_gguf_rows_multi_avx1(const uint8_t* w, size_t row_bytes, int nblocks, const ActQ* const* a, int nt,
float* const* out, int r0, int r1) {
bool bp = kBitplane && nblocks % 2 == 0;
for (int t = 0; t < nt && bp; ++t) bp = a[t]->bp_pairs == nblocks / 2;
if (!bp) {
q2_0_gguf_rows_multi_avx1_legacy(w, row_bytes, nblocks, a, nt, out, r0, r1);
return;
}
const int np = nblocks / 2;
switch (nt) {
case 1: rows_bp<1>(w, row_bytes, np, a, out, r0, r1); break;
case 2: rows_bp<2>(w, row_bytes, np, a, out, r0, r1); break;
case 3: rows_bp<3>(w, row_bytes, np, a, out, r0, r1); break;
case 4: rows_bp<4>(w, row_bytes, np, a, out, r0, r1); break;
default:
for (int t0 = 0; t0 < nt; t0 += 4) {
const int k = nt - t0 < 4 ? nt - t0 : 4;
q2_0_gguf_rows_multi_avx1(w, row_bytes, nblocks, a + t0, k, out + t0, r0, r1);
}
}
}
void act_quant_q8_1_avx1(const float* x, int n, ActQ& a) {
a.nchunks = n / QKA;
const __m128 absmask = _mm_castsi128_ps(_mm_set1_epi32(0x7fffffff));
const __m128 half = _mm_set1_ps(0.5f), mhalf = _mm_set1_ps(-0.5f), zero = _mm_setzero_ps();
const __m128i lo = _mm_set1_epi32(-127), hi = _mm_set1_epi32(127);
for (int k = 0; k < a.nchunks; ++k) {
const float* xb = x + k * QKA;
__m128 v[8];
__m128 m = _mm_setzero_ps();
for (int i = 0; i < 8; ++i) {
v[i] = _mm_loadu_ps(xb + 4 * i);
m = _mm_max_ps(m, _mm_and_ps(v[i], absmask));
}
m = _mm_max_ps(m, _mm_movehl_ps(m, m));
const float amax = _mm_cvtss_f32(_mm_max_ss(m, _mm_movehdup_ps(m)));
const float s = amax > 0.f ? amax / 127.f : 0.f;
const float inv = s > 0.f ? 1.f / s : 0.f;
const __m128 vinv = _mm_set1_ps(inv);
__m128i sum = _mm_setzero_si128();
alignas(16) int32_t qi[QKA];
for (int i = 0; i < 8; ++i) {
const __m128 t = _mm_mul_ps(v[i], vinv);
const __m128 r = _mm_add_ps(t, _mm_blendv_ps(mhalf, half, _mm_cmp_ps(t, zero, _CMP_GE_OQ)));
__m128i q = _mm_cvttps_epi32(r);
q = _mm_min_epi32(_mm_max_epi32(q, lo), hi);
sum = _mm_add_epi32(sum, q);
_mm_store_si128((__m128i*) (qi + 4 * i), q);
}
for (int j = 0; j < QKA; ++j) a.q[k * QKA + j] = (int8_t) qi[j];
__m128i s4 = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x4E));
s4 = _mm_add_epi32(s4, _mm_shuffle_epi32(s4, 0xB1));
const int32_t total = _mm_cvtsi128_si32(s4);
a.scale[k] = s;
a.sum[k] = total;
a.hx[k] = s * (float) total;
}
if (kBitplane) bitplane_image(a);
else a.bp_pairs = 0;
}
} // namespace strata::kernels::cpu