Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 rel: one IQ1_M implementation in ds4_iq.cu (the textual merge kept both; port's kernels stay), win's dequant_rows (chunked-prompt dense dequant) kept and takes IQ1_S too; strix add4 dropped with win's IQ1_M decode (port's decode has no __vadd4)

Niko1221ea081e33126f
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index 3148b68ac..e039c4d66 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -38,7 +38,7 @@ template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; }
// The 1.5- and 1.75-bit types (IQ1_S, IQ1_M: an 11-bit index into iq1s_grid_gpu, whose 8 values are 4-bit codes) and
// IQ4_NL (32 values a block: its codebook's 16 int8 values are the first 4 words of its table)
template <> struct Ty<kIQ1_S> { static constexpr int B = 50, TAB = 2048, MIN = 0; };
-template <> struct Ty<kIQ1_M> { static constexpr int B = 56, TAB = 2048, MIN = 0; }; // (the device kernels: ggml's iq1s_grid_gpu, a u32 of eight 4-bit grid+1 values)
+template <> struct Ty<kIQ1_M> { static constexpr int B = 56, TAB = 2048, MIN = 0; };
template <> struct Ty<kIQ4_NL> { static constexpr int B = 18, TAB = 16, MIN = 0; };
template <> struct Ty<kQ2_K> { static constexpr int B = 84, TAB = 0, MIN = 1; };
template <> struct Ty<kQ3_K> { static constexpr int B = 110, TAB = 0, MIN = 0; };
@@ -97,10 +97,6 @@ __device__ __forceinline__ int sgn4(uint32_t g, uint32_t b) {
const uint32_t lo = (((b & 15u) * 0x10204080u) & 0x80808080u) >> 7;
return (int) ((g ^ (lo * 255u)) + lo);
}
-// per-byte wrapping add (CUDA's __vadd4, which HIP has not): no carry crosses a byte
-__device__ __forceinline__ uint32_t add4(uint32_t a, uint32_t b) {
- return ((a & 0x7f7f7f7fu) + (b & 0x7f7f7f7fu)) ^ ((a ^ b) & 0x80808080u);
-}
__device__ __forceinline__ int2 tab16(uint32_t q4, const uint32_t* t) { // vecdotq's get_int_from_table_16
uint32_t tmp[2];
const uint32_t sel = 0x32103210u | ((q4 & 0x88888888u) >> 1);
@@ -217,27 +213,6 @@ __device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb,
}
s.s0 = s.s1 = 2 * (int) (aux >> 28) + 1;
s.d = h2f(b) * 0.25f;
- } else if constexpr (T == kIQ1_M) { // qs[32], qh[16], scales u16[4] (four 3-bit scales each; their top nibbles are d)
- // weight = d (2 sc + 1) (grid + delta), grid in {-1, 0, 1}, delta = +-1/8: with 8 (grid + delta) = 8 v - 8 +- 1 (v =
- // grid + 1, the table's nibbles: values 0..3 are nibbles 0,2,4,6 and 4..7 nibbles 1,3,5,7) as an int8 every 8-value
- // group is an exact integer code, and d / 8 is the scale
- const uint16_t* scp = (const uint16_t*) (b + 48);
- const uint32_t sc = (uint32_t) scp[ib >> 1] >> (6 * (ib & 1));
- const uint8_t* qs = b + 4 * ib;
- const uint8_t* qh = b + 32 + 2 * ib;
-#pragma unroll
- for (int l = 0; l < 4; ++l) {
- const uint32_t h = (uint32_t) qh[l >> 1] >> (4 * (l & 1));
- const uint32_t e = tab[(uint32_t) qs[l] | ((h & 7u) << 8)];
- const uint32_t dl = (h & 8u) ? 0xf7f7f7f7u : 0xf9f9f9f9u; // -8 -+ 1 in each byte
- s.q[2 * l] = (int) add4((e & 0x0f0f0f0fu) << 3, dl);
- s.q[2 * l + 1] = (int) add4(((e >> 4) & 0x0f0f0f0fu) << 3, dl);
- }
- s.s0 = 2 * (int) (sc & 7u) + 1;
- s.s1 = 2 * (int) ((sc >> 3) & 7u) + 1;
- const uint32_t du = ((uint32_t) scp[0] >> 12) | (((uint32_t) scp[1] >> 8) & 0x00f0u) | (((uint32_t) scp[2] >> 4) & 0x0f00u) |
- ((uint32_t) scp[3] & 0xf000u);
- s.d = __half2float(__ushort_as_half((unsigned short) du)) * 0.125f;
} else if constexpr (T == kIQ3_S) {
const uint8_t* qs = b + 2;
const uint32_t w0 = ld2(qs + 8 * ib), w1 = ld2(qs + 8 * ib + 4), h = b[66 + ib], s4 = ld2(b + 74 + 4 * ib);
@@ -430,20 +405,6 @@ __device__ __forceinline__ void deq8(const uint8_t* __restrict__ row, int kb, in
const float gv = (float) ((j < 4 ? g.x >> (8 * j) : g.y >> (8 * (j - 4))) & 255u);
w[j] = (sg >> j) & 1u ? -(dl * gv) : dl * gv;
}
- } else if constexpr (T == kIQ1_M) {
- const uint16_t* scp = (const uint16_t*) (b + 48);
- const uint32_t sc = (uint32_t) scp[ib >> 1] >> (6 * (ib & 1));
- const uint32_t h = (uint32_t) b[32 + 2 * ib + (p >> 1)] >> (4 * (p & 1));
- const uint32_t e = tab[(uint32_t) b[4 * ib + p] | ((h & 7u) << 8)];
- const uint32_t du = ((uint32_t) scp[0] >> 12) | (((uint32_t) scp[1] >> 8) & 0x00f0u) | (((uint32_t) scp[2] >> 4) & 0x0f00u) |
- ((uint32_t) scp[3] & 0xf000u);
- const float dl = __half2float(__ushort_as_half((unsigned short) du)) * (float) (2 * (int) ((sc >> (p < 2 ? 0 : 3)) & 7u) + 1);
- const float delta = (h & 8u) ? -0.125f : 0.125f;
-#pragma unroll
- for (int j = 0; j < 8; ++j) {
- const int gv = (int) (((j < 4 ? e : e >> 4) >> (8 * (j & 3))) & 15u) - 1;
- w[j] = dl * ((float) gv + delta);
- }
} else if constexpr (T == kIQ3_XXS || T == kIQ3_S) {
const uint8_t* qs = b + 2;
uint32_t g1, g2, sg;
@@ -977,7 +938,6 @@ bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keeps
switch (t) { \
case kIQ2_XXS: F(kIQ2_XXS); break; \
case kIQ2_XS: F(kIQ2_XS); break; \
- case kIQ1_M: F(kIQ1_M); break; \
case kIQ2_S: F(kIQ2_S); break; \
case kIQ3_XXS: F(kIQ3_XXS); break; \
case kIQ3_S: F(kIQ3_S); break; \
@@ -1139,6 +1099,7 @@ bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t*
case kIQ3_XXS: DS4_DQ(kIQ3_XXS); break;
case kIQ3_S: DS4_DQ(kIQ3_S); break;
case kIQ4_XS: DS4_DQ(kIQ4_XS); break;
+ case kIQ1_S: DS4_DQ(kIQ1_S); break;
case kIQ1_M: DS4_DQ(kIQ1_M); break;
case kQ4_K: DS4_DQ(kQ4_K); break;
case kQ5_K: DS4_DQ(kQ5_K); break;