Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: the IQ2_XXS matvec read its sub-blocks 4 bytes apart (8), IQ2_XS signs, IQ3_XXS matvec added; more dumps and checks
Niko12219dbb13ee89a1
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 8ad2eea5f..143cef046 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -592,7 +592,8 @@ bool iq_mv(const void* jobs, int nj) {const Ds4IqMvJob& q = src[i];if (q.n > max_n) max_n = q.n;const int ncol = q.ncol > 0 ? q.ncol : 1;- const int step = ncol > 16 ? 16 : ncol; // both kernels keep 16 accumulators+ static const int maxstep = std::getenv("DS4_IQ_STEP") ? std::atoi(std::getenv("DS4_IQ_STEP")) : 16;+ const int step = ncol > maxstep ? maxstep : ncol; // both kernels keep 16 accumulatorsfor (int c0 = 0; c0 < ncol; c0 += step) {Ds4IqMvJob b = q;b.ncol = (c0 + step < ncol) ? step : ncol - c0;diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metalindex f76d59f9f..4bbea44f0 100644--- a/metal/src/engine/msl/ds4_iq.metal+++ b/metal/src/engine/msl/ds4_iq.metal@@ -24,7 +24,7 @@ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadif (int(row) >= b.n) return;const uint nsb = uint(b.k) / 32u;const uint nblk = uint(b.k) / 256u;- const uint bs = b.type == 16 ? 66u : (b.type == 17 ? 74u : 82u);+ const uint bs = b.type == 16 ? 66u : (b.type == 17 ? 74u : (b.type == 18 ? 98u : 82u));const device uchar* w = b.w + (size_t) row * b.ld;float acc[16];for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;@@ -32,14 +32,40 @@ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadconst device uchar* p = w + (size_t) blk * bs;const float d = float(as_type<half>(*(const device ushort*) p));for (uint ib = 0; ib < 8u; ++ib) {+ if (b.type == 18) { // IQ3_XXS: d, qs[64] (two grid bytes a 8 values), scales_and_signs[8] words (top nibble the scale)+ const device uchar* qs = p + 2;+ const uint w0 = uint(qs[8 * ib]) | (uint(qs[8 * ib + 1]) << 8) | (uint(qs[8 * ib + 2]) << 16) | (uint(qs[8 * ib + 3]) << 24);+ const uint w1 = uint(qs[8 * ib + 4]) | (uint(qs[8 * ib + 5]) << 8) | (uint(qs[8 * ib + 6]) << 16) | (uint(qs[8 * ib + 7]) << 24);+ const uint aux = uint(qs[64 + 4 * ib]) | (uint(qs[65 + 4 * ib]) << 8) | (uint(qs[66 + 4 * ib]) << 16) | (uint(qs[67 + 4 * ib]) << 24);+ const float db3 = d * (0.5f + float(aux >> 28)) * 0.5f;+ for (uint l = 0; l < 4u; ++l) {+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];+ const uint off = (size_t) blk * 256u + ib * 32u + l * 8u;+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ const device char* xq = b.xq + (size_t) c * b.k + off;+ int sum = 0;+ for (uint j = 0; j < 4u; ++j) {+ int v0 = int((g0 >> (8u * j)) & 0xffu), v1 = int((g1 >> (8u * j)) & 0xffu);+ if ((sgn & (1u << j)) != 0u) v0 = -v0;+ if ((sgn & (1u << (j + 4u))) != 0u) v1 = -v1;+ sum += v0 * int(xq[j]) + v1 * int(xq[j + 4u]);+ }+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * db3, float(sum), acc[c]);+ }+ }+ continue;+ }float db0, db1;uint idx[4];uint sg[4];if (b.type == 16) {- const uint aux0 = uint(p[2 + 4 * ib]) | (uint(p[3 + 4 * ib]) << 8) | (uint(p[4 + 4 * ib]) << 16) |- (uint(p[5 + 4 * ib]) << 24);- const uint aux1 = uint(p[6 + 4 * ib]) | (uint(p[7 + 4 * ib]) << 8) | (uint(p[8 + 4 * ib]) << 16) |- (uint(p[9 + 4 * ib]) << 24);+ const uint aux0 = uint(p[2 + 8 * ib]) | (uint(p[3 + 8 * ib]) << 8) | (uint(p[4 + 8 * ib]) << 16) |+ (uint(p[5 + 8 * ib]) << 24); // (8 bytes a sub-block: grid bytes, then signs and scale)+ const uint aux1 = uint(p[6 + 8 * ib]) | (uint(p[7 + 8 * ib]) << 8) | (uint(p[8 + 8 * ib]) << 16) |+ (uint(p[9 + 8 * ib]) << 24);const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;db0 = db1 = db;for (uint l = 0; l < 4u; ++l) {@@ -54,7 +80,7 @@ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadfor (uint l = 0; l < 4u; ++l) {const uint q = uint(qs[4u * ib + l]);idx[l] = q & 511u;- sg[l] = 0xffffffffu ^ ((q >> 9) * 0x00204081u); // the 7 sign bits, expanded (ggml's sign trick)+ sg[l] = uint(ksigns_iq2xs[(q >> 9) & 127u]); // the sign byte of the 7 stored bits (bit j set: value j negated)}} else {const device uchar* qs = p + 2;diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex caec54b39..6595a041f 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -7,6 +7,7 @@#endif#include "ds4_iq.cuh"+#include "ds4_gguf_cpu.hpp"#include "ds4_numerics.cuh"#include <cuda_fp16.h>@@ -1228,6 +1229,28 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {fail("mv: a low-bit job not covered");return false;}+#if defined(STRATA_METAL)+ if (std::getenv("DS4_DBG_IQ") && nij > 0) { // DBG: the low-bit matvec's first job against a host decode of its rows+ ds4_metal_flush();+ for (int ji = 0; ji < std::min(nij, 2); ++ji) {+ const iq::MvJob& b = IJ[ji];+ const uint8_t* w = (const uint8_t*) strata::gpu::host_view(b.w);+ const int8_t* xq = (const int8_t*) strata::gpu::host_view(b.xq);+ const float* xd = (const float*) strata::gpu::host_view(b.xd);+ const float* y = (const float*) strata::gpu::host_view(b.y);+ std::vector<uint16_t> row((size_t) b.k);+ for (int r = 0; r < 3; ++r) {+ if (!gg_row_to_bf16(b.type, w + (size_t) r * b.ld, b.k, row.data())) { std::fprintf(stderr, "DBG iq: no host decode for type %d\n", b.type); break; }+ double acc = 0.0;+ for (int i = 0; i < b.k; ++i) {+ uint32_t u = (uint32_t) row[(size_t) i] << 16; float wf; std::memcpy(&wf, &u, 4);+ acc += (double) wf * xq[i] * xd[i / 32];+ }+ std::fprintf(stderr, "DBG iq job %d type %d k=%d n=%d ncol=%d ld=%zu r=%d host=%.5g kernel=%.5g w=%p\n", ji, b.type, b.k, b.n, b.ncol, b.ld, r, acc, y[r], (const void*) b.w);+ }+ }+ }+#endifcheck("mv");return true;}diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex bfdca034e..a64ed75f6 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -7754,6 +7754,21 @@ struct Model::Impl {const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;const ArGroup& ag = groups[(size_t) L0.grp].ar;const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;+ if (std::getenv("DS4_DBG_SLOTS") && l.il < 2 && l.lay.fmt == gg::kFmtMXFP4) { // DBG: a resident expert's bytes against its file's+ ds4_metal_flush();+ int shown = 0;+ for (int e = 0; e < l.n_exp && shown < 3; ++e) {+ if (l.slot[(size_t) e] < 0) continue;+ std::vector<uint8_t> want(l.blob, 0);+ read_mx_file(l, e, want.data());+ const uint8_t* have = (const uint8_t*) strata::gpu::host_view(l.vram_base + (size_t) l.slot[(size_t) e] * unit(l));+ size_t bad = 0, first = 0;+ for (size_t i = 0; i < l.blob; ++i)+ if (want[i] != have[i]) { if (!bad) first = i; ++bad; }+ std::fprintf(stderr, "DBG slot il=%d e=%d slot=%d blob=%zu mismatching bytes=%zu first=%zu\n", l.il, e, l.slot[(size_t) e], l.blob, bad, first);+ ++shown;+ }+ }if (std::getenv("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBGstd::vector<float> hv((size_t) T * H * D);@@ -8246,6 +8261,9 @@ struct Model::Impl {st.cpu_experts += nslots;if (nrows > 0) {gather_rows(d.x, ga, nrows, D, d.xe, s);+ dump_sub(d, "xe", l.il, p0, nrows, d.xe, (size_t) nrows * D * 4);+ { std::vector<int> vmap; for (int i = 0; i < nvex; ++i) { vmap.push_back(vex[i]); vmap.push_back(vrow0[i]); vmap.push_back(vn[i]); }+ dump_sub_host("vmap", l.il, p0, nvex, vmap.data(), vmap.size() * 4); }x3::MvJob j[2 * kWin * kPicks];int nj = 0;for (int i = 0; i < nvex; ++i) {@@ -8254,12 +8272,16 @@ struct Model::Impl {j[nj++] = job(vm[1], d.xe + (size_t) vrow0[i] * D, d.xu + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);}if (!mv_n(d, j, nj, err)) return false;+ dump_sub(d, "xg", l.il, p0, nrows, d.xg, (size_t) nrows * g.ff * 4);+ dump_sub(d, "xu", l.il, p0, nrows, d.xu, (size_t) nrows * g.ff * 4);swiglu(d.xg, d.xu, nrows * g.ff, g.lim, d.xhh, s);+ dump_sub(d, "xhh", l.il, p0, nrows, d.xhh, (size_t) nrows * g.ff * 4);nj = 0;for (int i = 0; i < nvex; ++i)j[nj++] = job(l.vm[(size_t) vex[i]][2], d.xhh + (size_t) vrow0[i] * g.ff, d.ed + (size_t) vrow0[i] * D,vn[i], g.ff, D);if (!mv_n(d, j, nj, err)) return false;+ dump_sub(d, "ed", l.il, p0, nrows, d.ed, (size_t) nrows * D * 4);}if (!ents.empty()) {const double tc = now_ms();