Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: a type with no device kernel is known, laid out, and kept on the CPU lane (stream H1, second half)
Niko1221996b8a4044dd
diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cppindex d9116df59..24b3635e9 100644--- a/src/dsv41/ds4_gg_cpu.cpp+++ b/src/dsv41/ds4_gg_cpu.cpp@@ -27,8 +27,11 @@ size_t row_size(int type, int64_t k) { return ggml_row_size((ggml_type) type, k)} // namespacebool gg_cpu_type(int type) {+ // every type ggml-cpu has a row dot for (ds4_gguf.cu's known types); the i-quants with a device kernel are usually+ // placed in VRAM, but a pack may hand any of them to this laneswitch (type) {- case 10: case 11: case 12: case 13: case 14: case 16: case 17: case 18: case 21: case 22: case 23: return true;+ case 10: case 11: case 12: case 13: case 14: case 16: case 17: case 18: case 19: case 20: case 21: case 22:+ case 23: case 29: return true;default: return false;}}diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex 9b8e87014..55d849448 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -1002,10 +1002,12 @@ int launch_errors() { return g_errors.load(std::memory_order_relaxed); }size_t type_row_bytes(int type, int64_t k) {if (type == kTypeMXFP4) return k % kMxBlock == 0 ? mx_row_bytes(k) : 0;- return iq::supported(type) && k % iq::block_values(type) == 0 ? iq::row_bytes(type, k) : 0;+ // known(), not supported(): a type with no device kernel still has a layout, and its rows are read by the CPU lane+ return iq::known(type) && k % iq::block_values(type) == 0 ? iq::row_bytes(type, k) : 0;}-int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::supported(type) ? iq::block_values(type) : 0; }+int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::known(type) ? iq::block_values(type) : 0; }const char* type_name(int type) { return type == kTypeMXFP4 ? "MXFP4" : iq::name(type); }+bool type_on_device(int type) { return type == kTypeMXFP4 || iq::supported(type); }namespace {size_t row_stride(const x3::Mat& m) { // values (BF16) or bytes (MXFP4, Q8_0, a low-bit type) a rowdiff --git a/src/dsv41/ds4_gguf.cuh b/src/dsv41/ds4_gguf.cuhindex 0b012395c..05bb2696e 100644--- a/src/dsv41/ds4_gguf.cuh+++ b/src/dsv41/ds4_gguf.cuh@@ -49,6 +49,9 @@ size_t type_row_bytes(int type, int64_t k);int type_block(int type);/// The type's name (ggml's), "?" when unknown.const char* type_name(int type);+/// Whether the device kernels take this type (MXFP4 is its own path; the i-quants and K-quants ds4_iq.cu instantiates).+/// A false here with a layout known to type_row_bytes() means the CPU lane reads it and it must not enter the VRAM tier.+bool type_on_device(int type);/// The per-stream scratch of the MXFP4 experts on the current device (once per stream, as x3::prepare).bool prepare(cudaStream_t s);diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex e08bd2d35..9112befe5 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -34,6 +34,11 @@ template <> struct Ty<kIQ3_XXS> { static constexpr int B = 98, TAB = 256, MIN =template <> struct Ty<kIQ3_S> { static constexpr int B = 110, TAB = 512, MIN = 0; };template <> struct Ty<kIQ4_XS> { static constexpr int B = 136, TAB = 4, MIN = 0; };template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; };+// The types with a known layout and no device kernel (the CPU lane's ggml dots take them): their row bytes are needed to+// lay a pack out, nothing else. Ty<> gives the block; the kernels are never instantiated for them.+template <> struct Ty<kIQ1_S> { static constexpr int B = 50, TAB = 2048, MIN = 0; };+template <> struct Ty<kIQ1_M> { static constexpr int B = 56, TAB = 2048, MIN = 0; };+template <> struct Ty<kIQ4_NL> { static constexpr int B = 18, TAB = 16, MIN = 0; };template <> struct Ty<kQ2_K> { static constexpr int B = 84, TAB = 0, MIN = 1; };template <> struct Ty<kQ3_K> { static constexpr int B = 110, TAB = 0, MIN = 0; };// the dense matrices' 5- and 6-bit K-quants (GEMV only: ds4_iq's mv; not an expert type)@@ -967,9 +972,23 @@ void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {} // namespace-bool supported(int t) { return block_bytes(t) > 0; }+bool supported(int t) {+ // the types the kernels below are instantiated for (DS4_IQ_SWITCH + the trellis ones + the dense K-quants)+ switch (t) {+ case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS:+ case kQ4_K: case kQ5_K: case kQ6_K: case kQ2_K: case kQ3_K:+ return true;+ default:+ return is_tq(t);+ }+}++bool known(int t) { return block_bytes(t) > 0; }-int block_values(int t) { return is_tq(t) ? 128 : block_bytes(t) > 0 ? 256 : 0; }+int block_values(int t) {+ if (t == kIQ4_NL) return 32; // (ggml's QK4_NL: the only type here whose block is not 256 values)+ return is_tq(t) ? 128 : block_bytes(t) > 0 ? 256 : 0;+}int block_bytes(int t) {if (is_tq(t)) return 2 + 4 * tq_kbits(t);@@ -985,6 +1004,9 @@ int block_bytes(int t) {case kQ6_K: return Ty<kQ6_K>::B;case kQ2_K: return Ty<kQ2_K>::B;case kQ3_K: return Ty<kQ3_K>::B;+ case kIQ1_S: return Ty<kIQ1_S>::B; // (a known layout, no device kernel: the CPU lane's)+ case kIQ1_M: return Ty<kIQ1_M>::B;+ case kIQ4_NL: return Ty<kIQ4_NL>::B;default: return 0;}}@@ -1008,6 +1030,9 @@ const char* name(int t) {case kQ6_K: return "Q6_K";case kQ2_K: return "Q2_K";case kQ3_K: return "Q3_K";+ case kIQ1_S: return "IQ1_S";+ case kIQ1_M: return "IQ1_M";+ case kIQ4_NL: return "IQ4_NL";default: return "?";}}diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex 9a93c7e07..0cf49c957 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -20,8 +20,8 @@ namespace strata::ds4::gguf::iq {// ggml type idsconstexpr int kQ5_K = 13, kQ6_K = 14; // (the dense matrices' K-quants, GEMV only)-constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ2_XS = 17, kIQ3_XXS = 18, kIQ3_S = 21,- kIQ2_S = 22, kIQ4_XS = 23;+constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ2_XS = 17, kIQ3_XXS = 18, kIQ1_S = 19,+ kIQ4_NL = 20, kIQ3_S = 21, kIQ2_S = 22, kIQ4_XS = 23, kIQ1_M = 29;/// The trellis types (Strata's, ds4_tq.cu): ggml's TQK6 / TQK7 (145 / 146, the table codebook) and the experimental/// TQK5X / TQK4X (224 / 225, computed codebook) and TQK5T / TQK4T (226 / 227, table): a block of 128 values is@@ -34,6 +34,9 @@ constexpr bool is_tq(int type) { return tq_kbits(type) > 0; }/// Whether the device kernels take this ggml type.bool supported(int type);+/// Whether the type's block layout is known (its row bytes can be computed) - a superset of supported(): a type the CPU+/// lane's ggml dots take but no device kernel does (IQ1_S, IQ1_M, IQ4_NL today) is known and must stay out of VRAM.+bool known(int type);/// Values a block of the type holds (256; the trellis types 128; 0: not one of these).int block_values(int type);/// Bytes of a block of the type (0: not one of these).diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 72979866c..cd4a5222c 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -629,6 +629,7 @@ struct Layer {x3::ExpertLayout lay;size_t blob = 0; // bytes of one expert's blobint n_vram = 0;+ bool cpu_only = false; // a role's type has no device kernel: every expert of this layer is the CPU lane'suint8_t* vram_base = nullptr;std::vector<int> slot; // expert -> VRAM slot or -1std::vector<int> ram; // expert -> RAM-tier slot or -1 (a GGUF's in-place experts; empty: none)@@ -1527,7 +1528,7 @@ struct Model::Impl {// expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)// (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)static int gg_type_of(const std::string& dtype) {- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 145, 146, 224, 225, 226, 227})if (dtype == gg::type_name(t)) return t;return -1;}@@ -1571,6 +1572,17 @@ struct Model::Impl {return false;}l.blob = off;+ // a type the device kernels do not take (IQ1_S / IQ1_M / IQ4_NL today): the layer stays on the CPU lane, where+ // ggml-cpu's row dots read it - one warning, then the placement code keeps it out of VRAM+ for (int r = 0; r < 3; ++r)+ if (!gg::type_on_device(l.lay.K[r])) {+ l.cpu_only = true;+ std::fprintf(stderr,+ "ds4: layer %d %s is %s: no GPU kernel for it, its experts stay on the CPU lane "+ "(ggml-cpu's rows; VRAM tier: none)\n", l.il, r == 2 ? "down" : "gate/up",+ gg::type_name(l.lay.K[r]));+ break;+ }return true;}const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's@@ -3536,10 +3548,14 @@ struct Model::Impl {}double want = 0;for (const Layer& l : L)- if (l.grp == (int) gi && !l.mtp) want += (double) ucount(l) * l.n_exp;+ if (l.grp == (int) gi && !l.mtp && !l.cpu_only) want += (double) ucount(l) * l.n_exp;const double frac = want > 0 ? std::min(1.0, std::max(0.0, budget_t) / want) : 0.0;for (Layer& l : L) {if (l.grp != (int) gi || l.mtp) continue;+ if (l.cpu_only) { // (its type has no device kernel: the CPU lane reads every expert of it)+ l.n_vram = 0;+ continue;+ }l.n_vram = std::min(l.n_exp, (int) std::floor(frac * l.n_exp));// never more than fits: the floor of each layer's share can still round up the totalwhile (l.n_vram > 0 && (double) ucount(l) * l.n_vram > budget) --l.n_vram;@@ -3549,7 +3565,7 @@ struct Model::Impl {if (l.grp == (int) gi) used += (double) ucount(l) * l.n_vram;// what the floors left: whole experts, layer by layerfor (Layer& l : L) {- if (l.grp != (int) gi) continue;+ if (l.grp != (int) gi || l.cpu_only) continue;while (l.n_vram < l.n_exp && used + (double) ucount(l) <= budget) {++l.n_vram;used += (double) ucount(l);diff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cppindex ff989087e..a7bb1894e 100644--- a/tests/dsv41/iq_expert_test.cpp+++ b/tests/dsv41/iq_expert_test.cpp@@ -160,7 +160,8 @@ int main(int argc, char** argv) {fails += !ok;};for (int type : types) {- if (!iq::supported(type)) { std::printf("%-7s not a device type here\n", iq::name(type)); ++fails; continue; }+ if (!iq::supported(type) && !iq::known(type)) { std::printf("%-7s not a type this engine knows\n", iq::name(type)); ++fails; continue; }+ const bool dev = iq::supported(type); // (IQ1_S / IQ1_M / IQ4_NL: a known layout, no device kernel - CPU lane only)const int E = 512, FF = 256, NEXP = 3, T = 4;const float lim = 7.0f;const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);@@ -188,10 +189,18 @@ int main(int argc, char** argv) {for (int t = 0; t < T; ++t) { qx[t] = quant(&x[(size_t) t * E], E); std::memcpy(&xq[(size_t) t * E], qx[t].q.data(), E); std::memcpy(&xd[(size_t) t * E / 32], qx[t].d.data(), E / 32 * 4); }const int8_t* dxq = up(xq); const float* dxd = up(xd);- // ---- token path+ // ---- the reference h of every expert's rows for token 0 (the CPU lane and the device share it)std::vector<float> href((size_t) NEXP * FF);std::vector<Q8> qh(NEXP);- {+ for (int e = 0; e < NEXP; ++e)+ for (int r = 0; r < FF; ++r) {+ double sc;+ href[(size_t) e * FF + r] = (float) refh(e, r, qx[0].xhat.data(), sc);+ }+ for (int e = 0; e < NEXP; ++e) qh[e] = quant(&href[(size_t) e * FF], FF);++ // ---- token path (device; skipped for a type with no device kernel)+ if (dev) {std::vector<unsigned long long> plan(NEXP);for (int e = 0; e < NEXP; ++e) plan[e] = (unsigned long long) (dblob + e * blob);iq::Args a{};@@ -206,12 +215,11 @@ int main(int argc, char** argv) {for (int r = 0; r < FF; ++r) {double sc;const double ref = refh(e, r, qx[0].xhat.data(), sc);- href[(size_t) e * FF + r] = (float) ref;worst = std::max(worst, std::fabs(h[(size_t) e * FF + r] - ref) / sc);}report("token gate_up", type, worst, 1e-5);std::vector<int8_t> hq((size_t) NEXP * FF); std::vector<float> hd((size_t) NEXP * FF / 32);- for (int e = 0; e < NEXP; ++e) { qh[e] = quant(&href[(size_t) e * FF], FF); std::memcpy(&hq[(size_t) e * FF], qh[e].q.data(), FF); std::memcpy(&hd[(size_t) e * FF / 32], qh[e].d.data(), FF / 32 * 4); }+ for (int e = 0; e < NEXP; ++e) { std::memcpy(&hq[(size_t) e * FF], qh[e].q.data(), FF); std::memcpy(&hd[(size_t) e * FF / 32], qh[e].d.data(), FF / 32 * 4); }a.hq = up(hq); a.hd = up(hd);float* dp; CK(hipMalloc(&dp, (size_t) NEXP * E * 4)); a.part = dp;if (!iq::down(a, NEXP, 1, 0)) { std::printf("down refused\n"); return 1; }