Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: a type with no device kernel is known, laid out, and kept on the CPU lane (stream H1, second half)

Niko1221996b8a4044dd
diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cpp
index d9116df59..24b3635e9 100644
--- a/src/dsv41/ds4_gg_cpu.cpp
+++ b/src/dsv41/ds4_gg_cpu.cpp
@@ -27,8 +27,11 @@ size_t row_size(int type, int64_t k) { return ggml_row_size((ggml_type) type, k)
} // namespace
bool gg_cpu_type(int type) {
+ // every type ggml-cpu has a row dot for (ds4_gguf.cu's known types); the i-quants with a device kernel are usually
+ // placed in VRAM, but a pack may hand any of them to this lane
switch (type) {
- case 10: case 11: case 12: case 13: case 14: case 16: case 17: case 18: case 21: case 22: case 23: return true;
+ case 10: case 11: case 12: case 13: case 14: case 16: case 17: case 18: case 19: case 20: case 21: case 22:
+ case 23: case 29: return true;
default: return false;
}
}
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
index 9b8e87014..55d849448 100644
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@ -1002,10 +1002,12 @@ int launch_errors() { return g_errors.load(std::memory_order_relaxed); }
size_t type_row_bytes(int type, int64_t k) {
if (type == kTypeMXFP4) return k % kMxBlock == 0 ? mx_row_bytes(k) : 0;
- return iq::supported(type) && k % iq::block_values(type) == 0 ? iq::row_bytes(type, k) : 0;
+ // known(), not supported(): a type with no device kernel still has a layout, and its rows are read by the CPU lane
+ return iq::known(type) && k % iq::block_values(type) == 0 ? iq::row_bytes(type, k) : 0;
}
-int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::supported(type) ? iq::block_values(type) : 0; }
+int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::known(type) ? iq::block_values(type) : 0; }
const char* type_name(int type) { return type == kTypeMXFP4 ? "MXFP4" : iq::name(type); }
+bool type_on_device(int type) { return type == kTypeMXFP4 || iq::supported(type); }
namespace {
size_t row_stride(const x3::Mat& m) { // values (BF16) or bytes (MXFP4, Q8_0, a low-bit type) a row
diff --git a/src/dsv41/ds4_gguf.cuh b/src/dsv41/ds4_gguf.cuh
index 0b012395c..05bb2696e 100644
--- a/src/dsv41/ds4_gguf.cuh
+++ b/src/dsv41/ds4_gguf.cuh
@@ -49,6 +49,9 @@ size_t type_row_bytes(int type, int64_t k);
int type_block(int type);
/// The type's name (ggml's), "?" when unknown.
const char* type_name(int type);
+/// Whether the device kernels take this type (MXFP4 is its own path; the i-quants and K-quants ds4_iq.cu instantiates).
+/// A false here with a layout known to type_row_bytes() means the CPU lane reads it and it must not enter the VRAM tier.
+bool type_on_device(int type);
/// The per-stream scratch of the MXFP4 experts on the current device (once per stream, as x3::prepare).
bool prepare(cudaStream_t s);
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index e08bd2d35..9112befe5 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -34,6 +34,11 @@ template <> struct Ty<kIQ3_XXS> { static constexpr int B = 98, TAB = 256, MIN =
template <> struct Ty<kIQ3_S> { static constexpr int B = 110, TAB = 512, MIN = 0; };
template <> struct Ty<kIQ4_XS> { static constexpr int B = 136, TAB = 4, MIN = 0; };
template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; };
+// The types with a known layout and no device kernel (the CPU lane's ggml dots take them): their row bytes are needed to
+// lay a pack out, nothing else. Ty<> gives the block; the kernels are never instantiated for them.
+template <> struct Ty<kIQ1_S> { static constexpr int B = 50, TAB = 2048, MIN = 0; };
+template <> struct Ty<kIQ1_M> { static constexpr int B = 56, TAB = 2048, MIN = 0; };
+template <> struct Ty<kIQ4_NL> { static constexpr int B = 18, TAB = 16, MIN = 0; };
template <> struct Ty<kQ2_K> { static constexpr int B = 84, TAB = 0, MIN = 1; };
template <> struct Ty<kQ3_K> { static constexpr int B = 110, TAB = 0, MIN = 0; };
// the dense matrices' 5- and 6-bit K-quants (GEMV only: ds4_iq's mv; not an expert type)
@@ -967,9 +972,23 @@ void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {
} // namespace
-bool supported(int t) { return block_bytes(t) > 0; }
+bool supported(int t) {
+ // the types the kernels below are instantiated for (DS4_IQ_SWITCH + the trellis ones + the dense K-quants)
+ switch (t) {
+ case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS:
+ case kQ4_K: case kQ5_K: case kQ6_K: case kQ2_K: case kQ3_K:
+ return true;
+ default:
+ return is_tq(t);
+ }
+}
+
+bool known(int t) { return block_bytes(t) > 0; }
-int block_values(int t) { return is_tq(t) ? 128 : block_bytes(t) > 0 ? 256 : 0; }
+int block_values(int t) {
+ if (t == kIQ4_NL) return 32; // (ggml's QK4_NL: the only type here whose block is not 256 values)
+ return is_tq(t) ? 128 : block_bytes(t) > 0 ? 256 : 0;
+}
int block_bytes(int t) {
if (is_tq(t)) return 2 + 4 * tq_kbits(t);
@@ -985,6 +1004,9 @@ int block_bytes(int t) {
case kQ6_K: return Ty<kQ6_K>::B;
case kQ2_K: return Ty<kQ2_K>::B;
case kQ3_K: return Ty<kQ3_K>::B;
+ case kIQ1_S: return Ty<kIQ1_S>::B; // (a known layout, no device kernel: the CPU lane's)
+ case kIQ1_M: return Ty<kIQ1_M>::B;
+ case kIQ4_NL: return Ty<kIQ4_NL>::B;
default: return 0;
}
}
@@ -1008,6 +1030,9 @@ const char* name(int t) {
case kQ6_K: return "Q6_K";
case kQ2_K: return "Q2_K";
case kQ3_K: return "Q3_K";
+ case kIQ1_S: return "IQ1_S";
+ case kIQ1_M: return "IQ1_M";
+ case kIQ4_NL: return "IQ4_NL";
default: return "?";
}
}
diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuh
index 9a93c7e07..0cf49c957 100644
--- a/src/dsv41/ds4_iq.cuh
+++ b/src/dsv41/ds4_iq.cuh
@@ -20,8 +20,8 @@ namespace strata::ds4::gguf::iq {
// ggml type ids
constexpr int kQ5_K = 13, kQ6_K = 14; // (the dense matrices' K-quants, GEMV only)
-constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ2_XS = 17, kIQ3_XXS = 18, kIQ3_S = 21,
- kIQ2_S = 22, kIQ4_XS = 23;
+constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ2_XS = 17, kIQ3_XXS = 18, kIQ1_S = 19,
+ kIQ4_NL = 20, kIQ3_S = 21, kIQ2_S = 22, kIQ4_XS = 23, kIQ1_M = 29;
/// The trellis types (Strata's, ds4_tq.cu): ggml's TQK6 / TQK7 (145 / 146, the table codebook) and the experimental
/// TQK5X / TQK4X (224 / 225, computed codebook) and TQK5T / TQK4T (226 / 227, table): a block of 128 values is
@@ -34,6 +34,9 @@ constexpr bool is_tq(int type) { return tq_kbits(type) > 0; }
/// Whether the device kernels take this ggml type.
bool supported(int type);
+/// Whether the type's block layout is known (its row bytes can be computed) - a superset of supported(): a type the CPU
+/// lane's ggml dots take but no device kernel does (IQ1_S, IQ1_M, IQ4_NL today) is known and must stay out of VRAM.
+bool known(int type);
/// Values a block of the type holds (256; the trellis types 128; 0: not one of these).
int block_values(int type);
/// Bytes of a block of the type (0: not one of these).
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 72979866c..cd4a5222c 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -629,6 +629,7 @@ struct Layer {
x3::ExpertLayout lay;
size_t blob = 0; // bytes of one expert's blob
int n_vram = 0;
+ bool cpu_only = false; // a role's type has no device kernel: every expert of this layer is the CPU lane's
uint8_t* vram_base = nullptr;
std::vector<int> slot; // expert -> VRAM slot or -1
std::vector<int> ram; // expert -> RAM-tier slot or -1 (a GGUF's in-place experts; empty: none)
@@ -1527,7 +1528,7 @@ struct Model::Impl {
// expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)
// (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)
static int gg_type_of(const std::string& dtype) {
- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})
+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 145, 146, 224, 225, 226, 227})
if (dtype == gg::type_name(t)) return t;
return -1;
}
@@ -1571,6 +1572,17 @@ struct Model::Impl {
return false;
}
l.blob = off;
+ // a type the device kernels do not take (IQ1_S / IQ1_M / IQ4_NL today): the layer stays on the CPU lane, where
+ // ggml-cpu's row dots read it - one warning, then the placement code keeps it out of VRAM
+ for (int r = 0; r < 3; ++r)
+ if (!gg::type_on_device(l.lay.K[r])) {
+ l.cpu_only = true;
+ std::fprintf(stderr,
+ "ds4: layer %d %s is %s: no GPU kernel for it, its experts stay on the CPU lane "
+ "(ggml-cpu's rows; VRAM tier: none)\n", l.il, r == 2 ? "down" : "gate/up",
+ gg::type_name(l.lay.K[r]));
+ break;
+ }
return true;
}
const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's
@@ -3536,10 +3548,14 @@ struct Model::Impl {
}
double want = 0;
for (const Layer& l : L)
- if (l.grp == (int) gi && !l.mtp) want += (double) ucount(l) * l.n_exp;
+ if (l.grp == (int) gi && !l.mtp && !l.cpu_only) want += (double) ucount(l) * l.n_exp;
const double frac = want > 0 ? std::min(1.0, std::max(0.0, budget_t) / want) : 0.0;
for (Layer& l : L) {
if (l.grp != (int) gi || l.mtp) continue;
+ if (l.cpu_only) { // (its type has no device kernel: the CPU lane reads every expert of it)
+ l.n_vram = 0;
+ continue;
+ }
l.n_vram = std::min(l.n_exp, (int) std::floor(frac * l.n_exp));
// never more than fits: the floor of each layer's share can still round up the total
while (l.n_vram > 0 && (double) ucount(l) * l.n_vram > budget) --l.n_vram;
@@ -3549,7 +3565,7 @@ struct Model::Impl {
if (l.grp == (int) gi) used += (double) ucount(l) * l.n_vram;
// what the floors left: whole experts, layer by layer
for (Layer& l : L) {
- if (l.grp != (int) gi) continue;
+ if (l.grp != (int) gi || l.cpu_only) continue;
while (l.n_vram < l.n_exp && used + (double) ucount(l) <= budget) {
++l.n_vram;
used += (double) ucount(l);
diff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cpp
index ff989087e..a7bb1894e 100644
--- a/tests/dsv41/iq_expert_test.cpp
+++ b/tests/dsv41/iq_expert_test.cpp
@@ -160,7 +160,8 @@ int main(int argc, char** argv) {
fails += !ok;
};
for (int type : types) {
- if (!iq::supported(type)) { std::printf("%-7s not a device type here\n", iq::name(type)); ++fails; continue; }
+ if (!iq::supported(type) && !iq::known(type)) { std::printf("%-7s not a type this engine knows\n", iq::name(type)); ++fails; continue; }
+ const bool dev = iq::supported(type); // (IQ1_S / IQ1_M / IQ4_NL: a known layout, no device kernel - CPU lane only)
const int E = 512, FF = 256, NEXP = 3, T = 4;
const float lim = 7.0f;
const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);
@@ -188,10 +189,18 @@ int main(int argc, char** argv) {
for (int t = 0; t < T; ++t) { qx[t] = quant(&x[(size_t) t * E], E); std::memcpy(&xq[(size_t) t * E], qx[t].q.data(), E); std::memcpy(&xd[(size_t) t * E / 32], qx[t].d.data(), E / 32 * 4); }
const int8_t* dxq = up(xq); const float* dxd = up(xd);
- // ---- token path
+ // ---- the reference h of every expert's rows for token 0 (the CPU lane and the device share it)
std::vector<float> href((size_t) NEXP * FF);
std::vector<Q8> qh(NEXP);
- {
+ for (int e = 0; e < NEXP; ++e)
+ for (int r = 0; r < FF; ++r) {
+ double sc;
+ href[(size_t) e * FF + r] = (float) refh(e, r, qx[0].xhat.data(), sc);
+ }
+ for (int e = 0; e < NEXP; ++e) qh[e] = quant(&href[(size_t) e * FF], FF);
+
+ // ---- token path (device; skipped for a type with no device kernel)
+ if (dev) {
std::vector<unsigned long long> plan(NEXP);
for (int e = 0; e < NEXP; ++e) plan[e] = (unsigned long long) (dblob + e * blob);
iq::Args a{};
@@ -206,12 +215,11 @@ int main(int argc, char** argv) {
for (int r = 0; r < FF; ++r) {
double sc;
const double ref = refh(e, r, qx[0].xhat.data(), sc);
- href[(size_t) e * FF + r] = (float) ref;
worst = std::max(worst, std::fabs(h[(size_t) e * FF + r] - ref) / sc);
}
report("token gate_up", type, worst, 1e-5);
std::vector<int8_t> hq((size_t) NEXP * FF); std::vector<float> hd((size_t) NEXP * FF / 32);
- for (int e = 0; e < NEXP; ++e) { qh[e] = quant(&href[(size_t) e * FF], FF); std::memcpy(&hq[(size_t) e * FF], qh[e].q.data(), FF); std::memcpy(&hd[(size_t) e * FF / 32], qh[e].d.data(), FF / 32 * 4); }
+ for (int e = 0; e < NEXP; ++e) { std::memcpy(&hq[(size_t) e * FF], qh[e].q.data(), FF); std::memcpy(&hd[(size_t) e * FF / 32], qh[e].d.data(), FF / 32 * 4); }
a.hq = up(hq); a.hd = up(hd);
float* dp; CK(hipMalloc(&dp, (size_t) NEXP * E * 4)); a.part = dp;
if (!iq::down(a, NEXP, 1, 0)) { std::printf("down refused\n"); return 1; }