Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: dense K-quant GEMV (Q5_K/Q6_K) + dense_requant, ggml-type CPU lane, imatrix builder, requant fixes (C0 experiments)

Niko12215278ba00431e
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b4082fdd6..a054a73ee 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -777,7 +777,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_link_libraries(strata_dsv41_kernels PUBLIC ${_strata_gpu_runtime_target})
add_executable(strata-dsv41 src/dsv41/ds4_main.cpp src/dsv41/ds4_model.cu src/dsv41/ds4_kernels.cu src/dsv41/ds4_vision.cu
src/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp
- src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp)
+ src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp)
target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)
target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})
diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cpp
new file mode 100644
index 000000000..3a9a7728d
--- /dev/null
+++ b/src/dsv41/ds4_gg_cpu.cpp
@@ -0,0 +1,118 @@
+// Strata. The CPU lane of the DeepSeek-V4.1 engine path for ggml's block types (IQ2_XXS .. Q4_K, the experts of a requantized model):
+// ggml-cpu's own row dots (type traits: vec_dot over the Q8_K / Q8_0 activation of vec_dot_type), one row at a time. Maya's lane for
+// these types (native_expert split-base rows) is not in Strata's tree; this one is plain and complete, not tuned.
+#include "ds4_gguf_cpu.hpp"
+
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+
+#if defined(STRATA_DSV41_TQ_CPU)
+#include "ggml-cpu.h"
+#include "ggml.h"
+
+#include <mutex>
+#include <vector>
+
+namespace strata::ds4::gguf {
+namespace {
+void init_once() {
+ static std::once_flag f;
+ std::call_once(f, [] { ggml_cpu_init(); });
+}
+size_t row_size(int type, int64_t k) { return ggml_row_size((ggml_type) type, k); }
+} // namespace
+
+bool gg_cpu_type(int type) {
+ switch (type) {
+ case 10: case 11: case 12: case 13: case 14: case 16: case 17: case 18: case 21: case 22: case 23: return true;
+ default: return false;
+ }
+}
+
+static bool dbg() { static const bool on = std::getenv("DS4_DBG_GG") != nullptr; return on; }
+bool gg_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {
+ if (dbg()) std::fprintf(stderr, "gg_cpu_fmt %d %d\n", tg, td);
+ init_once();
+ if (!gg_cpu_type(tg) || !gg_cpu_type(td)) {
+ err = "not ggml block types";
+ return false;
+ }
+ const ggml_type_traits_cpu* a = ggml_get_type_traits_cpu((ggml_type) tg);
+ const ggml_type_traits_cpu* b = ggml_get_type_traits_cpu((ggml_type) td);
+ if (a == nullptr || b == nullptr || a->vec_dot == nullptr || b->vec_dot == nullptr) {
+ err = "ggml-cpu has no row dot for these types";
+ return false;
+ }
+ f.tg = tg;
+ f.td = td;
+ f.n_embd = n_embd;
+ f.n_ff = n_ff;
+ f.gu_act = (int) a->vec_dot_type;
+ f.d_act = (int) b->vec_dot_type;
+ f.gu_row = row_size(tg, n_embd);
+ f.d_row = row_size(td, n_ff);
+ f.act_bytes = row_size(f.gu_act, n_embd);
+ f.h_bytes = row_size(f.d_act, n_ff);
+ return true;
+}
+
+// x (k values) -> the activation type (Q8_K, Q8_0 ..) ggml-cpu's dot takes
+void gg_cpu_quant(int act, const float* x, int k, void* dst) {
+ init_once();
+ if (dbg()) std::fprintf(stderr, "gg_cpu_quant act %d k %d\n", act, k);
+ const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) act);
+ if (tc != nullptr && tc->from_float != nullptr) {
+ tc->from_float(x, dst, k);
+ return;
+ }
+ const ggml_type_traits* t = ggml_get_type_traits((ggml_type) act);
+ if (dbg()) std::fprintf(stderr, "gg_cpu_quant: no cpu from_float, ref %p\n", (void*) (t ? t->from_float_ref : nullptr));
+ t->from_float_ref(x, dst, k);
+}
+
+static void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {
+ const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) type);
+ if (dbg()) std::fprintf(stderr, "gg dots type %d n %d rows %d..%d nt %d\n", type, n, r0, r1, nt);
+ for (int r = r0; r < r1; ++r)
+ for (int t = 0; t < nt; ++t) tc->vec_dot(n, &out[t][r], 0, w + (size_t) r * rb, 0, act[t], 0, 1);
+}
+
+void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff, int r0,
+ int r1, float limit) {
+ static thread_local std::vector<float> tmp[8];
+ float* uo[8];
+ for (int t = 0; t < nt; ++t) {
+ tmp[t].resize((size_t) r1);
+ uo[t] = tmp[t].data();
+ }
+ dots(f.tg, gate, f.gu_row, f.n_embd, act, nt, ff, r0, r1);
+ dots(f.tg, up, f.gu_row, f.n_embd, act, nt, uo, r0, r1);
+ for (int t = 0; t < nt; ++t)
+ for (int r = r0; r < r1; ++r) {
+ float g = ff[t][r], u = uo[t][r];
+ if (limit > 0.0f) {
+ g = g < limit ? g : limit;
+ u = u < -limit ? -limit : (u > limit ? limit : u);
+ }
+ ff[t][r] = g / (1.0f + std::exp(-g)) * u;
+ }
+}
+
+void gg_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
+ dots(f.td, down, f.d_row, f.n_ff, hq, nt, out, r0, r1);
+}
+} // namespace strata::ds4::gguf
+#else
+namespace strata::ds4::gguf {
+bool gg_cpu_type(int) { return false; }
+bool gg_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {
+ err = "this build has no ggml-cpu lane (STRATA_NATIVE_EXPERTS=OFF)";
+ return false;
+}
+void gg_cpu_quant(int, const float*, int, void*) {}
+void gg_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int, float) {}
+void gg_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}
+} // namespace strata::ds4::gguf
+#endif
diff --git a/src/dsv41/ds4_gguf_cpu.hpp b/src/dsv41/ds4_gguf_cpu.hpp
index 286afd2ea..cc9f0d8e8 100644
--- a/src/dsv41/ds4_gguf_cpu.hpp
+++ b/src/dsv41/ds4_gguf_cpu.hpp
@@ -46,6 +46,14 @@ void tq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, cons
int r0, int r1, float limit);
void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);
+/// ggml block types on the CPU lane through ggml-cpu's row dots (ds4_gg_cpu.cpp), for the IQ / K-quant experts.
+bool gg_cpu_type(int type);
+bool gg_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);
+void gg_cpu_quant(int act, const float* x, int k, void* dst);
+void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
+ int r0, int r1, float limit);
+void gg_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);
+
/// The CPU lane's MXFP4 rows in the device's arithmetic (ds4_gguf.cu's mx_dot): the activations rounded to int8 per 32
/// values as the device rounds them (q8_round's q and scale), each block's products summed exactly in integers, then
/// times (weight scale x activation scale) into F32 sums - the same numbers as the device up to the order of the
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index ba62fccce..e99c1dcb5 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -35,6 +35,9 @@ template <> struct Ty<kIQ4_XS> { static constexpr int B = 136, TAB = 4, MIN = 0;
template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; };
template <> struct Ty<kQ2_K> { static constexpr int B = 84, TAB = 0, MIN = 1; };
template <> struct Ty<kQ3_K> { static constexpr int B = 110, TAB = 0, MIN = 0; };
+// the dense matrices' 5- and 6-bit K-quants (GEMV only: ds4_iq's mv; not an expert type)
+template <> struct Ty<kQ5_K> { static constexpr int B = 176, TAB = 0, MIN = 1; };
+template <> struct Ty<kQ6_K> { static constexpr int B = 210, TAB = 0, MIN = 0; };
// MXFP4 (ds4_gguf.cu's own kernels everywhere else): a block of 32 values, an E8M0 scale then 16 code bytes; the light
// sets take it too
constexpr int kMXFP4 = 39;
@@ -207,6 +210,32 @@ __device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb,
s.s0 = s.s1 = sc;
s.d = h2f(b);
s.m0 = s.m1 = h2f(b + 2) * (float) m;
+ } else if constexpr (T == kQ5_K) { // d, dmin, scales[12], qh[32], qs[128]: weight = d sc (nibble + 16 hbit) - dmin m
+ int sc, m;
+ q4k_scale(b + 4, ib, sc, m);
+ const uint8_t* qs = b + 48 + 32 * (ib >> 1);
+ const uint8_t* qh = b + 16;
+ const int sh = 4 * (ib & 1);
+#pragma unroll
+ for (int t = 0; t < 8; ++t)
+ s.q[t] = (int) (((ld4(qs + 4 * t) >> sh) & 0x0f0f0f0fu) | (((ld4(qh + 4 * t) >> ib) & 0x01010101u) << 4));
+ s.s0 = s.s1 = sc;
+ s.d = h2f(b);
+ s.m0 = s.m1 = h2f(b + 2) * (float) m;
+ } else if constexpr (T == kQ6_K) { // ql[128], qh[64], scales int8[16], d: weight = d sc (q - 32), q 6 bits
+ const int h = ib >> 2, quarter = ib & 3;
+ const uint8_t* ql = b + 64 * h + (quarter & 1 ? 32 : 0);
+ const uint8_t* qh = b + 128 + 32 * h;
+ const int sh = 4 * (quarter >> 1);
+#pragma unroll
+ for (int t = 0; t < 8; ++t) {
+ const uint32_t x = ((ld2(ql + 4 * t) >> sh) & 0x0f0f0f0fu) | (((ld2(qh + 4 * t) >> (2 * quarter)) & 0x03030303u) << 4);
+ s.q[t] = (int) __vsub4(x, 0x20202020u);
+ }
+ const int8_t* scl = (const int8_t*) (b + 192) + 8 * h + 2 * quarter;
+ s.s0 = scl[0];
+ s.s1 = scl[1];
+ s.d = h2f(b + 208);
} else if constexpr (T == kQ2_K) {
const int n = ib >> 2, j = ib & 3;
const uint8_t* qs = b + 16 + 32 * n;
@@ -921,6 +950,8 @@ int block_bytes(int t) {
case kIQ3_S: return Ty<kIQ3_S>::B;
case kIQ4_XS: return Ty<kIQ4_XS>::B;
case kQ4_K: return Ty<kQ4_K>::B;
+ case kQ5_K: return Ty<kQ5_K>::B;
+ case kQ6_K: return Ty<kQ6_K>::B;
case kQ2_K: return Ty<kQ2_K>::B;
case kQ3_K: return Ty<kQ3_K>::B;
default: return 0;
@@ -941,6 +972,8 @@ const char* name(int t) {
case kIQ3_S: return "IQ3_S";
case kIQ4_XS: return "IQ4_XS";
case kQ4_K: return "Q4_K";
+ case kQ5_K: return "Q5_K";
+ case kQ6_K: return "Q6_K";
case kQ2_K: return "Q2_K";
case kQ3_K: return "Q3_K";
default: return "?";
@@ -1037,7 +1070,11 @@ bool mv(const MvJob* jobs, int n, cudaStream_t s) {
done[k] = true;
}
#define MV_(T) mv_nc<T>(J, nj, max_n, max_c, s)
- DS4_IQ_SWITCH(t, MV_)
+ switch (t) { // the experts' types and the dense matrices' 5- and 6-bit ones
+ case kQ5_K: MV_(kQ5_K); break;
+ case kQ6_K: MV_(kQ6_K); break;
+ default: DS4_IQ_SWITCH(t, MV_)
+ }
#undef MV_
}
return true;
diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuh
index f13199fdc..3f0c8f52b 100644
--- a/src/dsv41/ds4_iq.cuh
+++ b/src/dsv41/ds4_iq.cuh
@@ -19,6 +19,7 @@
namespace strata::ds4::gguf::iq {
// ggml type ids
+constexpr int kQ5_K = 13, kQ6_K = 14; // (the dense matrices' K-quants, GEMV only)
constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ3_XXS = 18, kIQ3_S = 21, kIQ2_S = 22,
kIQ4_XS = 23;
diff --git a/src/dsv41/ds4_iq_cpu.cpp b/src/dsv41/ds4_iq_cpu.cpp
index e0f04678a..d472dacfc 100644
--- a/src/dsv41/ds4_iq_cpu.cpp
+++ b/src/dsv41/ds4_iq_cpu.cpp
@@ -759,18 +759,22 @@ bool iq_cpu_own_rows(int type) { return own(type); }
// (no ggml-cpu lane for the low-bit types here; the trellis types have their own: ds4_tq_cpu.cpp)
bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {
if (tq_cpu_type(tg) && tq_cpu_type(td)) return tq_cpu_fmt(tg, td, n_embd, n_ff, f, err);
+ if (gg_cpu_type(tg) && gg_cpu_type(td)) return gg_cpu_fmt(tg, td, n_embd, n_ff, f, err);
err = "this build has no ggml-cpu (STRATA_NATIVE_EXPERTS=OFF): the low-bit experts CPU lane is not in it";
return false;
}
void iq_cpu_quant(int act, const float* x, int k, void* dst) {
if (act == tq_cpu_act()) tq_cpu_quant(x, k, dst);
+ else gg_cpu_quant(act, x, k, dst);
}
void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,
int r0, int r1, float limit) {
if (tq_cpu_type(f.tg)) tq_cpu_gate_up(f, gate, up, act, nt, ff, r0, r1, limit);
+ else gg_cpu_gate_up(f, gate, up, act, nt, ff, r0, r1, limit);
}
void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {
if (tq_cpu_type(f.td)) tq_cpu_down(f, down, hq, nt, out, r0, r1);
+ else gg_cpu_down(f, down, hq, nt, out, r0, r1);
}
bool iq_cpu_own_rows(int) { return false; }
#endif
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index ce187491d..e4adb8ad8 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -1246,12 +1246,60 @@ struct Model::Impl {
CK(cudaMemcpy(dst, tmp.data(), t.bytes, cudaMemcpyHostToDevice));
return true;
}
+ // a dense matrix in a ggml block type (x3::Mat fmt kFmtGG, row = k / 256 blocks): whole, or part of its rows / inputs
+ bool mat_gg(Dev& d, const std::string& pre, const Tensor& t, int ggt, x3::Mat& m, std::string& err, size_t* acct, int part,
+ int np, bool cols) {
+ if (t.shape.size() != 2 || t.shape[1] % 256 != 0) {
+ err = pre + ": a " + t.dtype + " tensor that is not a matrix of whole blocks";
+ return false;
+ }
+ int n = (int) t.shape[0], k = (int) t.shape[1];
+ const size_t rb = gg::type_row_bytes(ggt, k), bb = rb / ((size_t) k / 256);
+ std::vector<uint8_t> w((size_t) n * rb);
+ if (t.bytes != w.size() || !pk.read(t, w.data())) {
+ err = pre + ": read failed";
+ return false;
+ }
+ if (np > 1) {
+ const int whole = cols ? n : k, wd = whole / np;
+ if (whole % np != 0 || wd % (cols ? 1 : 256) != 0) {
+ err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " + std::to_string(np) + " parts";
+ return false;
+ }
+ std::vector<uint8_t> p;
+ if (cols) {
+ p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));
+ n = wd;
+ } else {
+ const size_t pb = (size_t) (wd / 256) * bb, off = (size_t) (part * wd / 256) * bb;
+ p.resize((size_t) n * pb);
+ for (size_t r = 0; r < (size_t) n; ++r) std::memcpy(p.data() + r * pb, w.data() + r * rb + off, pb);
+ k = wd;
+ }
+ w.swap(p);
+ }
+ uint8_t* dp = nullptr;
+ if (!dalloc(d, dp, w.size(), err, acct)) return false;
+ cudaSetDevice(d.id);
+ CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));
+ m = x3::Mat{};
+ m.fmt = gg::kFmtGG;
+ m.K = ggt;
+ m.raw = dp;
+ m.k = k;
+ m.n = n;
+ m.ld = (int) gg::type_row_bytes(ggt, k);
+ return true;
+ }
// a GGUF's BF16 or Q8_0 matrix ([out][in]) onto the device, as mat() takes an EXL3 one: whole, or part `part` of its
// columns (outputs: a range of its rows) or of its rows (inputs: a range of each row, whole Q8_0 blocks) - a
// standalone [n][k] matrix
bool mat_bf16(Dev& d, const std::string& pre, const Tensor& t, x3::Mat& m, std::string& err, size_t* acct, int part,
int np, bool cols) {
const bool q8 = t.dtype == "Q8_0";
+ // a ggml K-quant dense matrix (Q4_K / Q5_K / Q6_K: 256 values a block; the GEMV path of ds4_iq)
+ const int ggt = (t.dtype == "Q4_K" || t.dtype == "Q5_K" || t.dtype == "Q6_K") ? gg_type_of(t.dtype) : -1;
+ if (ggt > 0) return mat_gg(d, pre, t, ggt, m, err, acct, part, np, cols);
if (t.shape.size() != 2 || (q8 && t.shape[1] % 32 != 0)) {
err = pre + ": a " + t.dtype + " tensor that is not a matrix";
return false;
@@ -1313,7 +1361,7 @@ struct Model::Impl {
// splits evenly, its groups only as wide as that goes)
bool mat(Dev& d, const std::string& pre, x3::Mat& m, std::string& err, size_t* acct, int part = 0, int np = 1,
bool cols = true, int rot = 0) {
- if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0"))
+ if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0" || tw->dtype == "Q4_K" || tw->dtype == "Q5_K" || tw->dtype == "Q6_K"))
return mat_bf16(d, pre, *tw, m, err, acct, part, np, cols);
const Tensor* tr = pk.get(pre + ".trellis");
const Tensor* su = pk.get(pre + ".suh");
@@ -1479,7 +1527,7 @@ struct Model::Impl {
// expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)
// (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)
static int gg_type_of(const std::string& dtype) {
- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 16, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})
+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})
if (dtype == gg::type_name(t)) return t;
return -1;
}
diff --git a/tools/dsv41/dense_requant.py b/tools/dsv41/dense_requant.py
new file mode 100644
index 000000000..e424e3ca3
--- /dev/null
+++ b/tools/dsv41/dense_requant.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""dense_requant.py - the dense matrices of a DeepSeek-V4.1 pack (attention, compressor, indexer, shared experts; BF16 in the
+unsloth GGUF) in a ggml K-quant, through ggml's own quantizer (libggml-base, ctypes), written beside the model as a small file the
+pack reads instead of the originals - the experts, the Engram and the embedding/head are not touched, nothing big is copied:
+
+ python tools/dsv41/dense_requant.py --pack PACK --out DIR --type Q5_K [--lib libggml-base.so] [--jobs 8]
+ python tools/dsv41/pack_gguf.py --model GGUF_DIR --out PACK2 --dense-override DIR
+
+DIR/dense.bin holds the quantized tensors (4096-aligned), DIR/dense.json {tensor name: [type, file, offset, bytes, shape]}.
+Types: Q4_K (4.5 bpw), Q5_K (5.5), Q6_K (6.56), Q8_0 (8.5). --include / --exclude are regexes on the checkpoint name
+(default: layers.N.attn.* and layers.N.ffn.shared_experts.*, the main layers only, matrices whose rows are whole 256 blocks).
+"""
+import argparse
+import ctypes
+import json
+import multiprocessing as mp
+import os
+import pathlib
+import re
+import sys
+import time
+
+import numpy as np
+
+GT = {"Q8_0": (8, 32, 34), "Q4_K": (12, 256, 144), "Q5_K": (13, 256, 176), "Q6_K": (14, 256, 210)}
+_lib = None
+
+
+def _init(libpath):
+ global _lib
+ try:
+ os.nice(10)
+ except OSError:
+ pass
+ _lib = ctypes.CDLL(libpath)
+ _lib.ggml_quantize_chunk.restype = ctypes.c_size_t
+ _lib.ggml_quantize_chunk.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_int64,
+ ctypes.c_int64, ctypes.c_void_p]
+
+
+def _job(j):
+ name, path, off, nbytes, rows, cols, tid, qrb, out_path, out_off = j
+ raw = np.memmap(path, dtype=np.uint16, mode="r", offset=off, shape=(rows * cols,))
+ x = (raw.astype(np.uint32) << 16).view(np.float32).reshape(rows, cols)
+ x = np.ascontiguousarray(x)
+ dst = np.empty(rows * qrb, dtype=np.uint8)
+ got = _lib.ggml_quantize_chunk(tid, x.ctypes.data, dst.ctypes.data, 0, rows, cols, None)
+ if got != rows * qrb:
+ raise RuntimeError("%s: ggml_quantize_chunk wrote %d of %d bytes" % (name, got, rows * qrb))
+ # relative error of the result against the source (dequantize by the type's own reference is not exposed here:
+ # the engine's tests compare the kernels; this tool only checks the size)
+ fd = os.open(out_path, os.O_WRONLY)
+ os.pwrite(fd, dst.tobytes(), out_off)
+ os.close(fd)
+ return name, rows * qrb
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--pack", required=True)
+ ap.add_argument("--out", required=True)
+ ap.add_argument("--type", required=True, choices=sorted(GT))
+ ap.add_argument("--lib", default="/data/dsv41/build-hip/bin/libggml-base.so")
+ ap.add_argument("--jobs", type=int, default=8)
+ ap.add_argument("--include", default=r"^layers\.\d+\.(attn\.|ffn\.shared_experts\.)")
+ ap.add_argument("--exclude", default=r"(norm|sink|gate\.bias|\.gate\.weight$|hc_|weights_proj)")
+ a = ap.parse_args()
+ tid, per, bpb = GT[a.type]
+ files, tensors, cfg = {}, [], {}
+ for line in open(os.path.join(a.pack, "tensors.txt"), encoding="utf-8"):
+ f = line.split()
+ if f and f[0] == "file":
+ files[int(f[1])] = line.split(None, 2)[2].strip()
+ elif f and f[0] == "t":
+ tensors.append((f[1], f[2], int(f[3]), int(f[4]), int(f[5]), [int(x) for x in f[6:]]))
+ for line in open(os.path.join(a.pack, "ds4.txt"), encoding="utf-8"):
+ k, _, v = line.strip().partition(" ")
+ cfg[k] = v
+ model_dir = cfg["model_dir"]
+ inc, exc = re.compile(a.include), re.compile(a.exclude)
+ sel = [t for t in tensors if t[1] == "BF16" and len(t[5]) == 2 and t[5][1] % per == 0 and inc.search(t[0]) and not exc.search(t[0])]
+ os.makedirs(a.out, exist_ok=True)
+ out_path = os.path.join(a.out, "dense.bin")
+ jobs, idx, off = [], {}, 0
+ for name, dt, fi, o, nb, shape in sel:
+ rows, cols = shape
+ qrb = cols // per * bpb
+ jobs.append((name, os.path.join(model_dir, files[fi]), o, nb, rows, cols, tid, qrb, out_path, off))
+ idx[name] = [a.type, "dense.bin", off, rows * qrb, shape]
+ off += (rows * qrb + 4095) // 4096 * 4096
+ with open(out_path, "wb") as f:
+ f.truncate(off)
+ src_b = sum(t[4] for t in sel)
+ t0 = time.time()
+ with mp.get_context("fork").Pool(a.jobs, initializer=_init, initargs=(a.lib,)) as pool:
+ for i, (name, n) in enumerate(pool.imap_unordered(_job, jobs)):
+ if i % 40 == 0:
+ print("%d/%d %s %.0f s" % (i, len(jobs), name, time.time() - t0), flush=True)
+ json.dump(idx, open(os.path.join(a.out, "dense.json"), "w"))
+ print("dense %s: %d tensors, %.2f GB BF16 -> %.2f GB, %.0f s" % (a.type, len(sel), src_b / 1e9, sum(v[3] for v in idx.values()) / 1e9, time.time() - t0))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/dsv41/make_imatrix.py b/tools/dsv41/make_imatrix.py
new file mode 100644
index 000000000..1827ee775
--- /dev/null
+++ b/tools/dsv41/make_imatrix.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+"""make_imatrix.py CAL_DIR OUT.dat : llama.cpp's legacy imatrix (.dat) for requant_gguf.py from the per-expert calibration sums the
+gate-3 run dumped (CAL_DIR/L<l>_gu.f64 [384][5120] sums of x^2 of the gate/up input, L<l>_dn.f64 [384][2304] of the down input,
+L<l>_cnt.u64 [384] picks). One entry per expert tensor: blk.<l>.ffn_{gate,up,down}_exps.weight with 384 x ne0 values (E[x^2] per
+expert and input channel; an expert with few picks is shrunk toward the layer mean with a prior of 64 picks, as gate 3 did)."""
+import struct
+import sys
+
+import numpy as np
+
+cal, out = sys.argv[1], sys.argv[2]
+PRIOR = 64.0
+ents = []
+for l in range(40):
+ gu = np.fromfile("%s/L%d_gu.f64" % (cal, l), dtype=np.float64).reshape(384, 5120)
+ dn = np.fromfile("%s/L%d_dn.f64" % (cal, l), dtype=np.float64).reshape(384, 2304)
+ cnt = np.fromfile("%s/L%d_cnt.u64" % (cal, l), dtype=np.uint64).astype(np.float64)
+ tot = max(cnt.sum(), 1.0)
+ for names, s in ((("gate", "up"), gu), (("down",), dn)):
+ mean = s.sum(0) / tot
+ h = (s + PRIOR * mean) / (cnt[:, None] + PRIOR)
+ h = np.maximum(h, 1e-8).astype(np.float32)
+ for nm in names:
+ ents.append(("blk.%d.ffn_%s_exps.weight" % (l, nm), h.reshape(-1)))
+with open(out, "wb") as f:
+ f.write(struct.pack("<i", len(ents)))
+ for name, v in ents:
+ b = name.encode()
+ f.write(struct.pack("<i", len(b)) + b + struct.pack("<ii", 1, v.size) + v.tobytes())
+print("wrote", out, len(ents), "entries")
diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.py
index 9c7e4796e..af7d0f94d 100644
--- a/tools/dsv41/pack_gguf.py
+++ b/tools/dsv41/pack_gguf.py
@@ -333,6 +333,8 @@ def main() -> int:
"checkpoint's vision.* / aligner.* (default: the first mmproj*.gguf in <model>/..)")
ap.add_argument("--tq-experts", nargs="+", help="folders tq_convert.py wrote (tq.json + tq-L<layer>.bin): the routed experts of the layers "
"listed there are read from it as trellis types (TQK5X ..), the others keep their MXFP4")
+ ap.add_argument("--dense-override", help="a folder dense_requant.py wrote (dense.json + dense.bin): its tensors replace the same-named "
+ "dense matrices of the GGUF (K-quants of the attention / shared experts)")
ap.add_argument("--tq-layers", help="only these layers (comma list) of the --tq-experts folders")
ap.add_argument("--drafter", help="the DSpark drafter's .json (drafter_pack.py's; default <model>/../dspark/dspark.json)")
ap.add_argument("--engram", help="the folder of the Engram tables' GGUF set, for a quant that leaves them out "
@@ -442,6 +444,25 @@ def main() -> int:
files = names
if eg_from:
print("Engram tables from %s (%d shards)" % (eg_dir, len(eg_from)))
+ # dense matrices requantized by dense_requant.py replace the GGUF's BF16 ones
+ n_dense = 0
+ if a.dense_override:
+ dd = pathlib.Path(a.dense_override).expanduser().resolve()
+ dj = json.loads((dd / "dense.json").read_text())
+ try:
+ relf = os.path.relpath(dd / "dense.bin", model).replace(os.sep, "/")
+ except ValueError:
+ raise SystemExit("%s must be on the model's drive" % dd)
+ if relf not in files:
+ files.append(relf)
+ dfi = files.index(relf)
+ pos = {t[0]: i for i, t in enumerate(tensors)}
+ for name, (tname, fname, off, nb, shape) in dj.items():
+ if name not in pos:
+ raise SystemExit("dense override: %s is not in the pack" % name)
+ tensors[pos[name]] = (name, tname, dfi, off, nb, shape)
+ n_dense += 1
+ print("dense override from %s: %d tensors" % (dd, n_dense))
# the DSpark drafter from unsloth's GGUF (architecture dflash): blk.N.* -> mtp.N.*, the head tensors under the
# checkpoint's names (fc -> main_proj, enc.output_norm -> main_norm, output_norm -> norm of the last block,
# conf_proj -> confidence_head.proj, markov_w1 / w2 -> markov_head.embed / head)
@@ -658,7 +679,7 @@ def main() -> int:
by = {}
for t in tensors:
by[t[1]] = by.get(t[1], 0) + t[4]
- extras = [w for w, there in (("the DSpark drafter", drafter_in or dspark_gguf), ("the vision tower", bool(vision)), ("trellis experts in layers %s" % tq_layers, bool(tq_layers))) if there]
+ extras = [w for w, there in (("the DSpark drafter", drafter_in or dspark_gguf), ("the vision tower", bool(vision)), ("trellis experts in layers %s" % tq_layers, bool(tq_layers)), ("%d dense matrices requantized" % n_dense, bool(n_dense))) if there]
print("%s: %d tensors in %d files (%s), Engram %d layers x %d hash columns%s -> %s" %
(model.name, len(tensors), len(files), ", ".join("%s %.1f GB" % (k, v / 1e9) for k, v in sorted(by.items())),
nl, cols, "".join(", " + w for w in extras), out))
diff --git a/tools/dsv41/requant_gguf.py b/tools/dsv41/requant_gguf.py
index bc28b9c13..6066a1e42 100644
--- a/tools/dsv41/requant_gguf.py
+++ b/tools/dsv41/requant_gguf.py
@@ -48,7 +48,7 @@ GT = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256,
12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176), 14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66),
17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50), 20: ("IQ4_NL", 32, 18),
21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
- 30: ("BF16", 1, 2), 39: ("MXFP4", 32, 17), 67: ("MXFP8", 32, 33)}
+ 30: ("BF16", 1, 2), 39: ("MXFP4", 32, 17), 43: ("MXFP8", 32, 33), 67: ("MXFP8", 32, 33)}
BYNAME = {v[0]: k for k, v in GT.items()}
NEED_IMATRIX = {16, 17, 19, 29}
ROLE = {"ffn_gate_exps.weight": "gate", "ffn_up_exps.weight": "up", "ffn_down_exps.weight": "down"}
@@ -152,7 +152,7 @@ def plan_type(recipe: dict, name: str, typ: int, dims: list[int]) -> int | None:
"""The output type of a tensor (None: copied as it is, -1: left out)."""
m = re.match(r"blk\.(\d+)\.(.+)$", name)
short = m.group(2) if m else name
- if short.startswith("engram_"):
+ if short.startswith("engram_") or short.startswith("per_layer_token_embd"):
return None if recipe.get("engram", "skip") == "keep" else -1
if m and short in ROLE:
role = ROLE[short]