Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: dense K-quant GEMV (Q5_K/Q6_K) + dense_requant, ggml-type CPU lane, imatrix builder, requant fixes (C0 experiments)
Niko12215278ba00431e
diff --git a/CMakeLists.txt b/CMakeLists.txtindex b4082fdd6..a054a73ee 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -777,7 +777,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_link_libraries(strata_dsv41_kernels PUBLIC ${_strata_gpu_runtime_target})add_executable(strata-dsv41 src/dsv41/ds4_main.cpp src/dsv41/ds4_model.cu src/dsv41/ds4_kernels.cu src/dsv41/ds4_vision.cusrc/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp- src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp)+ src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp)target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})diff --git a/src/dsv41/ds4_gg_cpu.cpp b/src/dsv41/ds4_gg_cpu.cppnew file mode 100644index 000000000..3a9a7728d--- /dev/null+++ b/src/dsv41/ds4_gg_cpu.cpp@@ -0,0 +1,118 @@+// Strata. The CPU lane of the DeepSeek-V4.1 engine path for ggml's block types (IQ2_XXS .. Q4_K, the experts of a requantized model):+// ggml-cpu's own row dots (type traits: vec_dot over the Q8_K / Q8_0 activation of vec_dot_type), one row at a time. Maya's lane for+// these types (native_expert split-base rows) is not in Strata's tree; this one is plain and complete, not tuned.+#include "ds4_gguf_cpu.hpp"++#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <string>++#if defined(STRATA_DSV41_TQ_CPU)+#include "ggml-cpu.h"+#include "ggml.h"++#include <mutex>+#include <vector>++namespace strata::ds4::gguf {+namespace {+void init_once() {+ static std::once_flag f;+ std::call_once(f, [] { ggml_cpu_init(); });+}+size_t row_size(int type, int64_t k) { return ggml_row_size((ggml_type) type, k); }+} // namespace++bool gg_cpu_type(int type) {+ switch (type) {+ case 10: case 11: case 12: case 13: case 14: case 16: case 17: case 18: case 21: case 22: case 23: return true;+ default: return false;+ }+}++static bool dbg() { static const bool on = std::getenv("DS4_DBG_GG") != nullptr; return on; }+bool gg_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {+ if (dbg()) std::fprintf(stderr, "gg_cpu_fmt %d %d\n", tg, td);+ init_once();+ if (!gg_cpu_type(tg) || !gg_cpu_type(td)) {+ err = "not ggml block types";+ return false;+ }+ const ggml_type_traits_cpu* a = ggml_get_type_traits_cpu((ggml_type) tg);+ const ggml_type_traits_cpu* b = ggml_get_type_traits_cpu((ggml_type) td);+ if (a == nullptr || b == nullptr || a->vec_dot == nullptr || b->vec_dot == nullptr) {+ err = "ggml-cpu has no row dot for these types";+ return false;+ }+ f.tg = tg;+ f.td = td;+ f.n_embd = n_embd;+ f.n_ff = n_ff;+ f.gu_act = (int) a->vec_dot_type;+ f.d_act = (int) b->vec_dot_type;+ f.gu_row = row_size(tg, n_embd);+ f.d_row = row_size(td, n_ff);+ f.act_bytes = row_size(f.gu_act, n_embd);+ f.h_bytes = row_size(f.d_act, n_ff);+ return true;+}++// x (k values) -> the activation type (Q8_K, Q8_0 ..) ggml-cpu's dot takes+void gg_cpu_quant(int act, const float* x, int k, void* dst) {+ init_once();+ if (dbg()) std::fprintf(stderr, "gg_cpu_quant act %d k %d\n", act, k);+ const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) act);+ if (tc != nullptr && tc->from_float != nullptr) {+ tc->from_float(x, dst, k);+ return;+ }+ const ggml_type_traits* t = ggml_get_type_traits((ggml_type) act);+ if (dbg()) std::fprintf(stderr, "gg_cpu_quant: no cpu from_float, ref %p\n", (void*) (t ? t->from_float_ref : nullptr));+ t->from_float_ref(x, dst, k);+}++static void dots(int type, const uint8_t* w, size_t rb, int n, const void* const* act, int nt, float* const* out, int r0, int r1) {+ const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) type);+ if (dbg()) std::fprintf(stderr, "gg dots type %d n %d rows %d..%d nt %d\n", type, n, r0, r1, nt);+ for (int r = r0; r < r1; ++r)+ for (int t = 0; t < nt; ++t) tc->vec_dot(n, &out[t][r], 0, w + (size_t) r * rb, 0, act[t], 0, 1);+}++void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff, int r0,+ int r1, float limit) {+ static thread_local std::vector<float> tmp[8];+ float* uo[8];+ for (int t = 0; t < nt; ++t) {+ tmp[t].resize((size_t) r1);+ uo[t] = tmp[t].data();+ }+ dots(f.tg, gate, f.gu_row, f.n_embd, act, nt, ff, r0, r1);+ dots(f.tg, up, f.gu_row, f.n_embd, act, nt, uo, r0, r1);+ for (int t = 0; t < nt; ++t)+ for (int r = r0; r < r1; ++r) {+ float g = ff[t][r], u = uo[t][r];+ if (limit > 0.0f) {+ g = g < limit ? g : limit;+ u = u < -limit ? -limit : (u > limit ? limit : u);+ }+ ff[t][r] = g / (1.0f + std::exp(-g)) * u;+ }+}++void gg_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {+ dots(f.td, down, f.d_row, f.n_ff, hq, nt, out, r0, r1);+}+} // namespace strata::ds4::gguf+#else+namespace strata::ds4::gguf {+bool gg_cpu_type(int) { return false; }+bool gg_cpu_fmt(int, int, int, int, IqCpu&, std::string& err) {+ err = "this build has no ggml-cpu lane (STRATA_NATIVE_EXPERTS=OFF)";+ return false;+}+void gg_cpu_quant(int, const float*, int, void*) {}+void gg_cpu_gate_up(const IqCpu&, const uint8_t*, const uint8_t*, const void* const*, int, float* const*, int, int, float) {}+void gg_cpu_down(const IqCpu&, const uint8_t*, const void* const*, int, float* const*, int, int) {}+} // namespace strata::ds4::gguf+#endifdiff --git a/src/dsv41/ds4_gguf_cpu.hpp b/src/dsv41/ds4_gguf_cpu.hppindex 286afd2ea..cc9f0d8e8 100644--- a/src/dsv41/ds4_gguf_cpu.hpp+++ b/src/dsv41/ds4_gguf_cpu.hpp@@ -46,6 +46,14 @@ void tq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, consint r0, int r1, float limit);void tq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);+/// ggml block types on the CPU lane through ggml-cpu's row dots (ds4_gg_cpu.cpp), for the IQ / K-quant experts.+bool gg_cpu_type(int type);+bool gg_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err);+void gg_cpu_quant(int act, const float* x, int k, void* dst);+void gg_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,+ int r0, int r1, float limit);+void gg_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1);+/// The CPU lane's MXFP4 rows in the device's arithmetic (ds4_gguf.cu's mx_dot): the activations rounded to int8 per 32/// values as the device rounds them (q8_round's q and scale), each block's products summed exactly in integers, then/// times (weight scale x activation scale) into F32 sums - the same numbers as the device up to the order of thediff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex ba62fccce..e99c1dcb5 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -35,6 +35,9 @@ template <> struct Ty<kIQ4_XS> { static constexpr int B = 136, TAB = 4, MIN = 0;template <> struct Ty<kQ4_K> { static constexpr int B = 144, TAB = 0, MIN = 1; };template <> struct Ty<kQ2_K> { static constexpr int B = 84, TAB = 0, MIN = 1; };template <> struct Ty<kQ3_K> { static constexpr int B = 110, TAB = 0, MIN = 0; };+// the dense matrices' 5- and 6-bit K-quants (GEMV only: ds4_iq's mv; not an expert type)+template <> struct Ty<kQ5_K> { static constexpr int B = 176, TAB = 0, MIN = 1; };+template <> struct Ty<kQ6_K> { static constexpr int B = 210, TAB = 0, MIN = 0; };// MXFP4 (ds4_gguf.cu's own kernels everywhere else): a block of 32 values, an E8M0 scale then 16 code bytes; the light// sets take it tooconstexpr int kMXFP4 = 39;@@ -207,6 +210,32 @@ __device__ __forceinline__ void decode(const uint8_t* __restrict__ row, int kb,s.s0 = s.s1 = sc;s.d = h2f(b);s.m0 = s.m1 = h2f(b + 2) * (float) m;+ } else if constexpr (T == kQ5_K) { // d, dmin, scales[12], qh[32], qs[128]: weight = d sc (nibble + 16 hbit) - dmin m+ int sc, m;+ q4k_scale(b + 4, ib, sc, m);+ const uint8_t* qs = b + 48 + 32 * (ib >> 1);+ const uint8_t* qh = b + 16;+ const int sh = 4 * (ib & 1);+#pragma unroll+ for (int t = 0; t < 8; ++t)+ s.q[t] = (int) (((ld4(qs + 4 * t) >> sh) & 0x0f0f0f0fu) | (((ld4(qh + 4 * t) >> ib) & 0x01010101u) << 4));+ s.s0 = s.s1 = sc;+ s.d = h2f(b);+ s.m0 = s.m1 = h2f(b + 2) * (float) m;+ } else if constexpr (T == kQ6_K) { // ql[128], qh[64], scales int8[16], d: weight = d sc (q - 32), q 6 bits+ const int h = ib >> 2, quarter = ib & 3;+ const uint8_t* ql = b + 64 * h + (quarter & 1 ? 32 : 0);+ const uint8_t* qh = b + 128 + 32 * h;+ const int sh = 4 * (quarter >> 1);+#pragma unroll+ for (int t = 0; t < 8; ++t) {+ const uint32_t x = ((ld2(ql + 4 * t) >> sh) & 0x0f0f0f0fu) | (((ld2(qh + 4 * t) >> (2 * quarter)) & 0x03030303u) << 4);+ s.q[t] = (int) __vsub4(x, 0x20202020u);+ }+ const int8_t* scl = (const int8_t*) (b + 192) + 8 * h + 2 * quarter;+ s.s0 = scl[0];+ s.s1 = scl[1];+ s.d = h2f(b + 208);} else if constexpr (T == kQ2_K) {const int n = ib >> 2, j = ib & 3;const uint8_t* qs = b + 16 + 32 * n;@@ -921,6 +950,8 @@ int block_bytes(int t) {case kIQ3_S: return Ty<kIQ3_S>::B;case kIQ4_XS: return Ty<kIQ4_XS>::B;case kQ4_K: return Ty<kQ4_K>::B;+ case kQ5_K: return Ty<kQ5_K>::B;+ case kQ6_K: return Ty<kQ6_K>::B;case kQ2_K: return Ty<kQ2_K>::B;case kQ3_K: return Ty<kQ3_K>::B;default: return 0;@@ -941,6 +972,8 @@ const char* name(int t) {case kIQ3_S: return "IQ3_S";case kIQ4_XS: return "IQ4_XS";case kQ4_K: return "Q4_K";+ case kQ5_K: return "Q5_K";+ case kQ6_K: return "Q6_K";case kQ2_K: return "Q2_K";case kQ3_K: return "Q3_K";default: return "?";@@ -1037,7 +1070,11 @@ bool mv(const MvJob* jobs, int n, cudaStream_t s) {done[k] = true;}#define MV_(T) mv_nc<T>(J, nj, max_n, max_c, s)- DS4_IQ_SWITCH(t, MV_)+ switch (t) { // the experts' types and the dense matrices' 5- and 6-bit ones+ case kQ5_K: MV_(kQ5_K); break;+ case kQ6_K: MV_(kQ6_K); break;+ default: DS4_IQ_SWITCH(t, MV_)+ }#undef MV_}return true;diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex f13199fdc..3f0c8f52b 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -19,6 +19,7 @@namespace strata::ds4::gguf::iq {// ggml type ids+constexpr int kQ5_K = 13, kQ6_K = 14; // (the dense matrices' K-quants, GEMV only)constexpr int kQ2_K = 10, kQ3_K = 11, kQ4_K = 12, kIQ2_XXS = 16, kIQ3_XXS = 18, kIQ3_S = 21, kIQ2_S = 22,kIQ4_XS = 23;diff --git a/src/dsv41/ds4_iq_cpu.cpp b/src/dsv41/ds4_iq_cpu.cppindex e0f04678a..d472dacfc 100644--- a/src/dsv41/ds4_iq_cpu.cpp+++ b/src/dsv41/ds4_iq_cpu.cpp@@ -759,18 +759,22 @@ bool iq_cpu_own_rows(int type) { return own(type); }// (no ggml-cpu lane for the low-bit types here; the trellis types have their own: ds4_tq_cpu.cpp)bool iq_cpu_fmt(int tg, int td, int n_embd, int n_ff, IqCpu& f, std::string& err) {if (tq_cpu_type(tg) && tq_cpu_type(td)) return tq_cpu_fmt(tg, td, n_embd, n_ff, f, err);+ if (gg_cpu_type(tg) && gg_cpu_type(td)) return gg_cpu_fmt(tg, td, n_embd, n_ff, f, err);err = "this build has no ggml-cpu (STRATA_NATIVE_EXPERTS=OFF): the low-bit experts CPU lane is not in it";return false;}void iq_cpu_quant(int act, const float* x, int k, void* dst) {if (act == tq_cpu_act()) tq_cpu_quant(x, k, dst);+ else gg_cpu_quant(act, x, k, dst);}void iq_cpu_gate_up(const IqCpu& f, const uint8_t* gate, const uint8_t* up, const void* const* act, int nt, float* const* ff,int r0, int r1, float limit) {if (tq_cpu_type(f.tg)) tq_cpu_gate_up(f, gate, up, act, nt, ff, r0, r1, limit);+ else gg_cpu_gate_up(f, gate, up, act, nt, ff, r0, r1, limit);}void iq_cpu_down(const IqCpu& f, const uint8_t* down, const void* const* hq, int nt, float* const* out, int r0, int r1) {if (tq_cpu_type(f.td)) tq_cpu_down(f, down, hq, nt, out, r0, r1);+ else gg_cpu_down(f, down, hq, nt, out, r0, r1);}bool iq_cpu_own_rows(int) { return false; }#endifdiff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex ce187491d..e4adb8ad8 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -1246,12 +1246,60 @@ struct Model::Impl {CK(cudaMemcpy(dst, tmp.data(), t.bytes, cudaMemcpyHostToDevice));return true;}+ // a dense matrix in a ggml block type (x3::Mat fmt kFmtGG, row = k / 256 blocks): whole, or part of its rows / inputs+ bool mat_gg(Dev& d, const std::string& pre, const Tensor& t, int ggt, x3::Mat& m, std::string& err, size_t* acct, int part,+ int np, bool cols) {+ if (t.shape.size() != 2 || t.shape[1] % 256 != 0) {+ err = pre + ": a " + t.dtype + " tensor that is not a matrix of whole blocks";+ return false;+ }+ int n = (int) t.shape[0], k = (int) t.shape[1];+ const size_t rb = gg::type_row_bytes(ggt, k), bb = rb / ((size_t) k / 256);+ std::vector<uint8_t> w((size_t) n * rb);+ if (t.bytes != w.size() || !pk.read(t, w.data())) {+ err = pre + ": read failed";+ return false;+ }+ if (np > 1) {+ const int whole = cols ? n : k, wd = whole / np;+ if (whole % np != 0 || wd % (cols ? 1 : 256) != 0) {+ err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " + std::to_string(np) + " parts";+ return false;+ }+ std::vector<uint8_t> p;+ if (cols) {+ p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));+ n = wd;+ } else {+ const size_t pb = (size_t) (wd / 256) * bb, off = (size_t) (part * wd / 256) * bb;+ p.resize((size_t) n * pb);+ for (size_t r = 0; r < (size_t) n; ++r) std::memcpy(p.data() + r * pb, w.data() + r * rb + off, pb);+ k = wd;+ }+ w.swap(p);+ }+ uint8_t* dp = nullptr;+ if (!dalloc(d, dp, w.size(), err, acct)) return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));+ m = x3::Mat{};+ m.fmt = gg::kFmtGG;+ m.K = ggt;+ m.raw = dp;+ m.k = k;+ m.n = n;+ m.ld = (int) gg::type_row_bytes(ggt, k);+ return true;+ }// a GGUF's BF16 or Q8_0 matrix ([out][in]) onto the device, as mat() takes an EXL3 one: whole, or part `part` of its// columns (outputs: a range of its rows) or of its rows (inputs: a range of each row, whole Q8_0 blocks) - a// standalone [n][k] matrixbool mat_bf16(Dev& d, const std::string& pre, const Tensor& t, x3::Mat& m, std::string& err, size_t* acct, int part,int np, bool cols) {const bool q8 = t.dtype == "Q8_0";+ // a ggml K-quant dense matrix (Q4_K / Q5_K / Q6_K: 256 values a block; the GEMV path of ds4_iq)+ const int ggt = (t.dtype == "Q4_K" || t.dtype == "Q5_K" || t.dtype == "Q6_K") ? gg_type_of(t.dtype) : -1;+ if (ggt > 0) return mat_gg(d, pre, t, ggt, m, err, acct, part, np, cols);if (t.shape.size() != 2 || (q8 && t.shape[1] % 32 != 0)) {err = pre + ": a " + t.dtype + " tensor that is not a matrix";return false;@@ -1313,7 +1361,7 @@ struct Model::Impl {// splits evenly, its groups only as wide as that goes)bool mat(Dev& d, const std::string& pre, x3::Mat& m, std::string& err, size_t* acct, int part = 0, int np = 1,bool cols = true, int rot = 0) {- if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0"))+ if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0" || tw->dtype == "Q4_K" || tw->dtype == "Q5_K" || tw->dtype == "Q6_K"))return mat_bf16(d, pre, *tw, m, err, acct, part, np, cols);const Tensor* tr = pk.get(pre + ".trellis");const Tensor* su = pk.get(pre + ".suh");@@ -1479,7 +1527,7 @@ struct Model::Impl {// expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)// (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)static int gg_type_of(const std::string& dtype) {- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 16, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 18, 21, 22, 23, 145, 146, 224, 225, 226, 227})if (dtype == gg::type_name(t)) return t;return -1;}diff --git a/tools/dsv41/dense_requant.py b/tools/dsv41/dense_requant.pynew file mode 100644index 000000000..e424e3ca3--- /dev/null+++ b/tools/dsv41/dense_requant.py@@ -0,0 +1,104 @@+#!/usr/bin/env python3+"""dense_requant.py - the dense matrices of a DeepSeek-V4.1 pack (attention, compressor, indexer, shared experts; BF16 in the+unsloth GGUF) in a ggml K-quant, through ggml's own quantizer (libggml-base, ctypes), written beside the model as a small file the+pack reads instead of the originals - the experts, the Engram and the embedding/head are not touched, nothing big is copied:++ python tools/dsv41/dense_requant.py --pack PACK --out DIR --type Q5_K [--lib libggml-base.so] [--jobs 8]+ python tools/dsv41/pack_gguf.py --model GGUF_DIR --out PACK2 --dense-override DIR++DIR/dense.bin holds the quantized tensors (4096-aligned), DIR/dense.json {tensor name: [type, file, offset, bytes, shape]}.+Types: Q4_K (4.5 bpw), Q5_K (5.5), Q6_K (6.56), Q8_0 (8.5). --include / --exclude are regexes on the checkpoint name+(default: layers.N.attn.* and layers.N.ffn.shared_experts.*, the main layers only, matrices whose rows are whole 256 blocks).+"""+import argparse+import ctypes+import json+import multiprocessing as mp+import os+import pathlib+import re+import sys+import time++import numpy as np++GT = {"Q8_0": (8, 32, 34), "Q4_K": (12, 256, 144), "Q5_K": (13, 256, 176), "Q6_K": (14, 256, 210)}+_lib = None+++def _init(libpath):+ global _lib+ try:+ os.nice(10)+ except OSError:+ pass+ _lib = ctypes.CDLL(libpath)+ _lib.ggml_quantize_chunk.restype = ctypes.c_size_t+ _lib.ggml_quantize_chunk.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_int64,+ ctypes.c_int64, ctypes.c_void_p]+++def _job(j):+ name, path, off, nbytes, rows, cols, tid, qrb, out_path, out_off = j+ raw = np.memmap(path, dtype=np.uint16, mode="r", offset=off, shape=(rows * cols,))+ x = (raw.astype(np.uint32) << 16).view(np.float32).reshape(rows, cols)+ x = np.ascontiguousarray(x)+ dst = np.empty(rows * qrb, dtype=np.uint8)+ got = _lib.ggml_quantize_chunk(tid, x.ctypes.data, dst.ctypes.data, 0, rows, cols, None)+ if got != rows * qrb:+ raise RuntimeError("%s: ggml_quantize_chunk wrote %d of %d bytes" % (name, got, rows * qrb))+ # relative error of the result against the source (dequantize by the type's own reference is not exposed here:+ # the engine's tests compare the kernels; this tool only checks the size)+ fd = os.open(out_path, os.O_WRONLY)+ os.pwrite(fd, dst.tobytes(), out_off)+ os.close(fd)+ return name, rows * qrb+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--pack", required=True)+ ap.add_argument("--out", required=True)+ ap.add_argument("--type", required=True, choices=sorted(GT))+ ap.add_argument("--lib", default="/data/dsv41/build-hip/bin/libggml-base.so")+ ap.add_argument("--jobs", type=int, default=8)+ ap.add_argument("--include", default=r"^layers\.\d+\.(attn\.|ffn\.shared_experts\.)")+ ap.add_argument("--exclude", default=r"(norm|sink|gate\.bias|\.gate\.weight$|hc_|weights_proj)")+ a = ap.parse_args()+ tid, per, bpb = GT[a.type]+ files, tensors, cfg = {}, [], {}+ for line in open(os.path.join(a.pack, "tensors.txt"), encoding="utf-8"):+ f = line.split()+ if f and f[0] == "file":+ files[int(f[1])] = line.split(None, 2)[2].strip()+ elif f and f[0] == "t":+ tensors.append((f[1], f[2], int(f[3]), int(f[4]), int(f[5]), [int(x) for x in f[6:]]))+ for line in open(os.path.join(a.pack, "ds4.txt"), encoding="utf-8"):+ k, _, v = line.strip().partition(" ")+ cfg[k] = v+ model_dir = cfg["model_dir"]+ inc, exc = re.compile(a.include), re.compile(a.exclude)+ sel = [t for t in tensors if t[1] == "BF16" and len(t[5]) == 2 and t[5][1] % per == 0 and inc.search(t[0]) and not exc.search(t[0])]+ os.makedirs(a.out, exist_ok=True)+ out_path = os.path.join(a.out, "dense.bin")+ jobs, idx, off = [], {}, 0+ for name, dt, fi, o, nb, shape in sel:+ rows, cols = shape+ qrb = cols // per * bpb+ jobs.append((name, os.path.join(model_dir, files[fi]), o, nb, rows, cols, tid, qrb, out_path, off))+ idx[name] = [a.type, "dense.bin", off, rows * qrb, shape]+ off += (rows * qrb + 4095) // 4096 * 4096+ with open(out_path, "wb") as f:+ f.truncate(off)+ src_b = sum(t[4] for t in sel)+ t0 = time.time()+ with mp.get_context("fork").Pool(a.jobs, initializer=_init, initargs=(a.lib,)) as pool:+ for i, (name, n) in enumerate(pool.imap_unordered(_job, jobs)):+ if i % 40 == 0:+ print("%d/%d %s %.0f s" % (i, len(jobs), name, time.time() - t0), flush=True)+ json.dump(idx, open(os.path.join(a.out, "dense.json"), "w"))+ print("dense %s: %d tensors, %.2f GB BF16 -> %.2f GB, %.0f s" % (a.type, len(sel), src_b / 1e9, sum(v[3] for v in idx.values()) / 1e9, time.time() - t0))+++if __name__ == "__main__":+ main()diff --git a/tools/dsv41/make_imatrix.py b/tools/dsv41/make_imatrix.pynew file mode 100644index 000000000..1827ee775--- /dev/null+++ b/tools/dsv41/make_imatrix.py@@ -0,0 +1,30 @@+#!/usr/bin/env python3+"""make_imatrix.py CAL_DIR OUT.dat : llama.cpp's legacy imatrix (.dat) for requant_gguf.py from the per-expert calibration sums the+gate-3 run dumped (CAL_DIR/L<l>_gu.f64 [384][5120] sums of x^2 of the gate/up input, L<l>_dn.f64 [384][2304] of the down input,+L<l>_cnt.u64 [384] picks). One entry per expert tensor: blk.<l>.ffn_{gate,up,down}_exps.weight with 384 x ne0 values (E[x^2] per+expert and input channel; an expert with few picks is shrunk toward the layer mean with a prior of 64 picks, as gate 3 did)."""+import struct+import sys++import numpy as np++cal, out = sys.argv[1], sys.argv[2]+PRIOR = 64.0+ents = []+for l in range(40):+ gu = np.fromfile("%s/L%d_gu.f64" % (cal, l), dtype=np.float64).reshape(384, 5120)+ dn = np.fromfile("%s/L%d_dn.f64" % (cal, l), dtype=np.float64).reshape(384, 2304)+ cnt = np.fromfile("%s/L%d_cnt.u64" % (cal, l), dtype=np.uint64).astype(np.float64)+ tot = max(cnt.sum(), 1.0)+ for names, s in ((("gate", "up"), gu), (("down",), dn)):+ mean = s.sum(0) / tot+ h = (s + PRIOR * mean) / (cnt[:, None] + PRIOR)+ h = np.maximum(h, 1e-8).astype(np.float32)+ for nm in names:+ ents.append(("blk.%d.ffn_%s_exps.weight" % (l, nm), h.reshape(-1)))+with open(out, "wb") as f:+ f.write(struct.pack("<i", len(ents)))+ for name, v in ents:+ b = name.encode()+ f.write(struct.pack("<i", len(b)) + b + struct.pack("<ii", 1, v.size) + v.tobytes())+print("wrote", out, len(ents), "entries")diff --git a/tools/dsv41/pack_gguf.py b/tools/dsv41/pack_gguf.pyindex 9c7e4796e..af7d0f94d 100644--- a/tools/dsv41/pack_gguf.py+++ b/tools/dsv41/pack_gguf.py@@ -333,6 +333,8 @@ def main() -> int:"checkpoint's vision.* / aligner.* (default: the first mmproj*.gguf in <model>/..)")ap.add_argument("--tq-experts", nargs="+", help="folders tq_convert.py wrote (tq.json + tq-L<layer>.bin): the routed experts of the layers ""listed there are read from it as trellis types (TQK5X ..), the others keep their MXFP4")+ ap.add_argument("--dense-override", help="a folder dense_requant.py wrote (dense.json + dense.bin): its tensors replace the same-named "+ "dense matrices of the GGUF (K-quants of the attention / shared experts)")ap.add_argument("--tq-layers", help="only these layers (comma list) of the --tq-experts folders")ap.add_argument("--drafter", help="the DSpark drafter's .json (drafter_pack.py's; default <model>/../dspark/dspark.json)")ap.add_argument("--engram", help="the folder of the Engram tables' GGUF set, for a quant that leaves them out "@@ -442,6 +444,25 @@ def main() -> int:files = namesif eg_from:print("Engram tables from %s (%d shards)" % (eg_dir, len(eg_from)))+ # dense matrices requantized by dense_requant.py replace the GGUF's BF16 ones+ n_dense = 0+ if a.dense_override:+ dd = pathlib.Path(a.dense_override).expanduser().resolve()+ dj = json.loads((dd / "dense.json").read_text())+ try:+ relf = os.path.relpath(dd / "dense.bin", model).replace(os.sep, "/")+ except ValueError:+ raise SystemExit("%s must be on the model's drive" % dd)+ if relf not in files:+ files.append(relf)+ dfi = files.index(relf)+ pos = {t[0]: i for i, t in enumerate(tensors)}+ for name, (tname, fname, off, nb, shape) in dj.items():+ if name not in pos:+ raise SystemExit("dense override: %s is not in the pack" % name)+ tensors[pos[name]] = (name, tname, dfi, off, nb, shape)+ n_dense += 1+ print("dense override from %s: %d tensors" % (dd, n_dense))# the DSpark drafter from unsloth's GGUF (architecture dflash): blk.N.* -> mtp.N.*, the head tensors under the# checkpoint's names (fc -> main_proj, enc.output_norm -> main_norm, output_norm -> norm of the last block,# conf_proj -> confidence_head.proj, markov_w1 / w2 -> markov_head.embed / head)@@ -658,7 +679,7 @@ def main() -> int:by = {}for t in tensors:by[t[1]] = by.get(t[1], 0) + t[4]- extras = [w for w, there in (("the DSpark drafter", drafter_in or dspark_gguf), ("the vision tower", bool(vision)), ("trellis experts in layers %s" % tq_layers, bool(tq_layers))) if there]+ extras = [w for w, there in (("the DSpark drafter", drafter_in or dspark_gguf), ("the vision tower", bool(vision)), ("trellis experts in layers %s" % tq_layers, bool(tq_layers)), ("%d dense matrices requantized" % n_dense, bool(n_dense))) if there]print("%s: %d tensors in %d files (%s), Engram %d layers x %d hash columns%s -> %s" %(model.name, len(tensors), len(files), ", ".join("%s %.1f GB" % (k, v / 1e9) for k, v in sorted(by.items())),nl, cols, "".join(", " + w for w in extras), out))diff --git a/tools/dsv41/requant_gguf.py b/tools/dsv41/requant_gguf.pyindex bc28b9c13..6066a1e42 100644--- a/tools/dsv41/requant_gguf.py+++ b/tools/dsv41/requant_gguf.py@@ -48,7 +48,7 @@ GT = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256,12: ("Q4_K", 256, 144), 13: ("Q5_K", 256, 176), 14: ("Q6_K", 256, 210), 16: ("IQ2_XXS", 256, 66),17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50), 20: ("IQ4_NL", 32, 18),21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),- 30: ("BF16", 1, 2), 39: ("MXFP4", 32, 17), 67: ("MXFP8", 32, 33)}+ 30: ("BF16", 1, 2), 39: ("MXFP4", 32, 17), 43: ("MXFP8", 32, 33), 67: ("MXFP8", 32, 33)}BYNAME = {v[0]: k for k, v in GT.items()}NEED_IMATRIX = {16, 17, 19, 29}ROLE = {"ffn_gate_exps.weight": "gate", "ffn_up_exps.weight": "up", "ffn_down_exps.weight": "down"}@@ -152,7 +152,7 @@ def plan_type(recipe: dict, name: str, typ: int, dims: list[int]) -> int | None:"""The output type of a tensor (None: copied as it is, -1: left out)."""m = re.match(r"blk\.(\d+)\.(.+)$", name)short = m.group(2) if m else name- if short.startswith("engram_"):+ if short.startswith("engram_") or short.startswith("per_layer_token_embd"):return None if recipe.get("engram", "skip") == "keep" else -1if m and short in ROLE:role = ROLE[short]