Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
mtp: the drafter's Q8_0 draft head and projections on the Pascal GEMV, opt-in STRATA_MTP_Q8_SM60=1 (#3751)
ruibeikaaf213f78baec8
diff --git a/include/strata/kernels/native_mmvq.hpp b/include/strata/kernels/native_mmvq.hppindex ce11095cb..c809e74e5 100644--- a/include/strata/kernels/native_mmvq.hpp+++ b/include/strata/kernels/native_mmvq.hpp@@ -110,8 +110,15 @@ void native_q8_0_packed_register(const void* gguf_weights, const void* packed, ivoid native_q8_0_packed_unregister(const void* gguf_weights);/// STRATA_Q8_SM60=1 (opt-in, Pascal GP100): every Q8_0 dense matrix is packed as above whatever its shape, and the/// Q8_0 decode GEMVs run a kernel written for compute capability 6.0 (q8_sm60.cuh; not bitwise the exact kernels).-/// native_q8_0_sm60_pack packs a device matrix here (the head) and owns the copy until native_q8_0_sm60_release.+/// native_q8_0_sm60_pack packs a device matrix here (the head) and owns the copy until native_q8_0_sm60_release;+/// native_q8_0_sm60_pack_quiet is the same with no line for a success (the caller sums several up).bool native_q8_0_sm60_pack(const void* weights, int n_in, int n_out, const char* what);+bool native_q8_0_sm60_pack_quiet(const void* weights, int n_in, int n_out, const char* what);+/// The same for a matrix that nothing but native_mmvq reads (the MTP drafter's draft head subset): where every column+/// count takes the Pascal kernel and the planes stay 16-byte aligned, they replace the GGUF blocks in the matrix's own+/// allocation (the same byte count, staged through the host: no second copy); otherwise a packed copy as+/// native_q8_0_sm60_pack makes. Either way native_q8_0_sm60_release must run before the caller frees the matrix.+bool native_q8_0_sm60_pack_in_place(void* weights, int n_in, int n_out, const char* what);void native_q8_0_sm60_release(const void* weights);/// STRATA_Q6_PACKED=1 (opt-in): a packed copy of a Q6_K matrix (the output heads) - the same bytes as ql / qh /diff --git a/metal/src/engine/stubs_0145.cpp b/metal/src/engine/stubs_0145.cppindex 0c1976f60..351665ecd 100644--- a/metal/src/engine/stubs_0145.cpp+++ b/metal/src/engine/stubs_0145.cpp@@ -147,5 +147,8 @@ namespace strata::kernels { bool gemm_iq_f16_grouped(int p0, void const* p1, uns// STRATA_Q8_SM60 (Pascal GP100, opt-in): a no-op on Metal; NativeHead::load()/unload() call these for a Q8_0 head, so they must exist and answer "nothing packed".namespace strata::kernels { bool native_q8_0_sm60_pack(void const* p0, int p1, int p2, char const* p3) { return false; } }namespace strata::kernels { void native_q8_0_sm60_release(void const* p0) { } }+// the MTP drafter's STRATA_MTP_Q8_SM60 packing (mtp.cpp): nothing packed on Metal+namespace strata::kernels { bool native_q8_0_sm60_pack_quiet(void const* p0, int p1, int p2, char const* p3) { return false; } }+namespace strata::kernels { bool native_q8_0_sm60_pack_in_place(void* p0, int p1, int p2, char const* p3) { return false; } }// only reached with pf_wmma (off on Metal)namespace strata::prefill { void swiglu_split_f16(float const* p0, unsigned short* p1, long long p2, int p3, bool p4, void* p5) { } }diff --git a/src/core/mtp.cpp b/src/core/mtp.cppindex 292a5d2b4..99ad68f28 100644--- a/src/core/mtp.cpp+++ b/src/core/mtp.cpp@@ -78,6 +78,16 @@ bool mtp_catchup_all() {}();return on;}+// STRATA_MTP_Q8_SM60=1 (opt-in, with STRATA_Q8_SM60=1): the drafter's own Q8_0 matrices (the draft head's subset,+// the projections) run on the Pascal GEMV too (the drafts differ in rounding only: the Pascal kernel sums in another+// order; the replies do not depend on the drafts)+bool mtp_q8_sm60() {+ static const bool on = [] {+ const char* v = std::getenv("STRATA_MTP_Q8_SM60");+ return v != nullptr && v[0] == '1';+ }();+ return on;+}constexpr int GGML_Q8_0 = 8;constexpr int GGML_Q4_0 = 2;using Clock = std::chrono::steady_clock;@@ -154,12 +164,20 @@ MtpDrafter::~MtpDrafter() {if (side_) cudaStreamDestroy(side_);if (sh_fork_) cudaEventDestroy(sh_fork_);if (sh_join_) cudaEventDestroy(sh_join_);- if (owns_weights_ && dense_) cudaFree(dense_);+ if (owns_weights_ && dense_) {+ for (const auto& t : tensors_) // the STRATA_MTP_Q8_SM60 registrations first (a no-op for the rest)+ if (t.kind == "q8_0") strata::kernels::native_q8_0_sm60_release(dense_ + t.off);+ cudaFree(dense_);+ }if (owns_weights_ && experts_) cudaFree(experts_);if (state_arena_) cudaFree(state_arena_);if (arena_) cudaFree(arena_);if (head_logits_) cudaFree(head_logits_);- if (owns_draft_head_ && dhead_) { strata::kernels::native_q6_k_unpack(dhead_); cudaFree(dhead_); }+ if (owns_draft_head_ && dhead_) {+ strata::kernels::native_q6_k_unpack(dhead_);+ strata::kernels::native_q8_0_sm60_release(dhead_);+ cudaFree(dhead_);+ }if (owns_draft_head_ && dvocab_) cudaFree(dvocab_);if (ev_chain_) cudaEventDestroy(ev_chain_);for (cudaEvent_t e : ev_step_) if (e) cudaEventDestroy(e);@@ -335,6 +353,26 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session"self_attn.v_proj.weight", "self_attn.o_proj.weight", "mlp.shared_expert.gate_proj.weight","mlp.shared_expert.up_proj.weight", "mlp.shared_expert.down_proj.weight"};for (const char* n : required) if (!q8(n)) { err = std::string("mtp: ") + n + " is missing (q8_0)"; return false; }+ // STRATA_MTP_Q8_SM60=1: the projections on the Pascal GEMV (--batch's shared drafters read the same pointers), as+ // packed copies like the main model's: the prompt path's draft_kv reads four of them in the GGUF layout (MMQ);+ // --mtp-q4 reads its Q4_0 copies instead+ if (shared == nullptr && dense4_ == nullptr && mtp_q8_sm60()) {+ int packed = 0;+ uint64_t packed_bytes = 0;+ for (const char* n : required)+ for (const auto& t : tensors_)+ if (t.name == n && t.kind == "q8_0" &&+ strata::kernels::native_q8_0_sm60_pack_quiet(dense_ + t.off, (int) t.cols, (int) t.rows,+ ("MTP " + t.name).c_str())) {+ ++packed;+ packed_bytes += t.bytes; // a copy is the same 34 bytes per 32 values+ }+ vram_ += packed_bytes;+ if (packed > 0)+ std::fprintf(stderr, "strata mtp: %d of %zu projections on the Pascal Q8_0 GEMV (STRATA_MTP_Q8_SM60=1, %.1f "+ "MiB more VRAM)\n", packed, sizeof(required) / sizeof(required[0]),+ (double) packed_bytes / 1048576.0);+ }// ---- the layer's own K/V (dense attention: no indexer state is read)const strata::kernels::QsaShapes s = shapes_of(g);@@ -742,6 +780,12 @@ bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const floatif (dhead_type_ == 14 && strata::kernels::native_q6_k_packed_enabled()) // STRATA_Q6_PACKED=1strata::kernels::native_q6_k_pack(dhead_, (int) (row_bytes / 210) * 256, (int) n_dvocab_,"MTP draft head");+ // STRATA_MTP_Q8_SM60=1: the subset on the Pascal GEMV like the main head, but packed in place - nothing but+ // record_rest's native_mmvq reads it (the prompt path never does), and a copy would cost its size again+ if (dhead_type_ == GGML_Q8_0 && owns_draft_head_ && mtp_q8_sm60() && row_bytes == g_->n_embd / 32 * 34 &&+ strata::kernels::native_q8_0_sm60_pack_in_place(dhead_, (int) g_->n_embd, (int) n_dvocab_,+ "MTP draft head"))+ std::fprintf(stderr, "strata mtp: the draft head on the Pascal Q8_0 GEMV (STRATA_MTP_Q8_SM60=1)\n");}}if (coupled_draft_env() && cparams_ == nullptr && !setup_coupled(err)) return false;diff --git a/src/kernels/cuda/native_mmvq.cu b/src/kernels/cuda/native_mmvq.cuindex 05f9b42d4..afaec6cb5 100644--- a/src/kernels/cuda/native_mmvq.cu+++ b/src/kernels/cuda/native_mmvq.cu@@ -2243,15 +2243,25 @@ bool native_q8_0_packed_eligible(int n_in, int n_out) {}namespace {-std::unordered_map<const void*, void*>& q8_sm60_owned() { // packed copies made here (the head), freed by release+// packed copies made here (the head), freed by release; null: packed in place (release only unregisters it)+std::unordered_map<const void*, void*>& q8_sm60_owned() {static std::unordered_map<const void*, void*> owned;return owned;}+bool q8_sm60_pack_impl(const void* weights, int n_in, int n_out, const char* what, bool quiet);} // namespacebool native_q8_0_sm60_pack(const void* weights, int n_in, int n_out, const char* what) {+ return q8_sm60_pack_impl(weights, n_in, n_out, what, false);+}+bool native_q8_0_sm60_pack_quiet(const void* weights, int n_in, int n_out, const char* what) {+ return q8_sm60_pack_impl(weights, n_in, n_out, what, true);+}++namespace {+bool q8_sm60_pack_impl(const void* weights, int n_in, int n_out, const char* what, bool quiet) {#if defined(__HIPCC__)- (void) weights; (void) n_in; (void) n_out; (void) what;+ (void) weights; (void) n_in; (void) n_out; (void) what; (void) quiet;return false;#elseif (!q8_sm60_on() || !weights || !native_q8_0_packed_eligible(n_in, n_out)) return false;@@ -2276,18 +2286,20 @@ bool native_q8_0_sm60_pack(const void* weights, int n_in, int n_out, const char*}q8_packed_registry()[weights] = Q8Packed{qs, d, n_in, n_out};q8_sm60_owned()[weights] = packed;- std::fprintf(stderr, "strata q8 sm60: packed %s (%d x %d, %.0f MiB more VRAM)\n", what, n_out, n_in,- double(bytes) / 1048576.0);+ if (!quiet)+ std::fprintf(stderr, "strata q8 sm60: packed %s (%d x %d, %.0f MiB more VRAM)\n", what, n_out, n_in,+ double(bytes) / 1048576.0);return true;#endif}+} // namespacevoid native_q8_0_sm60_release(const void* weights) {auto& owned = q8_sm60_owned();const auto it = owned.find(weights);if (it == owned.end()) return;q8_packed_registry().erase(weights);- cudaFree(it->second);+ if (it->second) cudaFree(it->second);owned.erase(it);}@@ -2313,6 +2325,40 @@ void native_q8_0_packed_register(const void* gguf_weights, const void* packed, ivoid native_q8_0_packed_unregister(const void* gguf_weights) { q8_packed_registry().erase(gguf_weights); }+bool native_q8_0_sm60_pack_in_place(void* weights, int n_in, int n_out, const char* what) {+#if defined(__HIPCC__)+ (void) weights; (void) n_in; (void) n_out; (void) what;+ return false;+#else+ if (!q8_sm60_on() || !weights || !native_q8_0_packed_eligible(n_in, n_out)) return false;+ if (q8_packed_registry().count(weights)) return true;+ // The GGUF blocks are gone afterwards, so only where q8_packed_mmvq can never fall back to the GGUF-layout kernels+ // (every column count fits the kernel's shared-memory planes) and its 16-byte weight loads stay aligned+ if (q8sm60::smem_bytes(MAX_NCOLS, n_in) > q8sm60::MAX_SMEM || reinterpret_cast<std::uintptr_t>(weights) % 16 != 0)+ return native_q8_0_sm60_pack(weights, n_in, n_out, what);+ const std::size_t bytes = std::size_t(n_in / 32) * std::size_t(n_out) * 34; // both layouts: 34 bytes per 32+ std::vector<uint8_t> blocks(bytes), planes(bytes);+ if (cudaMemcpy(blocks.data(), weights, bytes, cudaMemcpyDeviceToHost) != cudaSuccess) {+ cudaGetLastError();+ std::fprintf(stderr, "strata q8 sm60: reading the %s failed; it keeps the GGUF layout\n", what);+ return false;+ }+ native_q8_0_pack_host(blocks.data(), planes.data(), n_in, n_out);+ if (cudaMemcpy(weights, planes.data(), bytes, cudaMemcpyHostToDevice) != cudaSuccess) {+ cudaGetLastError();+ // a partial write leaves neither layout: put the blocks back+ const bool restored = cudaMemcpy(weights, blocks.data(), bytes, cudaMemcpyHostToDevice) == cudaSuccess;+ cudaGetLastError();+ std::fprintf(stderr, "strata q8 sm60: writing the packed %s failed; %s\n", what,+ restored ? "it keeps the GGUF layout" : "its GGUF blocks could not be restored either");+ return false;+ }+ native_q8_0_packed_register(weights, weights, n_in, n_out);+ q8_sm60_owned()[weights] = nullptr;+ return true; // the caller says what it packed+#endif+}+bool native_q6_k_packed_enabled() {static const bool enabled = [] {const char* v = std::getenv("STRATA_Q6_PACKED");