Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

mtp: the drafter's Q8_0 draft head and projections on the Pascal GEMV, opt-in STRATA_MTP_Q8_SM60=1 (#3751)

ruibeikaaf213f78baec8
diff --git a/include/strata/kernels/native_mmvq.hpp b/include/strata/kernels/native_mmvq.hpp
index ce11095cb..c809e74e5 100644
--- a/include/strata/kernels/native_mmvq.hpp
+++ b/include/strata/kernels/native_mmvq.hpp
@@ -110,8 +110,15 @@ void native_q8_0_packed_register(const void* gguf_weights, const void* packed, i
void native_q8_0_packed_unregister(const void* gguf_weights);
/// STRATA_Q8_SM60=1 (opt-in, Pascal GP100): every Q8_0 dense matrix is packed as above whatever its shape, and the
/// Q8_0 decode GEMVs run a kernel written for compute capability 6.0 (q8_sm60.cuh; not bitwise the exact kernels).
-/// native_q8_0_sm60_pack packs a device matrix here (the head) and owns the copy until native_q8_0_sm60_release.
+/// native_q8_0_sm60_pack packs a device matrix here (the head) and owns the copy until native_q8_0_sm60_release;
+/// native_q8_0_sm60_pack_quiet is the same with no line for a success (the caller sums several up).
bool native_q8_0_sm60_pack(const void* weights, int n_in, int n_out, const char* what);
+bool native_q8_0_sm60_pack_quiet(const void* weights, int n_in, int n_out, const char* what);
+/// The same for a matrix that nothing but native_mmvq reads (the MTP drafter's draft head subset): where every column
+/// count takes the Pascal kernel and the planes stay 16-byte aligned, they replace the GGUF blocks in the matrix's own
+/// allocation (the same byte count, staged through the host: no second copy); otherwise a packed copy as
+/// native_q8_0_sm60_pack makes. Either way native_q8_0_sm60_release must run before the caller frees the matrix.
+bool native_q8_0_sm60_pack_in_place(void* weights, int n_in, int n_out, const char* what);
void native_q8_0_sm60_release(const void* weights);
/// STRATA_Q6_PACKED=1 (opt-in): a packed copy of a Q6_K matrix (the output heads) - the same bytes as ql / qh /
diff --git a/metal/src/engine/stubs_0145.cpp b/metal/src/engine/stubs_0145.cpp
index 0c1976f60..351665ecd 100644
--- a/metal/src/engine/stubs_0145.cpp
+++ b/metal/src/engine/stubs_0145.cpp
@@ -147,5 +147,8 @@ namespace strata::kernels { bool gemm_iq_f16_grouped(int p0, void const* p1, uns
// STRATA_Q8_SM60 (Pascal GP100, opt-in): a no-op on Metal; NativeHead::load()/unload() call these for a Q8_0 head, so they must exist and answer "nothing packed".
namespace strata::kernels { bool native_q8_0_sm60_pack(void const* p0, int p1, int p2, char const* p3) { return false; } }
namespace strata::kernels { void native_q8_0_sm60_release(void const* p0) { } }
+// the MTP drafter's STRATA_MTP_Q8_SM60 packing (mtp.cpp): nothing packed on Metal
+namespace strata::kernels { bool native_q8_0_sm60_pack_quiet(void const* p0, int p1, int p2, char const* p3) { return false; } }
+namespace strata::kernels { bool native_q8_0_sm60_pack_in_place(void* p0, int p1, int p2, char const* p3) { return false; } }
// only reached with pf_wmma (off on Metal)
namespace strata::prefill { void swiglu_split_f16(float const* p0, unsigned short* p1, long long p2, int p3, bool p4, void* p5) { } }
diff --git a/src/core/mtp.cpp b/src/core/mtp.cpp
index 292a5d2b4..99ad68f28 100644
--- a/src/core/mtp.cpp
+++ b/src/core/mtp.cpp
@@ -78,6 +78,16 @@ bool mtp_catchup_all() {
}();
return on;
}
+// STRATA_MTP_Q8_SM60=1 (opt-in, with STRATA_Q8_SM60=1): the drafter's own Q8_0 matrices (the draft head's subset,
+// the projections) run on the Pascal GEMV too (the drafts differ in rounding only: the Pascal kernel sums in another
+// order; the replies do not depend on the drafts)
+bool mtp_q8_sm60() {
+ static const bool on = [] {
+ const char* v = std::getenv("STRATA_MTP_Q8_SM60");
+ return v != nullptr && v[0] == '1';
+ }();
+ return on;
+}
constexpr int GGML_Q8_0 = 8;
constexpr int GGML_Q4_0 = 2;
using Clock = std::chrono::steady_clock;
@@ -154,12 +164,20 @@ MtpDrafter::~MtpDrafter() {
if (side_) cudaStreamDestroy(side_);
if (sh_fork_) cudaEventDestroy(sh_fork_);
if (sh_join_) cudaEventDestroy(sh_join_);
- if (owns_weights_ && dense_) cudaFree(dense_);
+ if (owns_weights_ && dense_) {
+ for (const auto& t : tensors_) // the STRATA_MTP_Q8_SM60 registrations first (a no-op for the rest)
+ if (t.kind == "q8_0") strata::kernels::native_q8_0_sm60_release(dense_ + t.off);
+ cudaFree(dense_);
+ }
if (owns_weights_ && experts_) cudaFree(experts_);
if (state_arena_) cudaFree(state_arena_);
if (arena_) cudaFree(arena_);
if (head_logits_) cudaFree(head_logits_);
- if (owns_draft_head_ && dhead_) { strata::kernels::native_q6_k_unpack(dhead_); cudaFree(dhead_); }
+ if (owns_draft_head_ && dhead_) {
+ strata::kernels::native_q6_k_unpack(dhead_);
+ strata::kernels::native_q8_0_sm60_release(dhead_);
+ cudaFree(dhead_);
+ }
if (owns_draft_head_ && dvocab_) cudaFree(dvocab_);
if (ev_chain_) cudaEventDestroy(ev_chain_);
for (cudaEvent_t e : ev_step_) if (e) cudaEventDestroy(e);
@@ -335,6 +353,26 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
"self_attn.v_proj.weight", "self_attn.o_proj.weight", "mlp.shared_expert.gate_proj.weight",
"mlp.shared_expert.up_proj.weight", "mlp.shared_expert.down_proj.weight"};
for (const char* n : required) if (!q8(n)) { err = std::string("mtp: ") + n + " is missing (q8_0)"; return false; }
+ // STRATA_MTP_Q8_SM60=1: the projections on the Pascal GEMV (--batch's shared drafters read the same pointers), as
+ // packed copies like the main model's: the prompt path's draft_kv reads four of them in the GGUF layout (MMQ);
+ // --mtp-q4 reads its Q4_0 copies instead
+ if (shared == nullptr && dense4_ == nullptr && mtp_q8_sm60()) {
+ int packed = 0;
+ uint64_t packed_bytes = 0;
+ for (const char* n : required)
+ for (const auto& t : tensors_)
+ if (t.name == n && t.kind == "q8_0" &&
+ strata::kernels::native_q8_0_sm60_pack_quiet(dense_ + t.off, (int) t.cols, (int) t.rows,
+ ("MTP " + t.name).c_str())) {
+ ++packed;
+ packed_bytes += t.bytes; // a copy is the same 34 bytes per 32 values
+ }
+ vram_ += packed_bytes;
+ if (packed > 0)
+ std::fprintf(stderr, "strata mtp: %d of %zu projections on the Pascal Q8_0 GEMV (STRATA_MTP_Q8_SM60=1, %.1f "
+ "MiB more VRAM)\n", packed, sizeof(required) / sizeof(required[0]),
+ (double) packed_bytes / 1048576.0);
+ }
// ---- the layer's own K/V (dense attention: no indexer state is read)
const strata::kernels::QsaShapes s = shapes_of(g);
@@ -742,6 +780,12 @@ bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const float
if (dhead_type_ == 14 && strata::kernels::native_q6_k_packed_enabled()) // STRATA_Q6_PACKED=1
strata::kernels::native_q6_k_pack(dhead_, (int) (row_bytes / 210) * 256, (int) n_dvocab_,
"MTP draft head");
+ // STRATA_MTP_Q8_SM60=1: the subset on the Pascal GEMV like the main head, but packed in place - nothing but
+ // record_rest's native_mmvq reads it (the prompt path never does), and a copy would cost its size again
+ if (dhead_type_ == GGML_Q8_0 && owns_draft_head_ && mtp_q8_sm60() && row_bytes == g_->n_embd / 32 * 34 &&
+ strata::kernels::native_q8_0_sm60_pack_in_place(dhead_, (int) g_->n_embd, (int) n_dvocab_,
+ "MTP draft head"))
+ std::fprintf(stderr, "strata mtp: the draft head on the Pascal Q8_0 GEMV (STRATA_MTP_Q8_SM60=1)\n");
}
}
if (coupled_draft_env() && cparams_ == nullptr && !setup_coupled(err)) return false;
diff --git a/src/kernels/cuda/native_mmvq.cu b/src/kernels/cuda/native_mmvq.cu
index 05f9b42d4..afaec6cb5 100644
--- a/src/kernels/cuda/native_mmvq.cu
+++ b/src/kernels/cuda/native_mmvq.cu
@@ -2243,15 +2243,25 @@ bool native_q8_0_packed_eligible(int n_in, int n_out) {
}
namespace {
-std::unordered_map<const void*, void*>& q8_sm60_owned() { // packed copies made here (the head), freed by release
+// packed copies made here (the head), freed by release; null: packed in place (release only unregisters it)
+std::unordered_map<const void*, void*>& q8_sm60_owned() {
static std::unordered_map<const void*, void*> owned;
return owned;
}
+bool q8_sm60_pack_impl(const void* weights, int n_in, int n_out, const char* what, bool quiet);
} // namespace
bool native_q8_0_sm60_pack(const void* weights, int n_in, int n_out, const char* what) {
+ return q8_sm60_pack_impl(weights, n_in, n_out, what, false);
+}
+bool native_q8_0_sm60_pack_quiet(const void* weights, int n_in, int n_out, const char* what) {
+ return q8_sm60_pack_impl(weights, n_in, n_out, what, true);
+}
+
+namespace {
+bool q8_sm60_pack_impl(const void* weights, int n_in, int n_out, const char* what, bool quiet) {
#if defined(__HIPCC__)
- (void) weights; (void) n_in; (void) n_out; (void) what;
+ (void) weights; (void) n_in; (void) n_out; (void) what; (void) quiet;
return false;
#else
if (!q8_sm60_on() || !weights || !native_q8_0_packed_eligible(n_in, n_out)) return false;
@@ -2276,18 +2286,20 @@ bool native_q8_0_sm60_pack(const void* weights, int n_in, int n_out, const char*
}
q8_packed_registry()[weights] = Q8Packed{qs, d, n_in, n_out};
q8_sm60_owned()[weights] = packed;
- std::fprintf(stderr, "strata q8 sm60: packed %s (%d x %d, %.0f MiB more VRAM)\n", what, n_out, n_in,
- double(bytes) / 1048576.0);
+ if (!quiet)
+ std::fprintf(stderr, "strata q8 sm60: packed %s (%d x %d, %.0f MiB more VRAM)\n", what, n_out, n_in,
+ double(bytes) / 1048576.0);
return true;
#endif
}
+} // namespace
void native_q8_0_sm60_release(const void* weights) {
auto& owned = q8_sm60_owned();
const auto it = owned.find(weights);
if (it == owned.end()) return;
q8_packed_registry().erase(weights);
- cudaFree(it->second);
+ if (it->second) cudaFree(it->second);
owned.erase(it);
}
@@ -2313,6 +2325,40 @@ void native_q8_0_packed_register(const void* gguf_weights, const void* packed, i
void native_q8_0_packed_unregister(const void* gguf_weights) { q8_packed_registry().erase(gguf_weights); }
+bool native_q8_0_sm60_pack_in_place(void* weights, int n_in, int n_out, const char* what) {
+#if defined(__HIPCC__)
+ (void) weights; (void) n_in; (void) n_out; (void) what;
+ return false;
+#else
+ if (!q8_sm60_on() || !weights || !native_q8_0_packed_eligible(n_in, n_out)) return false;
+ if (q8_packed_registry().count(weights)) return true;
+ // The GGUF blocks are gone afterwards, so only where q8_packed_mmvq can never fall back to the GGUF-layout kernels
+ // (every column count fits the kernel's shared-memory planes) and its 16-byte weight loads stay aligned
+ if (q8sm60::smem_bytes(MAX_NCOLS, n_in) > q8sm60::MAX_SMEM || reinterpret_cast<std::uintptr_t>(weights) % 16 != 0)
+ return native_q8_0_sm60_pack(weights, n_in, n_out, what);
+ const std::size_t bytes = std::size_t(n_in / 32) * std::size_t(n_out) * 34; // both layouts: 34 bytes per 32
+ std::vector<uint8_t> blocks(bytes), planes(bytes);
+ if (cudaMemcpy(blocks.data(), weights, bytes, cudaMemcpyDeviceToHost) != cudaSuccess) {
+ cudaGetLastError();
+ std::fprintf(stderr, "strata q8 sm60: reading the %s failed; it keeps the GGUF layout\n", what);
+ return false;
+ }
+ native_q8_0_pack_host(blocks.data(), planes.data(), n_in, n_out);
+ if (cudaMemcpy(weights, planes.data(), bytes, cudaMemcpyHostToDevice) != cudaSuccess) {
+ cudaGetLastError();
+ // a partial write leaves neither layout: put the blocks back
+ const bool restored = cudaMemcpy(weights, blocks.data(), bytes, cudaMemcpyHostToDevice) == cudaSuccess;
+ cudaGetLastError();
+ std::fprintf(stderr, "strata q8 sm60: writing the packed %s failed; %s\n", what,
+ restored ? "it keeps the GGUF layout" : "its GGUF blocks could not be restored either");
+ return false;
+ }
+ native_q8_0_packed_register(weights, weights, n_in, n_out);
+ q8_sm60_owned()[weights] = nullptr;
+ return true; // the caller says what it packed
+#endif
+}
+
bool native_q6_k_packed_enabled() {
static const bool enabled = [] {
const char* v = std::getenv("STRATA_Q6_PACKED");