Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Prefill: opt-in STRATA_DEQUANT_SIDE=1 dequantizes the dense weights on a side stream into a double-buffered scratch (#3779, #3712)
linxuhaodc390c38647a
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 69a3fa7c0..a7183b2e7 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1184,6 +1184,12 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)add_executable(gemm_bf16_parity src/prefill/gemm_bf16_parity.cu)target_link_libraries(gemm_bf16_parity PRIVATE strata_prefill )add_test(NAME gemm_bf16_parity COMMAND gemm_bf16_parity)+ # STRATA_DEQUANT_SIDE: Gemm::native with the dense dequant on a side stream (two scratches) against the serial form,+ # bitwise (GPU, synthetic, no model; 77 without a device)+ add_executable(gemm_dequant_side_parity src/prefill/gemm_dequant_side_parity.cu)+ target_link_libraries(gemm_dequant_side_parity PRIVATE strata_prefill )+ add_test(NAME gemm_dequant_side_parity COMMAND gemm_dequant_side_parity)+ set_tests_properties(gemm_dequant_side_parity PROPERTIES SKIP_RETURN_CODE 77)# Pascal, opt-in STRATA_DENSE_F32=1: the dense projections' FP16 GEMMs as cublasSgemm on FP32 copies, bitwise vs# the cuBLAS FP16 call over a sweep of chunk sizes (GPU, synthetic, no model; --full sweeps every size, --bench# adds timings). Not a ctest: whether the two agree is a property of the cuBLAS build, which this checks.diff --git a/include/strata/prefill/gemm.hpp b/include/strata/prefill/gemm.hppindex 596b3e4bf..80f625ee5 100644--- a/include/strata/prefill/gemm.hpp+++ b/include/strata/prefill/gemm.hpp@@ -66,6 +66,13 @@ public:/// Caller-owned buffers only: the scratch and workspace moved (the prompt path laid its buffers out again).void rebind(uint16_t* scratch, int64_t scratch_elems, void* workspace, size_t ws_bytes);+ /// STRATA_DEQUANT_SIDE=1 (gemm_dequant_side): a second dequant scratch of scratch_elems() elements, caller-owned+ /// like the first. native() then dequantizes on a side stream into the two in turn, so a weight's dequant runs+ /// while the kernels queued ahead of its GEMM do; events order every write and read of each scratch, so the+ /// products are the serial form's, bit for bit. nullptr: the serial form. Called after init_external and+ /// after every rebind (the "scratch free" events are re-armed on the compute stream). CUDA only.+ bool set_dequant_side(uint16_t* scratch2, std::string& err);+uint16_t* scratch() const { return scratch_; }int64_t scratch_elems() const { return scratch_elems_; }void* stream() const { return stream_; }@@ -77,6 +84,14 @@ private:int64_t scratch_elems_ = 0;void* workspace_ = nullptr;bool external_ = false;+ // STRATA_DEQUANT_SIDE (set_dequant_side): the second scratch, the side stream and, per scratch, "the dequant+ // landed" (recorded on the side stream) and "the GEMM read it" (recorded on the compute stream); side_cur_ is+ // the scratch the next native() dequantizes into (0: scratch_, 1: side_scratch_)+ uint16_t* side_scratch_ = nullptr;+ void* side_stream_ = nullptr;+ void* side_ready_[2] = {};+ void* side_free_[2] = {};+ int side_cur_ = 0;void* hipblaslt_state_ = nullptr;void* rocblas_state_ = nullptr; // HIP: the rocBLAS solution table of f16_inplace (STRATA_ROCBLAS_TUNING)bool f16_io_ = false;@@ -122,6 +137,11 @@ void gemm_set_dense_f32(int mode);/// set_act_f16). Cached per device (a layer split can mix cards). STRATA_HIP_PROMPT_F16=0/1 overrides.bool prompt_f16();+/// STRATA_DEQUANT_SIDE=1: the prompt path's dense-weight dequant on a side stream with a second GEMM scratch+/// (Gemm::set_dequant_side; the prompt path takes the scratch from its borrowed region when this is on). Off by+/// default; always off on HIP (its padded / MMQ paths use the scratch on the compute stream without the events).+bool gemm_dequant_side();+} // namespace strata::prefilldiff --git a/metal/src/engine/stubs_0145.cpp b/metal/src/engine/stubs_0145.cppindex 791d1df4a..da99868cf 100644--- a/metal/src/engine/stubs_0145.cpp+++ b/metal/src/engine/stubs_0145.cpp@@ -151,3 +151,6 @@ namespace strata::kernels { void native_q8_0_sm60_release(void const* p0) { } }namespace strata::prefill { void swiglu_split_f16(float const* p0, unsigned short* p1, long long p2, int p3, bool p4, void* p5) { } }// STRATA_PLE_COMPACT (opt-in, CUDA / HIP prompt path): prefill.cpp references it; the Metal prompt runs through its window and never reaches it.namespace strata::kernels { void ple_iq4nl_dequant(const uint8_t* p0, int64_t p1, float* p2, void* p3) { strata::metal_engine::missing("strata::kernels::ple_iq4nl_dequant"); } }+// STRATA_DEQUANT_SIDE (opt-in, CUDA only): off on Metal; the prompt path takes no second scratch and set_dequant_side(nullptr) keeps the serial form.+namespace strata::prefill { bool gemm_dequant_side() { return false; } }+auto strata::prefill::Gemm::set_dequant_side(uint16_t* p0, std::string& p1) -> bool { if (p0 == nullptr) return true; strata::metal_engine::missing("strata::prefill::Gemm::set_dequant_side"); return false; }diff --git a/src/prefill/gemm.cu b/src/prefill/gemm.cuindex a52af7503..6d7816c96 100644--- a/src/prefill/gemm.cu+++ b/src/prefill/gemm.cu@@ -627,7 +627,21 @@ const uint16_t* Gemm::unpad_x(const uint16_t* X, int64_t T, int64_t K, int64_t lreturn unpadded_x_;}+bool gemm_dequant_side() {+#if defined(__HIPCC__)+ return false;+#else+ static const bool on = [] { const char* v = std::getenv("STRATA_DEQUANT_SIDE"); return v && v[0] == '1'; }();+ return on;+#endif+}+Gemm::~Gemm() {+ for (void* e : side_ready_)+ if (e) (void) cudaEventDestroy((cudaEvent_t) e);+ for (void* e : side_free_)+ if (e) (void) cudaEventDestroy((cudaEvent_t) e);+ if (side_stream_) (void) cudaStreamDestroy((cudaStream_t) side_stream_);#if defined(STRATA_PREFILL_MMQ) && defined(__HIPCC__)delete static_cast<strata::prefill::mmq::Context*>(mmq_ctx_);if (mmq_buf_) cudaFree(mmq_buf_);@@ -687,6 +701,38 @@ void Gemm::rebind(uint16_t* scratch, int64_t scratch_elems, void* workspace, siz#endif}+bool Gemm::set_dequant_side(uint16_t* scratch2, std::string& err) {+ side_scratch_ = scratch2;+ side_cur_ = 0;+ if (scratch2 == nullptr) return true;+ if (side_stream_ == nullptr) { // once: the stream and the four events outlive every rebind+ cudaStream_t s = nullptr;+ if (const cudaError_t e = cudaStreamCreateWithFlags(&s, cudaStreamNonBlocking); e != cudaSuccess) {+ err = std::string("prefill gemm: dequant side stream: ") + cudaGetErrorString(e);+ side_scratch_ = nullptr;+ return false;+ }+ side_stream_ = s;+ for (int i = 0; i < 2; ++i) {+ cudaEvent_t r = nullptr, f = nullptr;+ const bool made = cudaEventCreateWithFlags(&r, cudaEventDisableTiming) == cudaSuccess &&+ cudaEventCreateWithFlags(&f, cudaEventDisableTiming) == cudaSuccess;+ side_ready_[i] = r; // kept even on a failure: the destructor frees what was made+ side_free_[i] = f;+ if (!made) {+ err = "prefill gemm: dequant side events";+ side_scratch_ = nullptr;+ return false;+ }+ }+ }+ // both scratches are free once what the compute stream holds now is done (a rebind follows a stream sync, so+ // this is belt and braces; a wait on an event never recorded would pass anyway)+ for (int i = 0; i < 2; ++i)+ (void) cudaEventRecord((cudaEvent_t) side_free_[i], (cudaStream_t) stream_);+ return true;+}+bool Gemm::init(void* stream, int64_t scratch_elems, std::string& err) {// #240: every failure names the call and the real status, so "no VRAM" can be told from a broken installcublasHandle_t h = nullptr;@@ -1356,6 +1402,24 @@ void Gemm::native(const uint16_t* X, int ggml_type, const void* W_blocks, float*strata::kernels::dequant_f16(ggml_type, W_blocks, r0, n, K, scratch_, stream_);f16(X, scratch_, Y + r0, T, n, K, ldy, beta);}+ // the side stream's next dequant into scratch_ waits for these slices' reads and writes of it+ if (side_scratch_) (void) cudaEventRecord((cudaEvent_t) side_free_[0], (cudaStream_t) stream_);+ return;+ }+ if (side_scratch_) {+ // STRATA_DEQUANT_SIDE: the dequant on the side stream, into the scratch the compute stream read last but one.+ // free[cur] -> dequant -> ready[cur] -> GEMM -> free[cur] orders the dequant after the GEMM that last read+ // this scratch and the GEMM after the dequant that filled it, so every read and write of a scratch is where+ // the serial form had it - the same kernels, the same bits - and the dequant runs while the kernels queued+ // ahead of this GEMM do.+ uint16_t* const s = side_cur_ == 0 ? scratch_ : side_scratch_;+ (void) cudaStreamWaitEvent((cudaStream_t) side_stream_, (cudaEvent_t) side_free_[side_cur_], 0);+ strata::kernels::dequant_f16(ggml_type, W_blocks, 0, N, K, s, side_stream_);+ (void) cudaEventRecord((cudaEvent_t) side_ready_[side_cur_], (cudaStream_t) side_stream_);+ (void) cudaStreamWaitEvent((cudaStream_t) stream_, (cudaEvent_t) side_ready_[side_cur_], 0);+ f16(X, s, Y, T, N, K, ldy, beta);+ (void) cudaEventRecord((cudaEvent_t) side_free_[side_cur_], (cudaStream_t) stream_);+ side_cur_ ^= 1;return;}strata::kernels::dequant_f16(ggml_type, W_blocks, 0, N, K, scratch_, stream_);diff --git a/src/prefill/gemm_dequant_side_parity.cu b/src/prefill/gemm_dequant_side_parity.cunew file mode 100644index 000000000..2ad906aef--- /dev/null+++ b/src/prefill/gemm_dequant_side_parity.cu@@ -0,0 +1,121 @@+// src/prefill/gemm_dequant_side_parity.cu - STRATA_DEQUANT_SIDE: Gemm::native with the dense-weight dequant on a side+// stream into two scratches (Gemm::set_dequant_side) against the serial form, bitwise (GPU, synthetic, no model).+//+// A run of dense projections (Q4_K and Q6_K weights of the engine's shapes, one of them larger than the scratch so it takes+// the sliced path, which keeps the serial form) is executed twice from the same inputs: once serially, once with the side+// stream. Every output must be identical bit for bit. Also a run with a long queue of compute work ahead of each GEMM, so+// the side dequant really overlaps it. Exit 77 without a CUDA device.+#include "strata/prefill/gemm.hpp"++#include <cuda_fp16.h>+#include <cuda_runtime.h>++#include <cstdint>+#include <cstdio>+#include <cstring>+#include <random>+#include <string>+#include <vector>++namespace {+__global__ void fill_f16(__half* p, int64_t n, unsigned seed, float scale) {+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;+ if (i >= n) return;+ unsigned x = (unsigned) i * 2654435761u + seed;+ x ^= x >> 16; x *= 2246822519u; x ^= x >> 13; x *= 3266489917u; x ^= x >> 16;+ p[i] = __float2half(((float) (x & 0xffff) / 32768.0f - 1.0f) * scale);+}+// busy work on the compute stream ahead of a GEMM+__global__ void spin(float* p, int64_t n, int iters) {+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;+ if (i >= n) return;+ float v = p[i];+ for (int k = 0; k < iters; ++k) v = v * 1.0000001f + 1e-7f;+ p[i] = v;+}+bool ok(cudaError_t e, const char* what) {+ if (e != cudaSuccess) std::printf("FAIL: %s: %s\n", what, cudaGetErrorString(e));+ return e == cudaSuccess;+}+uint16_t sane_half(std::mt19937& rng) {+ const uint32_t r = rng();+ return (uint16_t) (((r >> 31) << 15) | ((5u + (r >> 10) % 5u) << 10) | (r & 0x3ffu));+}+struct Weight { int type, bytes_per_block; int at[2]; int64_t N, K; void* dev = nullptr; };+} // namespace++int main() {+ setvbuf(stdout, nullptr, _IONBF, 0);+ int n_dev = 0;+ if (cudaGetDeviceCount(&n_dev) != cudaSuccess || n_dev < 1) { std::printf("no CUDA device (skipped)\n"); return 77; }+ cudaStream_t st = nullptr;+ if (!ok(cudaStreamCreate(&st), "stream")) return 1;+ const int64_t T = 512;+ const int64_t scratch_elems = 2560LL * 4096; // 6144 x 2560 does not fit: the sliced path+ strata::prefill::Gemm gemm;+ std::string err;+ if (!gemm.init(st, scratch_elems, err)) { std::printf("FAIL: %s\n", err.c_str()); return 1; }+ uint16_t* scratch2 = nullptr;+ if (!ok(cudaMalloc((void**) &scratch2, (size_t) scratch_elems * 2), "scratch2")) return 1;++ std::vector<Weight> ws = {+ {12, 144, {0, 2}, 10240, 2560}, {14, 210, {208, -1}, 2560, 6144}, {12, 144, {0, 2}, 6144, 2560},+ {14, 210, {208, -1}, 3072, 2560}, {12, 144, {0, 2}, 2560, 4096}, {13, 176, {0, 2}, 2560, 2560},+ {12, 144, {0, 2}, 512, 2560}, {14, 210, {208, -1}, 12288, 2560},+ };+ std::mt19937 rng(20261026u);+ for (Weight& w : ws) {+ const size_t blocks = (size_t) (w.N * w.K / 256), bytes = blocks * (size_t) w.bytes_per_block;+ std::vector<uint8_t> h(bytes);+ for (auto& b : h) b = (uint8_t) (rng() & 0xff);+ for (size_t k = 0; k < blocks; ++k)+ for (int at : w.at)+ if (at >= 0) { const uint16_t v = sane_half(rng); std::memcpy(&h[k * (size_t) w.bytes_per_block + (size_t) at], &v, 2); }+ if (!ok(cudaMalloc(&w.dev, bytes), "W") || !ok(cudaMemcpy(w.dev, h.data(), bytes, cudaMemcpyHostToDevice), "W copy")) return 1;+ }+ int64_t max_k = 0, max_n = 0;+ for (const Weight& w : ws) { max_k = std::max(max_k, w.K); max_n = std::max(max_n, w.N); }+ uint16_t* X = nullptr;+ float *busy = nullptr, *Y[2] = {nullptr, nullptr};+ const int64_t busy_n = 1 << 22;+ if (!ok(cudaMalloc((void**) &X, (size_t) T * max_k * 2), "X") || !ok(cudaMalloc((void**) &busy, (size_t) busy_n * 4), "busy") ||+ !ok(cudaMalloc((void**) &Y[0], (size_t) T * max_n * 4 * ws.size()), "Y0") ||+ !ok(cudaMalloc((void**) &Y[1], (size_t) T * max_n * 4 * ws.size()), "Y1"))+ return 1;+ fill_f16<<<(unsigned) ((T * max_k + 255) / 256), 256, 0, st>>>((__half*) X, T * max_k, 17u, 2.0f);+ cudaMemset(busy, 0, (size_t) busy_n * 4);+ if (!ok(cudaStreamSynchronize(st), "fill")) return 1;++ int bad = 0;+ for (int busy_iters : {0, 20000}) {+ for (int side = 0; side < 2; ++side) {+ std::string e2;+ if (!gemm.set_dequant_side(side ? scratch2 : nullptr, e2)) { std::printf("FAIL: set_dequant_side: %s\n", e2.c_str()); return 1; }+ cudaMemset(Y[side], 0xff, (size_t) T * max_n * 4 * ws.size());+ for (int round = 0; round < 3; ++round) // the scratch parity of the second round differs from the first+ for (size_t i = 0; i < ws.size(); ++i) {+ if (busy_iters) spin<<<(unsigned) ((busy_n + 255) / 256), 256, 0, st>>>(busy, busy_n, busy_iters);+ gemm.native(X, ws[i].type, ws[i].dev, Y[side] + (size_t) i * T * max_n, T, ws[i].N, ws[i].K, max_n, 0.0f);+ }+ if (!ok(cudaStreamSynchronize(st), "run") || !ok(cudaDeviceSynchronize(), "device")) return 1;+ }+ std::vector<uint8_t> a((size_t) T * max_n * 4 * ws.size()), b(a.size());+ cudaMemcpy(a.data(), Y[0], a.size(), cudaMemcpyDeviceToHost);+ cudaMemcpy(b.data(), Y[1], b.size(), cudaMemcpyDeviceToHost);+ for (size_t i = 0; i < ws.size(); ++i) {+ // the written part of the output: T rows of N floats at stride max_n+ long long diff = 0;+ for (int64_t r = 0; r < T; ++r)+ if (std::memcmp(&a[((size_t) i * T * max_n + (size_t) r * max_n) * 4], &b[((size_t) i * T * max_n + (size_t) r * max_n) * 4],+ (size_t) ws[i].N * 4) != 0)+ ++diff;+ const bool sliced = ws[i].N * ws[i].K > scratch_elems;+ std::printf(" busy %-6d type %2d N %5lld K %4lld%s: %s\n", busy_iters, ws[i].type, (long long) ws[i].N, (long long) ws[i].K,+ sliced ? " (sliced)" : "", diff ? "*** DIFFERS ***" : "bitwise equal");+ if (diff) ++bad;+ }+ }+ gemm.set_dequant_side(nullptr, err);+ std::printf("gemm_dequant_side_parity: %s\n", bad ? "FAILED" : "OK");+ return bad ? 1 : 0;+}diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 21f65e6a4..d8aa780b9 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -1582,8 +1582,11 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, co{uint16_t* gs = o.take<uint16_t>((size_t) GEMM_SCRATCH, ok);void* ws = o.take<uint8_t>(GEMM_WS, ok);+ // STRATA_DEQUANT_SIDE: the second dequant scratch (counted by bytes_needed_impl, retaken by relayout)+ uint16_t* gs2 = gemm_dequant_side() ? o.take<uint16_t>((size_t) GEMM_SCRATCH, ok) : nullptr;if (!ok) { err = "prefill: GEMM scratch does not fit"; return false; }if (!m.gemm.init_external(stream, gs, GEMM_SCRATCH, ws, GEMM_WS, err)) return false;+ { std::string side_err; if (!m.gemm.set_dequant_side(gs2, side_err)) std::fprintf(stderr, "prefill: STRATA_DEQUANT_SIDE=1 not taken (%s): the dense weights are dequantized in front of their GEMMs\n", side_err.c_str()); }// this stage's device decides, once, here (init runs on the stage's own device): its gr_* kernels write the// image its GEMMs read, and set_act_f16 writes that device's symbolconst core::OnDevice on_stage(m.device);@@ -1739,7 +1742,9 @@ bool Prefill::relayout(int64_t chunk, void* borrow, uint64_t borrow_bytes, std::o.owned = &m.owned;uint16_t* gs = o.take<uint16_t>((size_t) GEMM_SCRATCH, ok);void* ws = o.take<uint8_t>(GEMM_WS, ok);+ uint16_t* gs2 = gemm_dequant_side() ? o.take<uint16_t>((size_t) GEMM_SCRATCH, ok) : nullptr; // STRATA_DEQUANT_SIDEif (ok) m.gemm.rebind(gs, GEMM_SCRATCH, ws, GEMM_WS);+ if (ok) { std::string side_err; if (!m.gemm.set_dequant_side(gs2, side_err)) std::fprintf(stderr, "prefill: STRATA_DEQUANT_SIDE=1 not taken (%s): the dense weights are dequantized in front of their GEMMs\n", side_err.c_str()); }if (!ok || !carve((size_t) chunk, &o)) {size_t fb = 0, tb = 0;cudaMemGetInfo(&fb, &tb);@@ -2388,6 +2393,7 @@ uint64_t Prefill::bytes_needed_impl(const core::ModelGeometry& g, const core::Seif (owned_pages) o.granule = 2ull << 20;o.take<uint16_t>((size_t) GEMM_SCRATCH, ok);o.take<uint8_t>(GEMM_WS, ok);+ if (gemm_dequant_side()) o.take<uint16_t>((size_t) GEMM_SCRATCH, ok); // STRATA_DEQUANT_SIDE: the second scratchauto f = [&](size_t n) { o.take<float>(n, ok); };// `carve`'s order, buffer for buffer: emb, R, xn, grs, xn16, lo, lo16, gated, inj, mixed, mixed_bf, mixed_h,// bo aliases emb. This counted `xn` unconditionally (carve takes it only under STRATA_GR_UNFUSED) and never counted