Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Prefill: opt-in STRATA_DEQUANT_SIDE=1 dequantizes the dense weights on a side stream into a double-buffered scratch (#3779, #3712)

linxuhaodc390c38647a
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 69a3fa7c0..a7183b2e7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1184,6 +1184,12 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
add_executable(gemm_bf16_parity src/prefill/gemm_bf16_parity.cu)
target_link_libraries(gemm_bf16_parity PRIVATE strata_prefill )
add_test(NAME gemm_bf16_parity COMMAND gemm_bf16_parity)
+ # STRATA_DEQUANT_SIDE: Gemm::native with the dense dequant on a side stream (two scratches) against the serial form,
+ # bitwise (GPU, synthetic, no model; 77 without a device)
+ add_executable(gemm_dequant_side_parity src/prefill/gemm_dequant_side_parity.cu)
+ target_link_libraries(gemm_dequant_side_parity PRIVATE strata_prefill )
+ add_test(NAME gemm_dequant_side_parity COMMAND gemm_dequant_side_parity)
+ set_tests_properties(gemm_dequant_side_parity PROPERTIES SKIP_RETURN_CODE 77)
# Pascal, opt-in STRATA_DENSE_F32=1: the dense projections' FP16 GEMMs as cublasSgemm on FP32 copies, bitwise vs
# the cuBLAS FP16 call over a sweep of chunk sizes (GPU, synthetic, no model; --full sweeps every size, --bench
# adds timings). Not a ctest: whether the two agree is a property of the cuBLAS build, which this checks.
diff --git a/include/strata/prefill/gemm.hpp b/include/strata/prefill/gemm.hpp
index 596b3e4bf..80f625ee5 100644
--- a/include/strata/prefill/gemm.hpp
+++ b/include/strata/prefill/gemm.hpp
@@ -66,6 +66,13 @@ public:
/// Caller-owned buffers only: the scratch and workspace moved (the prompt path laid its buffers out again).
void rebind(uint16_t* scratch, int64_t scratch_elems, void* workspace, size_t ws_bytes);
+ /// STRATA_DEQUANT_SIDE=1 (gemm_dequant_side): a second dequant scratch of scratch_elems() elements, caller-owned
+ /// like the first. native() then dequantizes on a side stream into the two in turn, so a weight's dequant runs
+ /// while the kernels queued ahead of its GEMM do; events order every write and read of each scratch, so the
+ /// products are the serial form's, bit for bit. nullptr: the serial form. Called after init_external and
+ /// after every rebind (the "scratch free" events are re-armed on the compute stream). CUDA only.
+ bool set_dequant_side(uint16_t* scratch2, std::string& err);
+
uint16_t* scratch() const { return scratch_; }
int64_t scratch_elems() const { return scratch_elems_; }
void* stream() const { return stream_; }
@@ -77,6 +84,14 @@ private:
int64_t scratch_elems_ = 0;
void* workspace_ = nullptr;
bool external_ = false;
+ // STRATA_DEQUANT_SIDE (set_dequant_side): the second scratch, the side stream and, per scratch, "the dequant
+ // landed" (recorded on the side stream) and "the GEMM read it" (recorded on the compute stream); side_cur_ is
+ // the scratch the next native() dequantizes into (0: scratch_, 1: side_scratch_)
+ uint16_t* side_scratch_ = nullptr;
+ void* side_stream_ = nullptr;
+ void* side_ready_[2] = {};
+ void* side_free_[2] = {};
+ int side_cur_ = 0;
void* hipblaslt_state_ = nullptr;
void* rocblas_state_ = nullptr; // HIP: the rocBLAS solution table of f16_inplace (STRATA_ROCBLAS_TUNING)
bool f16_io_ = false;
@@ -122,6 +137,11 @@ void gemm_set_dense_f32(int mode);
/// set_act_f16). Cached per device (a layer split can mix cards). STRATA_HIP_PROMPT_F16=0/1 overrides.
bool prompt_f16();
+/// STRATA_DEQUANT_SIDE=1: the prompt path's dense-weight dequant on a side stream with a second GEMM scratch
+/// (Gemm::set_dequant_side; the prompt path takes the scratch from its borrowed region when this is on). Off by
+/// default; always off on HIP (its padded / MMQ paths use the scratch on the compute stream without the events).
+bool gemm_dequant_side();
+
} // namespace strata::prefill
diff --git a/metal/src/engine/stubs_0145.cpp b/metal/src/engine/stubs_0145.cpp
index 791d1df4a..da99868cf 100644
--- a/metal/src/engine/stubs_0145.cpp
+++ b/metal/src/engine/stubs_0145.cpp
@@ -151,3 +151,6 @@ namespace strata::kernels { void native_q8_0_sm60_release(void const* p0) { } }
namespace strata::prefill { void swiglu_split_f16(float const* p0, unsigned short* p1, long long p2, int p3, bool p4, void* p5) { } }
// STRATA_PLE_COMPACT (opt-in, CUDA / HIP prompt path): prefill.cpp references it; the Metal prompt runs through its window and never reaches it.
namespace strata::kernels { void ple_iq4nl_dequant(const uint8_t* p0, int64_t p1, float* p2, void* p3) { strata::metal_engine::missing("strata::kernels::ple_iq4nl_dequant"); } }
+// STRATA_DEQUANT_SIDE (opt-in, CUDA only): off on Metal; the prompt path takes no second scratch and set_dequant_side(nullptr) keeps the serial form.
+namespace strata::prefill { bool gemm_dequant_side() { return false; } }
+auto strata::prefill::Gemm::set_dequant_side(uint16_t* p0, std::string& p1) -> bool { if (p0 == nullptr) return true; strata::metal_engine::missing("strata::prefill::Gemm::set_dequant_side"); return false; }
diff --git a/src/prefill/gemm.cu b/src/prefill/gemm.cu
index a52af7503..6d7816c96 100644
--- a/src/prefill/gemm.cu
+++ b/src/prefill/gemm.cu
@@ -627,7 +627,21 @@ const uint16_t* Gemm::unpad_x(const uint16_t* X, int64_t T, int64_t K, int64_t l
return unpadded_x_;
}
+bool gemm_dequant_side() {
+#if defined(__HIPCC__)
+ return false;
+#else
+ static const bool on = [] { const char* v = std::getenv("STRATA_DEQUANT_SIDE"); return v && v[0] == '1'; }();
+ return on;
+#endif
+}
+
Gemm::~Gemm() {
+ for (void* e : side_ready_)
+ if (e) (void) cudaEventDestroy((cudaEvent_t) e);
+ for (void* e : side_free_)
+ if (e) (void) cudaEventDestroy((cudaEvent_t) e);
+ if (side_stream_) (void) cudaStreamDestroy((cudaStream_t) side_stream_);
#if defined(STRATA_PREFILL_MMQ) && defined(__HIPCC__)
delete static_cast<strata::prefill::mmq::Context*>(mmq_ctx_);
if (mmq_buf_) cudaFree(mmq_buf_);
@@ -687,6 +701,38 @@ void Gemm::rebind(uint16_t* scratch, int64_t scratch_elems, void* workspace, siz
#endif
}
+bool Gemm::set_dequant_side(uint16_t* scratch2, std::string& err) {
+ side_scratch_ = scratch2;
+ side_cur_ = 0;
+ if (scratch2 == nullptr) return true;
+ if (side_stream_ == nullptr) { // once: the stream and the four events outlive every rebind
+ cudaStream_t s = nullptr;
+ if (const cudaError_t e = cudaStreamCreateWithFlags(&s, cudaStreamNonBlocking); e != cudaSuccess) {
+ err = std::string("prefill gemm: dequant side stream: ") + cudaGetErrorString(e);
+ side_scratch_ = nullptr;
+ return false;
+ }
+ side_stream_ = s;
+ for (int i = 0; i < 2; ++i) {
+ cudaEvent_t r = nullptr, f = nullptr;
+ const bool made = cudaEventCreateWithFlags(&r, cudaEventDisableTiming) == cudaSuccess &&
+ cudaEventCreateWithFlags(&f, cudaEventDisableTiming) == cudaSuccess;
+ side_ready_[i] = r; // kept even on a failure: the destructor frees what was made
+ side_free_[i] = f;
+ if (!made) {
+ err = "prefill gemm: dequant side events";
+ side_scratch_ = nullptr;
+ return false;
+ }
+ }
+ }
+ // both scratches are free once what the compute stream holds now is done (a rebind follows a stream sync, so
+ // this is belt and braces; a wait on an event never recorded would pass anyway)
+ for (int i = 0; i < 2; ++i)
+ (void) cudaEventRecord((cudaEvent_t) side_free_[i], (cudaStream_t) stream_);
+ return true;
+}
+
bool Gemm::init(void* stream, int64_t scratch_elems, std::string& err) {
// #240: every failure names the call and the real status, so "no VRAM" can be told from a broken install
cublasHandle_t h = nullptr;
@@ -1356,6 +1402,24 @@ void Gemm::native(const uint16_t* X, int ggml_type, const void* W_blocks, float*
strata::kernels::dequant_f16(ggml_type, W_blocks, r0, n, K, scratch_, stream_);
f16(X, scratch_, Y + r0, T, n, K, ldy, beta);
}
+ // the side stream's next dequant into scratch_ waits for these slices' reads and writes of it
+ if (side_scratch_) (void) cudaEventRecord((cudaEvent_t) side_free_[0], (cudaStream_t) stream_);
+ return;
+ }
+ if (side_scratch_) {
+ // STRATA_DEQUANT_SIDE: the dequant on the side stream, into the scratch the compute stream read last but one.
+ // free[cur] -> dequant -> ready[cur] -> GEMM -> free[cur] orders the dequant after the GEMM that last read
+ // this scratch and the GEMM after the dequant that filled it, so every read and write of a scratch is where
+ // the serial form had it - the same kernels, the same bits - and the dequant runs while the kernels queued
+ // ahead of this GEMM do.
+ uint16_t* const s = side_cur_ == 0 ? scratch_ : side_scratch_;
+ (void) cudaStreamWaitEvent((cudaStream_t) side_stream_, (cudaEvent_t) side_free_[side_cur_], 0);
+ strata::kernels::dequant_f16(ggml_type, W_blocks, 0, N, K, s, side_stream_);
+ (void) cudaEventRecord((cudaEvent_t) side_ready_[side_cur_], (cudaStream_t) side_stream_);
+ (void) cudaStreamWaitEvent((cudaStream_t) stream_, (cudaEvent_t) side_ready_[side_cur_], 0);
+ f16(X, s, Y, T, N, K, ldy, beta);
+ (void) cudaEventRecord((cudaEvent_t) side_free_[side_cur_], (cudaStream_t) stream_);
+ side_cur_ ^= 1;
return;
}
strata::kernels::dequant_f16(ggml_type, W_blocks, 0, N, K, scratch_, stream_);
diff --git a/src/prefill/gemm_dequant_side_parity.cu b/src/prefill/gemm_dequant_side_parity.cu
new file mode 100644
index 000000000..2ad906aef
--- /dev/null
+++ b/src/prefill/gemm_dequant_side_parity.cu
@@ -0,0 +1,121 @@
+// src/prefill/gemm_dequant_side_parity.cu - STRATA_DEQUANT_SIDE: Gemm::native with the dense-weight dequant on a side
+// stream into two scratches (Gemm::set_dequant_side) against the serial form, bitwise (GPU, synthetic, no model).
+//
+// A run of dense projections (Q4_K and Q6_K weights of the engine's shapes, one of them larger than the scratch so it takes
+// the sliced path, which keeps the serial form) is executed twice from the same inputs: once serially, once with the side
+// stream. Every output must be identical bit for bit. Also a run with a long queue of compute work ahead of each GEMM, so
+// the side dequant really overlaps it. Exit 77 without a CUDA device.
+#include "strata/prefill/gemm.hpp"
+
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <random>
+#include <string>
+#include <vector>
+
+namespace {
+__global__ void fill_f16(__half* p, int64_t n, unsigned seed, float scale) {
+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+ if (i >= n) return;
+ unsigned x = (unsigned) i * 2654435761u + seed;
+ x ^= x >> 16; x *= 2246822519u; x ^= x >> 13; x *= 3266489917u; x ^= x >> 16;
+ p[i] = __float2half(((float) (x & 0xffff) / 32768.0f - 1.0f) * scale);
+}
+// busy work on the compute stream ahead of a GEMM
+__global__ void spin(float* p, int64_t n, int iters) {
+ const int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
+ if (i >= n) return;
+ float v = p[i];
+ for (int k = 0; k < iters; ++k) v = v * 1.0000001f + 1e-7f;
+ p[i] = v;
+}
+bool ok(cudaError_t e, const char* what) {
+ if (e != cudaSuccess) std::printf("FAIL: %s: %s\n", what, cudaGetErrorString(e));
+ return e == cudaSuccess;
+}
+uint16_t sane_half(std::mt19937& rng) {
+ const uint32_t r = rng();
+ return (uint16_t) (((r >> 31) << 15) | ((5u + (r >> 10) % 5u) << 10) | (r & 0x3ffu));
+}
+struct Weight { int type, bytes_per_block; int at[2]; int64_t N, K; void* dev = nullptr; };
+} // namespace
+
+int main() {
+ setvbuf(stdout, nullptr, _IONBF, 0);
+ int n_dev = 0;
+ if (cudaGetDeviceCount(&n_dev) != cudaSuccess || n_dev < 1) { std::printf("no CUDA device (skipped)\n"); return 77; }
+ cudaStream_t st = nullptr;
+ if (!ok(cudaStreamCreate(&st), "stream")) return 1;
+ const int64_t T = 512;
+ const int64_t scratch_elems = 2560LL * 4096; // 6144 x 2560 does not fit: the sliced path
+ strata::prefill::Gemm gemm;
+ std::string err;
+ if (!gemm.init(st, scratch_elems, err)) { std::printf("FAIL: %s\n", err.c_str()); return 1; }
+ uint16_t* scratch2 = nullptr;
+ if (!ok(cudaMalloc((void**) &scratch2, (size_t) scratch_elems * 2), "scratch2")) return 1;
+
+ std::vector<Weight> ws = {
+ {12, 144, {0, 2}, 10240, 2560}, {14, 210, {208, -1}, 2560, 6144}, {12, 144, {0, 2}, 6144, 2560},
+ {14, 210, {208, -1}, 3072, 2560}, {12, 144, {0, 2}, 2560, 4096}, {13, 176, {0, 2}, 2560, 2560},
+ {12, 144, {0, 2}, 512, 2560}, {14, 210, {208, -1}, 12288, 2560},
+ };
+ std::mt19937 rng(20261026u);
+ for (Weight& w : ws) {
+ const size_t blocks = (size_t) (w.N * w.K / 256), bytes = blocks * (size_t) w.bytes_per_block;
+ std::vector<uint8_t> h(bytes);
+ for (auto& b : h) b = (uint8_t) (rng() & 0xff);
+ for (size_t k = 0; k < blocks; ++k)
+ for (int at : w.at)
+ if (at >= 0) { const uint16_t v = sane_half(rng); std::memcpy(&h[k * (size_t) w.bytes_per_block + (size_t) at], &v, 2); }
+ if (!ok(cudaMalloc(&w.dev, bytes), "W") || !ok(cudaMemcpy(w.dev, h.data(), bytes, cudaMemcpyHostToDevice), "W copy")) return 1;
+ }
+ int64_t max_k = 0, max_n = 0;
+ for (const Weight& w : ws) { max_k = std::max(max_k, w.K); max_n = std::max(max_n, w.N); }
+ uint16_t* X = nullptr;
+ float *busy = nullptr, *Y[2] = {nullptr, nullptr};
+ const int64_t busy_n = 1 << 22;
+ if (!ok(cudaMalloc((void**) &X, (size_t) T * max_k * 2), "X") || !ok(cudaMalloc((void**) &busy, (size_t) busy_n * 4), "busy") ||
+ !ok(cudaMalloc((void**) &Y[0], (size_t) T * max_n * 4 * ws.size()), "Y0") ||
+ !ok(cudaMalloc((void**) &Y[1], (size_t) T * max_n * 4 * ws.size()), "Y1"))
+ return 1;
+ fill_f16<<<(unsigned) ((T * max_k + 255) / 256), 256, 0, st>>>((__half*) X, T * max_k, 17u, 2.0f);
+ cudaMemset(busy, 0, (size_t) busy_n * 4);
+ if (!ok(cudaStreamSynchronize(st), "fill")) return 1;
+
+ int bad = 0;
+ for (int busy_iters : {0, 20000}) {
+ for (int side = 0; side < 2; ++side) {
+ std::string e2;
+ if (!gemm.set_dequant_side(side ? scratch2 : nullptr, e2)) { std::printf("FAIL: set_dequant_side: %s\n", e2.c_str()); return 1; }
+ cudaMemset(Y[side], 0xff, (size_t) T * max_n * 4 * ws.size());
+ for (int round = 0; round < 3; ++round) // the scratch parity of the second round differs from the first
+ for (size_t i = 0; i < ws.size(); ++i) {
+ if (busy_iters) spin<<<(unsigned) ((busy_n + 255) / 256), 256, 0, st>>>(busy, busy_n, busy_iters);
+ gemm.native(X, ws[i].type, ws[i].dev, Y[side] + (size_t) i * T * max_n, T, ws[i].N, ws[i].K, max_n, 0.0f);
+ }
+ if (!ok(cudaStreamSynchronize(st), "run") || !ok(cudaDeviceSynchronize(), "device")) return 1;
+ }
+ std::vector<uint8_t> a((size_t) T * max_n * 4 * ws.size()), b(a.size());
+ cudaMemcpy(a.data(), Y[0], a.size(), cudaMemcpyDeviceToHost);
+ cudaMemcpy(b.data(), Y[1], b.size(), cudaMemcpyDeviceToHost);
+ for (size_t i = 0; i < ws.size(); ++i) {
+ // the written part of the output: T rows of N floats at stride max_n
+ long long diff = 0;
+ for (int64_t r = 0; r < T; ++r)
+ if (std::memcmp(&a[((size_t) i * T * max_n + (size_t) r * max_n) * 4], &b[((size_t) i * T * max_n + (size_t) r * max_n) * 4],
+ (size_t) ws[i].N * 4) != 0)
+ ++diff;
+ const bool sliced = ws[i].N * ws[i].K > scratch_elems;
+ std::printf(" busy %-6d type %2d N %5lld K %4lld%s: %s\n", busy_iters, ws[i].type, (long long) ws[i].N, (long long) ws[i].K,
+ sliced ? " (sliced)" : "", diff ? "*** DIFFERS ***" : "bitwise equal");
+ if (diff) ++bad;
+ }
+ }
+ gemm.set_dequant_side(nullptr, err);
+ std::printf("gemm_dequant_side_parity: %s\n", bad ? "FAILED" : "OK");
+ return bad ? 1 : 0;
+}
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index 21f65e6a4..d8aa780b9 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -1582,8 +1582,11 @@ bool Prefill::init(const core::WeightTable& wt, const core::ModelGeometry& g, co
{
uint16_t* gs = o.take<uint16_t>((size_t) GEMM_SCRATCH, ok);
void* ws = o.take<uint8_t>(GEMM_WS, ok);
+ // STRATA_DEQUANT_SIDE: the second dequant scratch (counted by bytes_needed_impl, retaken by relayout)
+ uint16_t* gs2 = gemm_dequant_side() ? o.take<uint16_t>((size_t) GEMM_SCRATCH, ok) : nullptr;
if (!ok) { err = "prefill: GEMM scratch does not fit"; return false; }
if (!m.gemm.init_external(stream, gs, GEMM_SCRATCH, ws, GEMM_WS, err)) return false;
+ { std::string side_err; if (!m.gemm.set_dequant_side(gs2, side_err)) std::fprintf(stderr, "prefill: STRATA_DEQUANT_SIDE=1 not taken (%s): the dense weights are dequantized in front of their GEMMs\n", side_err.c_str()); }
// this stage's device decides, once, here (init runs on the stage's own device): its gr_* kernels write the
// image its GEMMs read, and set_act_f16 writes that device's symbol
const core::OnDevice on_stage(m.device);
@@ -1739,7 +1742,9 @@ bool Prefill::relayout(int64_t chunk, void* borrow, uint64_t borrow_bytes, std::
o.owned = &m.owned;
uint16_t* gs = o.take<uint16_t>((size_t) GEMM_SCRATCH, ok);
void* ws = o.take<uint8_t>(GEMM_WS, ok);
+ uint16_t* gs2 = gemm_dequant_side() ? o.take<uint16_t>((size_t) GEMM_SCRATCH, ok) : nullptr; // STRATA_DEQUANT_SIDE
if (ok) m.gemm.rebind(gs, GEMM_SCRATCH, ws, GEMM_WS);
+ if (ok) { std::string side_err; if (!m.gemm.set_dequant_side(gs2, side_err)) std::fprintf(stderr, "prefill: STRATA_DEQUANT_SIDE=1 not taken (%s): the dense weights are dequantized in front of their GEMMs\n", side_err.c_str()); }
if (!ok || !carve((size_t) chunk, &o)) {
size_t fb = 0, tb = 0;
cudaMemGetInfo(&fb, &tb);
@@ -2388,6 +2393,7 @@ uint64_t Prefill::bytes_needed_impl(const core::ModelGeometry& g, const core::Se
if (owned_pages) o.granule = 2ull << 20;
o.take<uint16_t>((size_t) GEMM_SCRATCH, ok);
o.take<uint8_t>(GEMM_WS, ok);
+ if (gemm_dequant_side()) o.take<uint16_t>((size_t) GEMM_SCRATCH, ok); // STRATA_DEQUANT_SIDE: the second scratch
auto f = [&](size_t n) { o.take<float>(n, ok); };
// `carve`'s order, buffer for buffer: emb, R, xn, grs, xn16, lo, lo16, gated, inj, mixed, mixed_bf, mixed_h,
// bo aliases emb. This counted `xn` unconditionally (carve takes it only under STRATA_GR_UNFUSED) and never counted