Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

tests: STRATA_QFUSE parity for a window of two groups and for the GDN q8_1 destination (#3948)

Niko122111b049022856
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1ec4aef94..09459ac06 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1245,6 +1245,10 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_include_directories(gdn_parity PRIVATE ${_strata_gpu_include_directories})
target_link_libraries(gdn_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
add_test(NAME gdn_parity COMMAND gdn_parity --selftest)
+ # #3948: the same checks through the 4-blocks-a-head GDN step (the default on gfx1151), whose output-norm kernel writes the
+ # STRATA_QFUSE q8_1 image on its own
+ add_test(NAME gdn_parity_split COMMAND gdn_parity --selftest)
+ set_tests_properties(gdn_parity_split PROPERTIES ENVIRONMENT "STRATA_GDN_SPLIT=1")
# v100/gdn-chunk: the chunked GDN prompt recurrence vs the token recurrence and a host reference (GPU,
# synthetic, no model). The chunked path changes the summation order - FP32-level, not bit-exact - so the
diff --git a/src/kernels/gdn_parity.cpp b/src/kernels/gdn_parity.cpp
index a9ea9214a..ae650031a 100644
--- a/src/kernels/gdn_parity.cpp
+++ b/src/kernels/gdn_parity.cpp
@@ -21,6 +21,7 @@
#include "strata/kernels/gdn.hpp"
#include "strata/kernels/fused_gdn.hpp"
#include "strata/kernels/verify_kernels.hpp"
+#include "strata/kernels/native_mmvq.hpp"
#include <cuda_runtime.h>
@@ -504,6 +505,36 @@ int main(int argc, char** argv) {
const std::vector<float>& want_st = c.n_keep < 0 ? st0 : sr;
if (std::memcmp(st.data(), want_st.data(), st.size() * 4) != 0) ok = false;
++sn_n;
+ // #3948 (STRATA_QFUSE): the same call with a q8_1 destination also writes the outputs' q8_1 image, row t - tb
+ // at (t - tb) * value_dim: the bytes native_quantize_q8_1 makes of y rows [tb, n), the outputs and the state
+ // unchanged by asking for it, and nothing written past those rows (the engine's second group has tb > 0)
+ if (c.tb < n) {
+ const size_t qb = (size_t) (vd / 32) * 36, rows = (size_t) (n - c.tb);
+ void *d_q = dev((size_t) T * qb), *d_qr = dev((size_t) T * qb);
+ check(cudaMemset(d_q, 0x5a, (size_t) T * qb), "q0");
+ check(cudaMemcpy(d_st, st0.data(), st0.size() * 4, cudaMemcpyHostToDevice), "s1");
+ std::vector<float> y2(y.size()), st2(st0.size());
+ check(cudaMemset(d_y, 0, (size_t) T * vd * 4), "y1");
+ strata::kernels::gdn_step_norm_multi(d_st, d_h, C, d_gate, d_beta, d_z, d_gm, eps, d_y, h_k, h_v, T,
+ c.n_keep >= 0 ? d_nk : nullptr, nullptr, c.tb, d_q);
+ strata::kernels::native_quantize_q8_1(d_y + (size_t) c.tb * vd, d_qr, vd, (int) rows, nullptr);
+ check(cudaDeviceSynchronize(), "q sync");
+ std::vector<uint8_t> hq((size_t) T * qb), hr((size_t) T * qb);
+ check(cudaMemcpy(hq.data(), d_q, hq.size(), cudaMemcpyDeviceToHost), "q read");
+ check(cudaMemcpy(hr.data(), d_qr, rows * qb, cudaMemcpyDeviceToHost), "qr read");
+ down(y2, d_y); down(st2, d_st);
+ bool q_ok = std::memcmp(hq.data(), hr.data(), rows * qb) == 0;
+ for (size_t i = rows * qb; q_ok && i < hq.size(); ++i) q_ok = hq[i] == 0x5a;
+ for (int t = c.tb; q_ok && t < n; ++t)
+ q_ok = std::memcmp(&y2[(size_t) t * vd], &y[(size_t) t * vd], (size_t) vd * 4) == 0;
+ q_ok = q_ok && std::memcmp(st2.data(), st.data(), st.size() * 4) == 0;
+ if (!q_ok) {
+ std::printf(" *** gdn_step_norm_multi T=%d '%s' with a q8_1 destination: bytes differ from native_quantize_q8_1 or outputs/state changed ***\n", T, c.name);
+ ++sn_bad;
+ }
+ ++sn_n;
+ cudaFree(d_q); cudaFree(d_qr);
+ }
if (!ok) {
std::printf(" *** gdn_step_norm_multi T=%d '%s' differs from fused_gdn_step_norm (memcmp): %zu of %zu y words%s ***\n",
T, c.name, y_bad, y_all,
diff --git a/src/kernels/gr_parity.cpp b/src/kernels/gr_parity.cpp
index 34d7510bf..18cc57eb4 100644
--- a/src/kernels/gr_parity.cpp
+++ b/src/kernels/gr_parity.cpp
@@ -460,6 +460,120 @@ int fused_multi_lds_parity(const float* d_norm, const uint16_t* d_down, const ui
bad += qbad;
cudaFree(d_q); cudaFree(d_ref); cudaFree(d_cnt);
}
+
+ // #3948: STRATA_QFUSE in a verify window of TWO GROUPS (verify.cpp gr_read_group with G = 2): the window's tokens are
+ // read in two calls one after the other, the same group counters, each call with its own tokens' destinations - the
+ // attention half's xq_ is indexed from the group's first token, the expert half's nat_xq_ by the window's token.
+ // Fresh random inputs every repetition, 200 repetitions for each T = 2..8 (a race that fires one launch in a hundred
+ // shows), direct and as a graph replay. After every repetition: both groups' q8_1 bytes equal what
+ // native_quantize_q8_1 makes of `mixed`, nothing outside the written ranges changed, and the counters are back at zero.
+ if (!v3) {
+ constexpr int REPS = 200;
+ const size_t B = (size_t) (N / 32) * 36; // one token's q8_1 image
+ uint8_t *d_qa = nullptr, *d_qb = nullptr, *d_qabs = nullptr, *d_ref2 = nullptr;
+ unsigned* d_cnt2 = nullptr;
+ check(cudaMalloc(&d_qa, (size_t) T * B), "qfuse2 qa");
+ check(cudaMalloc(&d_qb, (size_t) T * B), "qfuse2 qb");
+ check(cudaMalloc(&d_qabs, (size_t) T * B), "qfuse2 qabs");
+ check(cudaMalloc(&d_ref2, (size_t) T * B), "qfuse2 ref");
+ check(cudaMalloc(&d_cnt2, (N / 32) * sizeof(unsigned)), "qfuse2 counters");
+ check(cudaMemset(d_cnt2, 0, (N / 32) * sizeof(unsigned)), "qfuse2 counters zero");
+ std::mt19937 prng(0x3948u);
+ std::normal_distribution<float> pn(0.0f, 0.3f);
+ std::vector<uint8_t> g_all((size_t) T * B), g_ref((size_t) T * B), g_a((size_t) T * B), g_b((size_t) T * B);
+ std::vector<unsigned> h_cnt(N / 32);
+ int g2bad = 0;
+ long runs = 0;
+ for (int tt = 2; tt <= T && !g2bad; ++tt) {
+ const int h = (tt + 1) / 2; // verify.cpp: tb_/te_ = {0, (T + 1) / 2}, {(T + 1) / 2, T}
+ std::vector<FusedGrArgs> rel1(args.begin(), args.begin() + h), rel2(args.begin() + h, args.begin() + tt);
+ std::vector<FusedGrArgs> abs1 = rel1, abs2 = rel2;
+ for (int t = 0; t < h; ++t) {
+ rel1[t].q8_mixed = d_qa + (size_t) t * B; rel1[t].q8_cnt = d_cnt2;
+ abs1[t].q8_mixed = d_qabs + (size_t) t * B; abs1[t].q8_cnt = d_cnt2;
+ }
+ for (int t = h; t < tt; ++t) {
+ rel2[t - h].q8_mixed = d_qb + (size_t) (t - h) * B; rel2[t - h].q8_cnt = d_cnt2;
+ abs2[t - h].q8_mixed = d_qabs + (size_t) t * B; abs2[t - h].q8_cnt = d_cnt2;
+ }
+ cudaGraph_t gg = nullptr;
+ cudaGraphExec_t gx = nullptr;
+ check(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal), "qfuse2 begin");
+ const bool w1 = fused_gr_read_multi(rel1.data(), h, d_xn, stream);
+ const bool w2 = fused_gr_read_multi(rel2.data(), tt - h, d_xn + (size_t) h * D, stream);
+ check(cudaStreamEndCapture(stream, &gg), "qfuse2 end");
+ check(cudaGraphInstantiate(&gx, gg, nullptr, nullptr, 0), "qfuse2 instantiate");
+ if (!w1 || !w2) { std::printf(" QFUSE two groups: T=%d: a group did not write its q8_1\n", tt); ++g2bad; }
+ for (int rep = 0; rep < REPS && !g2bad; ++rep) {
+ for (auto& x : r) x = pn(prng);
+ for (auto& x : bo) x = pn(prng);
+ for (auto& x : inj) x = pn(prng);
+ check(cudaMemcpy(d_r, r.data(), r.size() * sizeof(float), cudaMemcpyHostToDevice), "qfuse2 R");
+ check(cudaMemcpy(d_bo, bo.data(), bo.size() * sizeof(float), cudaMemcpyHostToDevice), "qfuse2 bo");
+ check(cudaMemcpy(d_inj, inj.data(), inj.size() * sizeof(float), cudaMemcpyHostToDevice), "qfuse2 inj");
+ check(cudaMemsetAsync(d_qa, 0x5a, (size_t) T * B, stream), "qfuse2 poison a");
+ check(cudaMemsetAsync(d_qb, 0x5a, (size_t) T * B, stream), "qfuse2 poison b");
+ check(cudaMemsetAsync(d_qabs, 0x5a, (size_t) T * B, stream), "qfuse2 poison abs");
+ const int mode = rep % 3; // 0: group-relative, direct; 1: window-indexed, direct; 2: group-relative, graph replay
+ if (mode == 0) {
+ fused_gr_read_multi(rel1.data(), h, d_xn, stream);
+ fused_gr_read_multi(rel2.data(), tt - h, d_xn + (size_t) h * D, stream);
+ } else if (mode == 1) {
+ fused_gr_read_multi(abs1.data(), h, d_xn, stream);
+ fused_gr_read_multi(abs2.data(), tt - h, d_xn + (size_t) h * D, stream);
+ } else {
+ check(cudaGraphLaunch(gx, stream), "qfuse2 replay");
+ }
+ strata::kernels::native_quantize_q8_1(d_mixed, d_ref2, N, tt, stream);
+ check(cudaStreamSynchronize(stream), "qfuse2 sync");
+ ++runs;
+ check(cudaMemcpy(g_ref.data(), d_ref2, (size_t) tt * B, cudaMemcpyDeviceToHost), "qfuse2 read ref");
+ check(cudaMemcpy(h_cnt.data(), d_cnt2, h_cnt.size() * sizeof(unsigned), cudaMemcpyDeviceToHost), "qfuse2 read counters");
+ for (size_t i = 0; i < h_cnt.size(); ++i)
+ if (h_cnt[i] != 0) {
+ std::printf(" QFUSE two groups: T=%d rep %d: counter %zu left at %u\n", tt, rep, i, h_cnt[i]);
+ ++g2bad;
+ break;
+ }
+ if (mode == 1) {
+ check(cudaMemcpy(g_all.data(), d_qabs, (size_t) T * B, cudaMemcpyDeviceToHost), "qfuse2 read abs");
+ if (std::memcmp(g_all.data(), g_ref.data(), (size_t) tt * B) != 0) {
+ for (int t = 0; t < tt; ++t)
+ if (std::memcmp(g_all.data() + (size_t) t * B, g_ref.data() + (size_t) t * B, B) != 0) {
+ std::printf(" QFUSE two groups (window-indexed): T=%d rep %d: token %d (group %d) differs\n", tt, rep, t, t < h ? 0 : 1);
+ break;
+ }
+ ++g2bad;
+ }
+ for (size_t i = (size_t) tt * B; i < (size_t) T * B; ++i)
+ if (g_all[i] != 0x5a) { std::printf(" QFUSE two groups: T=%d rep %d: wrote past the window at byte %zu\n", tt, rep, i); ++g2bad; break; }
+ } else {
+ check(cudaMemcpy(g_a.data(), d_qa, (size_t) T * B, cudaMemcpyDeviceToHost), "qfuse2 read a");
+ check(cudaMemcpy(g_b.data(), d_qb, (size_t) T * B, cudaMemcpyDeviceToHost), "qfuse2 read b");
+ if (std::memcmp(g_a.data(), g_ref.data(), (size_t) h * B) != 0) {
+ std::printf(" QFUSE two groups (group-relative, %s): T=%d rep %d: group 0 differs\n", mode == 2 ? "graph" : "direct", tt, rep);
+ ++g2bad;
+ }
+ if (std::memcmp(g_b.data(), g_ref.data() + (size_t) h * B, (size_t) (tt - h) * B) != 0) {
+ std::printf(" QFUSE two groups (group-relative, %s): T=%d rep %d: group 1 differs\n", mode == 2 ? "graph" : "direct", tt, rep);
+ ++g2bad;
+ }
+ for (size_t i = (size_t) h * B; i < (size_t) T * B; ++i)
+ if (g_a[i] != 0x5a) { std::printf(" QFUSE two groups: T=%d rep %d: group 0 wrote past its tokens at byte %zu\n", tt, rep, i); ++g2bad; break; }
+ for (size_t i = (size_t) (tt - h) * B; i < (size_t) T * B; ++i)
+ if (g_b[i] != 0x5a) { std::printf(" QFUSE two groups: T=%d rep %d: group 1 wrote past its tokens at byte %zu\n", tt, rep, i); ++g2bad; break; }
+ }
+ }
+ cudaGraphExecDestroy(gx);
+ cudaGraphDestroy(gg);
+ }
+ // the inputs the rest of the test (and its caller) expect again
+ check(cudaDeviceSynchronize(), "qfuse2 done");
+ std::printf(" fused GR read + q8_1 (STRATA_QFUSE), a window of two groups, T 2..8, %ld runs (%d random inputs each T), "
+ "group-relative and window-indexed, direct and graph: %s\n", runs, REPS, g2bad ? "FAIL" : "pass");
+ bad += g2bad;
+ cudaFree(d_qa); cudaFree(d_qb); cudaFree(d_qabs); cudaFree(d_ref2); cudaFree(d_cnt2);
+ }
std::printf(" fused GR multi max-T=8 LDS launch and changing graph replay %s\n",
bad == 0 ? "pass" : "FAIL");
check(cudaGraphExecDestroy(graph_exec), "multi graph exec destroy");