Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
tests: STRATA_QFUSE parity for a window of two groups and for the GDN q8_1 destination (#3948)
Niko122111b049022856
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 1ec4aef94..09459ac06 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1245,6 +1245,10 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_include_directories(gdn_parity PRIVATE ${_strata_gpu_include_directories})target_link_libraries(gdn_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})add_test(NAME gdn_parity COMMAND gdn_parity --selftest)+ # #3948: the same checks through the 4-blocks-a-head GDN step (the default on gfx1151), whose output-norm kernel writes the+ # STRATA_QFUSE q8_1 image on its own+ add_test(NAME gdn_parity_split COMMAND gdn_parity --selftest)+ set_tests_properties(gdn_parity_split PROPERTIES ENVIRONMENT "STRATA_GDN_SPLIT=1")# v100/gdn-chunk: the chunked GDN prompt recurrence vs the token recurrence and a host reference (GPU,# synthetic, no model). The chunked path changes the summation order - FP32-level, not bit-exact - so thediff --git a/src/kernels/gdn_parity.cpp b/src/kernels/gdn_parity.cppindex a9ea9214a..ae650031a 100644--- a/src/kernels/gdn_parity.cpp+++ b/src/kernels/gdn_parity.cpp@@ -21,6 +21,7 @@#include "strata/kernels/gdn.hpp"#include "strata/kernels/fused_gdn.hpp"#include "strata/kernels/verify_kernels.hpp"+#include "strata/kernels/native_mmvq.hpp"#include <cuda_runtime.h>@@ -504,6 +505,36 @@ int main(int argc, char** argv) {const std::vector<float>& want_st = c.n_keep < 0 ? st0 : sr;if (std::memcmp(st.data(), want_st.data(), st.size() * 4) != 0) ok = false;++sn_n;+ // #3948 (STRATA_QFUSE): the same call with a q8_1 destination also writes the outputs' q8_1 image, row t - tb+ // at (t - tb) * value_dim: the bytes native_quantize_q8_1 makes of y rows [tb, n), the outputs and the state+ // unchanged by asking for it, and nothing written past those rows (the engine's second group has tb > 0)+ if (c.tb < n) {+ const size_t qb = (size_t) (vd / 32) * 36, rows = (size_t) (n - c.tb);+ void *d_q = dev((size_t) T * qb), *d_qr = dev((size_t) T * qb);+ check(cudaMemset(d_q, 0x5a, (size_t) T * qb), "q0");+ check(cudaMemcpy(d_st, st0.data(), st0.size() * 4, cudaMemcpyHostToDevice), "s1");+ std::vector<float> y2(y.size()), st2(st0.size());+ check(cudaMemset(d_y, 0, (size_t) T * vd * 4), "y1");+ strata::kernels::gdn_step_norm_multi(d_st, d_h, C, d_gate, d_beta, d_z, d_gm, eps, d_y, h_k, h_v, T,+ c.n_keep >= 0 ? d_nk : nullptr, nullptr, c.tb, d_q);+ strata::kernels::native_quantize_q8_1(d_y + (size_t) c.tb * vd, d_qr, vd, (int) rows, nullptr);+ check(cudaDeviceSynchronize(), "q sync");+ std::vector<uint8_t> hq((size_t) T * qb), hr((size_t) T * qb);+ check(cudaMemcpy(hq.data(), d_q, hq.size(), cudaMemcpyDeviceToHost), "q read");+ check(cudaMemcpy(hr.data(), d_qr, rows * qb, cudaMemcpyDeviceToHost), "qr read");+ down(y2, d_y); down(st2, d_st);+ bool q_ok = std::memcmp(hq.data(), hr.data(), rows * qb) == 0;+ for (size_t i = rows * qb; q_ok && i < hq.size(); ++i) q_ok = hq[i] == 0x5a;+ for (int t = c.tb; q_ok && t < n; ++t)+ q_ok = std::memcmp(&y2[(size_t) t * vd], &y[(size_t) t * vd], (size_t) vd * 4) == 0;+ q_ok = q_ok && std::memcmp(st2.data(), st.data(), st.size() * 4) == 0;+ if (!q_ok) {+ std::printf(" *** gdn_step_norm_multi T=%d '%s' with a q8_1 destination: bytes differ from native_quantize_q8_1 or outputs/state changed ***\n", T, c.name);+ ++sn_bad;+ }+ ++sn_n;+ cudaFree(d_q); cudaFree(d_qr);+ }if (!ok) {std::printf(" *** gdn_step_norm_multi T=%d '%s' differs from fused_gdn_step_norm (memcmp): %zu of %zu y words%s ***\n",T, c.name, y_bad, y_all,diff --git a/src/kernels/gr_parity.cpp b/src/kernels/gr_parity.cppindex 34d7510bf..18cc57eb4 100644--- a/src/kernels/gr_parity.cpp+++ b/src/kernels/gr_parity.cpp@@ -460,6 +460,120 @@ int fused_multi_lds_parity(const float* d_norm, const uint16_t* d_down, const uibad += qbad;cudaFree(d_q); cudaFree(d_ref); cudaFree(d_cnt);}++ // #3948: STRATA_QFUSE in a verify window of TWO GROUPS (verify.cpp gr_read_group with G = 2): the window's tokens are+ // read in two calls one after the other, the same group counters, each call with its own tokens' destinations - the+ // attention half's xq_ is indexed from the group's first token, the expert half's nat_xq_ by the window's token.+ // Fresh random inputs every repetition, 200 repetitions for each T = 2..8 (a race that fires one launch in a hundred+ // shows), direct and as a graph replay. After every repetition: both groups' q8_1 bytes equal what+ // native_quantize_q8_1 makes of `mixed`, nothing outside the written ranges changed, and the counters are back at zero.+ if (!v3) {+ constexpr int REPS = 200;+ const size_t B = (size_t) (N / 32) * 36; // one token's q8_1 image+ uint8_t *d_qa = nullptr, *d_qb = nullptr, *d_qabs = nullptr, *d_ref2 = nullptr;+ unsigned* d_cnt2 = nullptr;+ check(cudaMalloc(&d_qa, (size_t) T * B), "qfuse2 qa");+ check(cudaMalloc(&d_qb, (size_t) T * B), "qfuse2 qb");+ check(cudaMalloc(&d_qabs, (size_t) T * B), "qfuse2 qabs");+ check(cudaMalloc(&d_ref2, (size_t) T * B), "qfuse2 ref");+ check(cudaMalloc(&d_cnt2, (N / 32) * sizeof(unsigned)), "qfuse2 counters");+ check(cudaMemset(d_cnt2, 0, (N / 32) * sizeof(unsigned)), "qfuse2 counters zero");+ std::mt19937 prng(0x3948u);+ std::normal_distribution<float> pn(0.0f, 0.3f);+ std::vector<uint8_t> g_all((size_t) T * B), g_ref((size_t) T * B), g_a((size_t) T * B), g_b((size_t) T * B);+ std::vector<unsigned> h_cnt(N / 32);+ int g2bad = 0;+ long runs = 0;+ for (int tt = 2; tt <= T && !g2bad; ++tt) {+ const int h = (tt + 1) / 2; // verify.cpp: tb_/te_ = {0, (T + 1) / 2}, {(T + 1) / 2, T}+ std::vector<FusedGrArgs> rel1(args.begin(), args.begin() + h), rel2(args.begin() + h, args.begin() + tt);+ std::vector<FusedGrArgs> abs1 = rel1, abs2 = rel2;+ for (int t = 0; t < h; ++t) {+ rel1[t].q8_mixed = d_qa + (size_t) t * B; rel1[t].q8_cnt = d_cnt2;+ abs1[t].q8_mixed = d_qabs + (size_t) t * B; abs1[t].q8_cnt = d_cnt2;+ }+ for (int t = h; t < tt; ++t) {+ rel2[t - h].q8_mixed = d_qb + (size_t) (t - h) * B; rel2[t - h].q8_cnt = d_cnt2;+ abs2[t - h].q8_mixed = d_qabs + (size_t) t * B; abs2[t - h].q8_cnt = d_cnt2;+ }+ cudaGraph_t gg = nullptr;+ cudaGraphExec_t gx = nullptr;+ check(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal), "qfuse2 begin");+ const bool w1 = fused_gr_read_multi(rel1.data(), h, d_xn, stream);+ const bool w2 = fused_gr_read_multi(rel2.data(), tt - h, d_xn + (size_t) h * D, stream);+ check(cudaStreamEndCapture(stream, &gg), "qfuse2 end");+ check(cudaGraphInstantiate(&gx, gg, nullptr, nullptr, 0), "qfuse2 instantiate");+ if (!w1 || !w2) { std::printf(" QFUSE two groups: T=%d: a group did not write its q8_1\n", tt); ++g2bad; }+ for (int rep = 0; rep < REPS && !g2bad; ++rep) {+ for (auto& x : r) x = pn(prng);+ for (auto& x : bo) x = pn(prng);+ for (auto& x : inj) x = pn(prng);+ check(cudaMemcpy(d_r, r.data(), r.size() * sizeof(float), cudaMemcpyHostToDevice), "qfuse2 R");+ check(cudaMemcpy(d_bo, bo.data(), bo.size() * sizeof(float), cudaMemcpyHostToDevice), "qfuse2 bo");+ check(cudaMemcpy(d_inj, inj.data(), inj.size() * sizeof(float), cudaMemcpyHostToDevice), "qfuse2 inj");+ check(cudaMemsetAsync(d_qa, 0x5a, (size_t) T * B, stream), "qfuse2 poison a");+ check(cudaMemsetAsync(d_qb, 0x5a, (size_t) T * B, stream), "qfuse2 poison b");+ check(cudaMemsetAsync(d_qabs, 0x5a, (size_t) T * B, stream), "qfuse2 poison abs");+ const int mode = rep % 3; // 0: group-relative, direct; 1: window-indexed, direct; 2: group-relative, graph replay+ if (mode == 0) {+ fused_gr_read_multi(rel1.data(), h, d_xn, stream);+ fused_gr_read_multi(rel2.data(), tt - h, d_xn + (size_t) h * D, stream);+ } else if (mode == 1) {+ fused_gr_read_multi(abs1.data(), h, d_xn, stream);+ fused_gr_read_multi(abs2.data(), tt - h, d_xn + (size_t) h * D, stream);+ } else {+ check(cudaGraphLaunch(gx, stream), "qfuse2 replay");+ }+ strata::kernels::native_quantize_q8_1(d_mixed, d_ref2, N, tt, stream);+ check(cudaStreamSynchronize(stream), "qfuse2 sync");+ ++runs;+ check(cudaMemcpy(g_ref.data(), d_ref2, (size_t) tt * B, cudaMemcpyDeviceToHost), "qfuse2 read ref");+ check(cudaMemcpy(h_cnt.data(), d_cnt2, h_cnt.size() * sizeof(unsigned), cudaMemcpyDeviceToHost), "qfuse2 read counters");+ for (size_t i = 0; i < h_cnt.size(); ++i)+ if (h_cnt[i] != 0) {+ std::printf(" QFUSE two groups: T=%d rep %d: counter %zu left at %u\n", tt, rep, i, h_cnt[i]);+ ++g2bad;+ break;+ }+ if (mode == 1) {+ check(cudaMemcpy(g_all.data(), d_qabs, (size_t) T * B, cudaMemcpyDeviceToHost), "qfuse2 read abs");+ if (std::memcmp(g_all.data(), g_ref.data(), (size_t) tt * B) != 0) {+ for (int t = 0; t < tt; ++t)+ if (std::memcmp(g_all.data() + (size_t) t * B, g_ref.data() + (size_t) t * B, B) != 0) {+ std::printf(" QFUSE two groups (window-indexed): T=%d rep %d: token %d (group %d) differs\n", tt, rep, t, t < h ? 0 : 1);+ break;+ }+ ++g2bad;+ }+ for (size_t i = (size_t) tt * B; i < (size_t) T * B; ++i)+ if (g_all[i] != 0x5a) { std::printf(" QFUSE two groups: T=%d rep %d: wrote past the window at byte %zu\n", tt, rep, i); ++g2bad; break; }+ } else {+ check(cudaMemcpy(g_a.data(), d_qa, (size_t) T * B, cudaMemcpyDeviceToHost), "qfuse2 read a");+ check(cudaMemcpy(g_b.data(), d_qb, (size_t) T * B, cudaMemcpyDeviceToHost), "qfuse2 read b");+ if (std::memcmp(g_a.data(), g_ref.data(), (size_t) h * B) != 0) {+ std::printf(" QFUSE two groups (group-relative, %s): T=%d rep %d: group 0 differs\n", mode == 2 ? "graph" : "direct", tt, rep);+ ++g2bad;+ }+ if (std::memcmp(g_b.data(), g_ref.data() + (size_t) h * B, (size_t) (tt - h) * B) != 0) {+ std::printf(" QFUSE two groups (group-relative, %s): T=%d rep %d: group 1 differs\n", mode == 2 ? "graph" : "direct", tt, rep);+ ++g2bad;+ }+ for (size_t i = (size_t) h * B; i < (size_t) T * B; ++i)+ if (g_a[i] != 0x5a) { std::printf(" QFUSE two groups: T=%d rep %d: group 0 wrote past its tokens at byte %zu\n", tt, rep, i); ++g2bad; break; }+ for (size_t i = (size_t) (tt - h) * B; i < (size_t) T * B; ++i)+ if (g_b[i] != 0x5a) { std::printf(" QFUSE two groups: T=%d rep %d: group 1 wrote past its tokens at byte %zu\n", tt, rep, i); ++g2bad; break; }+ }+ }+ cudaGraphExecDestroy(gx);+ cudaGraphDestroy(gg);+ }+ // the inputs the rest of the test (and its caller) expect again+ check(cudaDeviceSynchronize(), "qfuse2 done");+ std::printf(" fused GR read + q8_1 (STRATA_QFUSE), a window of two groups, T 2..8, %ld runs (%d random inputs each T), "+ "group-relative and window-indexed, direct and graph: %s\n", runs, REPS, g2bad ? "FAIL" : "pass");+ bad += g2bad;+ cudaFree(d_qa); cudaFree(d_qb); cudaFree(d_qabs); cudaFree(d_ref2); cudaFree(d_cnt2);+ }std::printf(" fused GR multi max-T=8 LDS launch and changing graph replay %s\n",bad == 0 ? "pass" : "FAIL");check(cudaGraphExecDestroy(graph_exec), "multi graph exec destroy");