Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
prefill timing: the hyper-connection write-back on its own line (it was charged to the phase before it, e.g. the wait for the first chunk's PLE rows); hip_prefill_gr_write_bench
Niko1221aef32e224b32
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 9591fe3f8..82249d4b3 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -817,6 +817,9 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)add_test(NAME hip_prefill_hcd_exact_parity COMMAND hip_prefill_hcd_exact_parity)set_tests_properties(hip_prefill_hcd_exact_parity PROPERTIES SKIP_RETURN_CODE 77)list(APPEND _strata_hip_tests hip_prefill_hcd_exact_parity)+ # the HC write-back kernel timed at the prompt path's shapes with an output checksum (not a ctest; STRATA_GRW_V picks a variant)+ add_executable(hip_prefill_gr_write_bench tests/hip/prefill_gr_write_bench.cpp)+ target_link_libraries(hip_prefill_gr_write_bench PRIVATE strata_prefill strata_hip_runtime)# S6: the AMD fast router against the portable kernel, bitwiseadd_executable(hip_router_fast tests/hip/router_fast.cpp)target_link_libraries(hip_router_fast PRIVATE strata_kernels strata_hip_runtime)diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 5e62c37f3..88f595bdc 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -2460,12 +2460,12 @@ namespace {// host's expert grouping, or for an expert's copy) lands on the phase that was waiting. Events are reused: the marks// are folded at every MoE layer's host sync, after which all of them have completed.enum PfPhase { kPfStart, kPfHc, kPfGdn, kPfQsa, kPfQsaIdx, kPfQsaSel, kPfQsaAttn, kPfRouter, kPfHostGroup, kPfGather,- kPfWaitCopy, kPfDequant, kPfGemmGU, kPfGemmD, kPfCombine, kPfPle, kPfKvStage, kPfGdnConv, kPfGdnRec, kPfGdnOut,+ kPfWaitCopy, kPfDequant, kPfGemmGU, kPfGemmD, kPfCombine, kPfPle, kPfKvStage, kPfGdnConv, kPfGdnRec, kPfGdnOut, kPfHcWrite,kPfCount };const char* const kPfNames[kPfCount] = {"embed+steps", "hc read", "gdn", "qsa proj", "qsa indexer", "qsa select","qsa attn", "router+shared", "host grouping", "gather", "wait copy", "dequant","gemm gate/up", "gemm down", "combine", "ple", "kv stage", "gdn conv+gates",- "gdn recurrence", "gdn out proj"};+ "gdn recurrence", "gdn out proj", "hc write"};struct PfTimer {bool on = std::getenv("STRATA_PREFILL_TIMING") != nullptr;std::vector<cudaEvent_t> ev;@@ -4740,6 +4740,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri}}}+ pt.mark(kPfHcWrite, cs); // the write-back (and the next half's norm) on its own line; it used to be charged to the phase before// ---- the hyper-connection write of this half; F-2: fused with the next half's norm when nothing else// touches R in between (not the stage's last half, not before the PLE block of layer 1, not under a// control vector)diff --git a/tests/hip/prefill_gr_write_bench.cpp b/tests/hip/prefill_gr_write_bench.cppnew file mode 100644index 000000000..f38c55c3a--- /dev/null+++ b/tests/hip/prefill_gr_write_bench.cpp@@ -0,0 +1,84 @@+// tests/hip/prefill_gr_write_bench.cpp - the hyper-connection write-back (gr_write_norm_rs: R += bo * 2 sigmoid(inj / 4), then the next+// half's norm row scale and BF16 image) timed at the prompt path's shapes, with the bytes it moves, and a checksum of its+// outputs so a variant can be compared bit for bit (run twice with STRATA_GRW_V=0 / 1).+// gr_write_bench [T=16384] [reps=20]+#include "strata/prefill/kernels.hpp"++#include <cuda_runtime.h>++#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <vector>++namespace {+constexpr int N = 2560, HC = 4, D = N * HC;+bool ck(cudaError_t e, const char* what) {+ if (e != cudaSuccess) { std::printf("FAIL: %s: %s\n", what, cudaGetErrorString(e)); return false; }+ return true;+}+uint64_t fnv(const void* p, size_t n, uint64_t h = 1469598103934665603ull) {+ const uint8_t* b = (const uint8_t*) p;+ for (size_t i = 0; i < n; ++i) { h ^= b[i]; h *= 1099511628211ull; }+ return h;+}+} // namespace++int main(int argc, char** argv) {+ const int64_t T = argc > 1 ? std::atoll(argv[1]) : 16384;+ const int reps = argc > 2 ? std::atoi(argv[2]) : 20;+ std::mt19937 rng(7);+ std::normal_distribution<float> nd(0.f, 1.f);+ std::vector<float> R0((size_t) T * D), bo((size_t) T * N), inj((size_t) T * HC), w((size_t) HC * N);+ for (auto& v : R0) v = nd(rng);+ for (auto& v : bo) v = nd(rng);+ for (auto& v : inj) v = nd(rng);+ for (auto& v : w) v = 0.5f + 0.1f * nd(rng);+ float *dR = nullptr, *dR0 = nullptr, *dbo = nullptr, *dinj = nullptr, *dw = nullptr, *drs = nullptr;+ uint16_t* dx = nullptr;+ bool ok = ck(cudaMalloc(&dR, R0.size() * 4), "R") && ck(cudaMalloc(&dR0, R0.size() * 4), "R0") &&+ ck(cudaMalloc(&dbo, bo.size() * 4), "bo") && ck(cudaMalloc(&dinj, inj.size() * 4), "inj") &&+ ck(cudaMalloc(&dw, w.size() * 4), "w") && ck(cudaMalloc(&drs, (size_t) T * HC * 4), "rs") &&+ ck(cudaMalloc(&dx, (size_t) T * D * 2), "xn16");+ if (!ok) return 2;+ cudaMemcpy(dR0, R0.data(), R0.size() * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(dbo, bo.data(), bo.size() * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(dinj, inj.data(), inj.size() * 4, cudaMemcpyHostToDevice);+ cudaMemcpy(dw, w.data(), w.size() * 4, cudaMemcpyHostToDevice);+ cudaStream_t s;+ cudaStreamCreate(&s);+ // one reference pass for the checksum+ cudaMemcpy(dR, dR0, R0.size() * 4, cudaMemcpyDeviceToDevice);+ strata::prefill::gr_write_norm_rs(dR, dbo, dinj, HC, dw, 1e-6f, drs, dx, T, s, nullptr, D, false);+ cudaStreamSynchronize(s);+ std::vector<float> oR(R0.size()), ors((size_t) T * HC);+ std::vector<uint16_t> ox((size_t) T * D);+ cudaMemcpy(oR.data(), dR, oR.size() * 4, cudaMemcpyDeviceToHost);+ cudaMemcpy(ors.data(), drs, ors.size() * 4, cudaMemcpyDeviceToHost);+ cudaMemcpy(ox.data(), dx, ox.size() * 2, cudaMemcpyDeviceToHost);+ std::printf("checksum R %016llx rs %016llx xn16 %016llx\n", (unsigned long long) fnv(oR.data(), oR.size() * 4),+ (unsigned long long) fnv(ors.data(), ors.size() * 4), (unsigned long long) fnv(ox.data(), ox.size() * 2));+ cudaEvent_t e0, e1;+ cudaEventCreate(&e0);+ cudaEventCreate(&e1);+ float best = 1e30f, sum = 0;+ for (int r = 0; r < reps; ++r) {+ cudaMemcpy(dR, dR0, R0.size() * 4, cudaMemcpyDeviceToDevice);+ cudaStreamSynchronize(s);+ cudaEventRecord(e0, s);+ strata::prefill::gr_write_norm_rs(dR, dbo, dinj, HC, dw, 1e-6f, drs, dx, T, s, nullptr, D, false);+ cudaEventRecord(e1, s);+ cudaEventSynchronize(e1);+ float ms = 0;+ cudaEventElapsedTime(&ms, e0, e1);+ best = ms < best ? ms : best;+ sum += ms;+ }+ // bytes: R read + write, bo read (x4 rows, from cache after the first), inj, w, rs, xn16 write+ const double gb = ((double) T * D * 4 * 2 + (double) T * N * 4 + (double) T * D * 2) / 1e9;+ std::printf("gr_write_norm_rs T=%lld: best %.3f ms, mean %.3f ms, %.0f GB/s at best (%.2f GB moved, bo counted once)\n",+ (long long) T, best, sum / reps, gb / (best * 1e-3), gb);+ return 0;+}