Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

prefill timing: the hyper-connection write-back on its own line (it was charged to the phase before it, e.g. the wait for the first chunk's PLE rows); hip_prefill_gr_write_bench

Niko1221aef32e224b32
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9591fe3f8..82249d4b3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -817,6 +817,9 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
add_test(NAME hip_prefill_hcd_exact_parity COMMAND hip_prefill_hcd_exact_parity)
set_tests_properties(hip_prefill_hcd_exact_parity PROPERTIES SKIP_RETURN_CODE 77)
list(APPEND _strata_hip_tests hip_prefill_hcd_exact_parity)
+ # the HC write-back kernel timed at the prompt path's shapes with an output checksum (not a ctest; STRATA_GRW_V picks a variant)
+ add_executable(hip_prefill_gr_write_bench tests/hip/prefill_gr_write_bench.cpp)
+ target_link_libraries(hip_prefill_gr_write_bench PRIVATE strata_prefill strata_hip_runtime)
# S6: the AMD fast router against the portable kernel, bitwise
add_executable(hip_router_fast tests/hip/router_fast.cpp)
target_link_libraries(hip_router_fast PRIVATE strata_kernels strata_hip_runtime)
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index 5e62c37f3..88f595bdc 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -2460,12 +2460,12 @@ namespace {
// host's expert grouping, or for an expert's copy) lands on the phase that was waiting. Events are reused: the marks
// are folded at every MoE layer's host sync, after which all of them have completed.
enum PfPhase { kPfStart, kPfHc, kPfGdn, kPfQsa, kPfQsaIdx, kPfQsaSel, kPfQsaAttn, kPfRouter, kPfHostGroup, kPfGather,
- kPfWaitCopy, kPfDequant, kPfGemmGU, kPfGemmD, kPfCombine, kPfPle, kPfKvStage, kPfGdnConv, kPfGdnRec, kPfGdnOut,
+ kPfWaitCopy, kPfDequant, kPfGemmGU, kPfGemmD, kPfCombine, kPfPle, kPfKvStage, kPfGdnConv, kPfGdnRec, kPfGdnOut, kPfHcWrite,
kPfCount };
const char* const kPfNames[kPfCount] = {"embed+steps", "hc read", "gdn", "qsa proj", "qsa indexer", "qsa select",
"qsa attn", "router+shared", "host grouping", "gather", "wait copy", "dequant",
"gemm gate/up", "gemm down", "combine", "ple", "kv stage", "gdn conv+gates",
- "gdn recurrence", "gdn out proj"};
+ "gdn recurrence", "gdn out proj", "hc write"};
struct PfTimer {
bool on = std::getenv("STRATA_PREFILL_TIMING") != nullptr;
std::vector<cudaEvent_t> ev;
@@ -4740,6 +4740,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
}
}
}
+ pt.mark(kPfHcWrite, cs); // the write-back (and the next half's norm) on its own line; it used to be charged to the phase before
// ---- the hyper-connection write of this half; F-2: fused with the next half's norm when nothing else
// touches R in between (not the stage's last half, not before the PLE block of layer 1, not under a
// control vector)
diff --git a/tests/hip/prefill_gr_write_bench.cpp b/tests/hip/prefill_gr_write_bench.cpp
new file mode 100644
index 000000000..f38c55c3a
--- /dev/null
+++ b/tests/hip/prefill_gr_write_bench.cpp
@@ -0,0 +1,84 @@
+// tests/hip/prefill_gr_write_bench.cpp - the hyper-connection write-back (gr_write_norm_rs: R += bo * 2 sigmoid(inj / 4), then the next
+// half's norm row scale and BF16 image) timed at the prompt path's shapes, with the bytes it moves, and a checksum of its
+// outputs so a variant can be compared bit for bit (run twice with STRATA_GRW_V=0 / 1).
+// gr_write_bench [T=16384] [reps=20]
+#include "strata/prefill/kernels.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <vector>
+
+namespace {
+constexpr int N = 2560, HC = 4, D = N * HC;
+bool ck(cudaError_t e, const char* what) {
+ if (e != cudaSuccess) { std::printf("FAIL: %s: %s\n", what, cudaGetErrorString(e)); return false; }
+ return true;
+}
+uint64_t fnv(const void* p, size_t n, uint64_t h = 1469598103934665603ull) {
+ const uint8_t* b = (const uint8_t*) p;
+ for (size_t i = 0; i < n; ++i) { h ^= b[i]; h *= 1099511628211ull; }
+ return h;
+}
+} // namespace
+
+int main(int argc, char** argv) {
+ const int64_t T = argc > 1 ? std::atoll(argv[1]) : 16384;
+ const int reps = argc > 2 ? std::atoi(argv[2]) : 20;
+ std::mt19937 rng(7);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ std::vector<float> R0((size_t) T * D), bo((size_t) T * N), inj((size_t) T * HC), w((size_t) HC * N);
+ for (auto& v : R0) v = nd(rng);
+ for (auto& v : bo) v = nd(rng);
+ for (auto& v : inj) v = nd(rng);
+ for (auto& v : w) v = 0.5f + 0.1f * nd(rng);
+ float *dR = nullptr, *dR0 = nullptr, *dbo = nullptr, *dinj = nullptr, *dw = nullptr, *drs = nullptr;
+ uint16_t* dx = nullptr;
+ bool ok = ck(cudaMalloc(&dR, R0.size() * 4), "R") && ck(cudaMalloc(&dR0, R0.size() * 4), "R0") &&
+ ck(cudaMalloc(&dbo, bo.size() * 4), "bo") && ck(cudaMalloc(&dinj, inj.size() * 4), "inj") &&
+ ck(cudaMalloc(&dw, w.size() * 4), "w") && ck(cudaMalloc(&drs, (size_t) T * HC * 4), "rs") &&
+ ck(cudaMalloc(&dx, (size_t) T * D * 2), "xn16");
+ if (!ok) return 2;
+ cudaMemcpy(dR0, R0.data(), R0.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(dbo, bo.data(), bo.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(dinj, inj.data(), inj.size() * 4, cudaMemcpyHostToDevice);
+ cudaMemcpy(dw, w.data(), w.size() * 4, cudaMemcpyHostToDevice);
+ cudaStream_t s;
+ cudaStreamCreate(&s);
+ // one reference pass for the checksum
+ cudaMemcpy(dR, dR0, R0.size() * 4, cudaMemcpyDeviceToDevice);
+ strata::prefill::gr_write_norm_rs(dR, dbo, dinj, HC, dw, 1e-6f, drs, dx, T, s, nullptr, D, false);
+ cudaStreamSynchronize(s);
+ std::vector<float> oR(R0.size()), ors((size_t) T * HC);
+ std::vector<uint16_t> ox((size_t) T * D);
+ cudaMemcpy(oR.data(), dR, oR.size() * 4, cudaMemcpyDeviceToHost);
+ cudaMemcpy(ors.data(), drs, ors.size() * 4, cudaMemcpyDeviceToHost);
+ cudaMemcpy(ox.data(), dx, ox.size() * 2, cudaMemcpyDeviceToHost);
+ std::printf("checksum R %016llx rs %016llx xn16 %016llx\n", (unsigned long long) fnv(oR.data(), oR.size() * 4),
+ (unsigned long long) fnv(ors.data(), ors.size() * 4), (unsigned long long) fnv(ox.data(), ox.size() * 2));
+ cudaEvent_t e0, e1;
+ cudaEventCreate(&e0);
+ cudaEventCreate(&e1);
+ float best = 1e30f, sum = 0;
+ for (int r = 0; r < reps; ++r) {
+ cudaMemcpy(dR, dR0, R0.size() * 4, cudaMemcpyDeviceToDevice);
+ cudaStreamSynchronize(s);
+ cudaEventRecord(e0, s);
+ strata::prefill::gr_write_norm_rs(dR, dbo, dinj, HC, dw, 1e-6f, drs, dx, T, s, nullptr, D, false);
+ cudaEventRecord(e1, s);
+ cudaEventSynchronize(e1);
+ float ms = 0;
+ cudaEventElapsedTime(&ms, e0, e1);
+ best = ms < best ? ms : best;
+ sum += ms;
+ }
+ // bytes: R read + write, bo read (x4 rows, from cache after the first), inj, w, rs, xn16 write
+ const double gb = ((double) T * D * 4 * 2 + (double) T * N * 4 + (double) T * D * 2) / 1e9;
+ std::printf("gr_write_norm_rs T=%lld: best %.3f ms, mean %.3f ms, %.0f GB/s at best (%.2f GB moved, bo counted once)\n",
+ (long long) T, best, sum / reps, gb / (best * 1e-3), gb);
+ return 0;
+}