Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

mmvq_il_parity: the L2-residency arms (--hot / --gap / --persist / --ab), the clock warm-up and MMVQ_IL_TMIN (#3779, #3712)

linxuhaoc2d3fe500075
diff --git a/src/kernels/mmvq_il_parity.cpp b/src/kernels/mmvq_il_parity.cpp
index 9e78015a2..415baa4be 100644
--- a/src/kernels/mmvq_il_parity.cpp
+++ b/src/kernels/mmvq_il_parity.cpp
@@ -1,7 +1,15 @@
// src/kernels/mmvq_il_parity.cpp - native_mmvq_il (2-4 columns from the interleaved q8_1 copy, fork F4) against
// native_mmvq's multi-column kernels, bitwise, for every rows-a-warp choice; --bench times both per call.
//
-// build/mmvq_il_parity [--bench [--emit-table]]
+// build/mmvq_il_parity [--bench [--emit-table] [--hot] [--gap MB] [--persist] [--ab]]
+//
+// --hot times one copy of the weights (they sit in L2 after the first call: the ceiling of an L2
+// prefetch); --gap MB reads MB megabytes of OTHER weights (native_mmvq over other copies) before every timed call, as
+// the window's GEMVs do between a prefetch and its consumer (a memset gap was tried first: its write-backs drain
+// during the timed call and make every arm slower than cold); --persist puts the timed weights in a persisting L2
+// set-aside (cudaLimitPersistingL2CacheSize + an access-policy window on the stream) so the gap's reads stream past
+// them. --ab runs the arms cold / hot / hot+gap / hot+gap+persist / cold per case in ONE process (the clocks ramp
+// over the first ~100 ms of a run, so arms in separate runs are not comparable). MMVQ_IL_TMIN / MMVQ_IL_TMAX bound T.
//
// --emit-table (with --bench) prints the rows-a-warp table this card measured, in the source form of kIlRows in
// native_mmvq.cu: per cell the rows count (1/2/4) that takes at least 3% off native_mmvq's time, else 0.
@@ -29,6 +37,58 @@ using namespace strata::kernels;
struct Cell { int type, T, n_out; double ref; double il[5]; };
std::vector<Cell> g_cells;
+bool g_hot = false, g_persist = false, g_ab = false;
+std::size_t g_gap_bytes = 0;
+std::size_t g_gap_now = 0; // the gap in force for the current arm (--ab) or g_gap_bytes
+// time `reps` calls of `fn` on `s`; with --gap each call is preceded by `gap` (other weights' reads) and timed on its own
+template <class F, class G>
+double time_us(F fn, int reps, cudaStream_t s, G gap) {
+ cudaEvent_t e0, e1;
+ (void) cudaEventCreate(&e0);
+ (void) cudaEventCreate(&e1);
+ float total = 0;
+ for (int rep = 0; rep < 2; ++rep) { // rep 0 warms, rep 1 times
+ total = 0;
+ if (g_gap_now == 0) {
+ (void) cudaEventRecord(e0, s);
+ for (int i = 0; i < reps; ++i) fn(i);
+ (void) cudaEventRecord(e1, s);
+ (void) cudaEventSynchronize(e1);
+ (void) cudaEventElapsedTime(&total, e0, e1);
+ } else {
+ for (int i = 0; i < reps; ++i) {
+ gap();
+ (void) cudaEventRecord(e0, s);
+ fn(i);
+ (void) cudaEventRecord(e1, s);
+ (void) cudaEventSynchronize(e1);
+ float ms = 0;
+ (void) cudaEventElapsedTime(&ms, e0, e1);
+ total += ms;
+ }
+ }
+ }
+ (void) cudaEventDestroy(e0);
+ (void) cudaEventDestroy(e1);
+ return 1000.0 * total / reps;
+}
+// --persist: the matrix `w` (wbytes) persisting in L2 for every kernel on `s`; off: the window cleared
+void persist_window(cudaStream_t s, void* w, std::size_t wbytes, bool on) {
+#if !defined(STRATA_HIP_GFX906)
+ cudaStreamAttrValue v{};
+ v.accessPolicyWindow.base_ptr = on ? w : nullptr;
+ v.accessPolicyWindow.num_bytes = on ? wbytes : 0;
+ v.accessPolicyWindow.hitRatio = 1.0f;
+ v.accessPolicyWindow.hitProp = cudaAccessPropertyPersisting;
+ v.accessPolicyWindow.missProp = cudaAccessPropertyStreaming;
+ if (cudaStreamSetAttribute(s, cudaStreamAttributeAccessPolicyWindow, &v) != cudaSuccess) {
+ std::printf("--persist: the access policy window was refused: %s\n", cudaGetErrorString(cudaGetLastError()));
+ }
+#else
+ (void) s; (void) w; (void) wbytes; // (no L2 access policy window in the gfx906 compat layer)
+ std::printf("--persist: not available in this build (%s)\n", on ? "on" : "off");
+#endif
+}
struct Case {
const char* name;
int type, n_in, n_out, block_elems, block_bytes, scale_at[2];
@@ -98,15 +158,20 @@ int run_case(const Case& c, bool bench, cudaStream_t s) {
const uint16_t h = sane_half(rng);
std::memcpy(&w[k * (std::size_t) c.block_bytes + (std::size_t) at], &h, 2);
}
- const int copies = bench ? (int) std::max<std::size_t>(1, std::min<std::size_t>(24, (192u << 20) / wbytes)) : 1;
+ // --hot: one copy (timed hot); with --gap, 1 + the copies the gap reads (dw[1..]) - the timed call always reads dw[0]
+ const int gap_copies = g_gap_bytes > 0 ? (int) std::min<std::size_t>(23, (g_gap_bytes + wbytes - 1) / wbytes) : 0;
+ const int copies = bench && (!g_hot || g_ab) ? (int) std::max<std::size_t>(1, std::min<std::size_t>(24, (192u << 20) / wbytes)) : 1 + gap_copies;
std::vector<void*> dw(copies, nullptr);
for (int i = 0; i < copies; ++i) {
if (!ck(cudaMalloc(&dw[i], wbytes), "malloc w") || !ck(cudaMemcpy(dw[i], w.data(), wbytes, cudaMemcpyHostToDevice), "copy w"))
return 1;
}
int bad = 0;
+ if (bench && g_persist && !g_ab) persist_window(s, dw[0], wbytes, true);
+ g_gap_now = g_gap_bytes;
const int t_max = std::getenv("MMVQ_IL_TMAX") ? std::atoi(std::getenv("MMVQ_IL_TMAX")) : 8; // 147-verify: 5-8 columns
- for (int T = 1; T <= t_max; ++T) {
+ const int t_min = std::getenv("MMVQ_IL_TMIN") ? std::atoi(std::getenv("MMVQ_IL_TMIN")) : 1;
+ for (int T = t_min; T <= t_max; ++T) {
std::vector<float> x((std::size_t) T * c.n_in);
std::normal_distribution<float> nd(0.f, 1.f);
for (auto& v : x) v = nd(rng);
@@ -124,10 +189,25 @@ int run_case(const Case& c, bool bench, cudaStream_t s) {
if (T >= 2) native_q8_1_interleave(xq, xil, c.n_in, T, s);
native_mmvq(c.type, dw[0], xq, yref, c.n_in, c.n_out, T, s);
ck(cudaStreamSynchronize(s), "ref");
+ auto gap = [&]() { // --gap: other weights' reads between the timed calls (the window's traffic)
+ for (int j = 1; j <= gap_copies; ++j) native_mmvq(c.type, dw[j], xq, yref, c.n_in, c.n_out, T, s);
+ };
std::vector<uint32_t> a((std::size_t) T * c.n_out), b(a.size());
ck(cudaMemcpy(a.data(), yref, ybytes, cudaMemcpyDeviceToHost), "read ref");
double best_ref = 0, best_il[5] = {0, 0, 0, 0, 0};
int table_r = 0;
+ struct Arm { const char* name; bool hot; std::size_t gap; bool persist; };
+ const Arm arms_ab[] = {{"cold", false, 0, false}, {"hot", true, 0, false}, {"hot+gap", true, g_gap_bytes, false},
+ {"hot+gap+persist", true, g_gap_bytes, true}, {"cold2", false, 0, false}};
+ const Arm arms_one[] = {{g_hot ? "hot" : "cold", g_hot, g_gap_bytes, g_persist}};
+ const Arm* arms = g_ab ? arms_ab : arms_one;
+ const int n_arms = g_ab ? 5 : 1;
+ for (int ai = 0; ai < (bench ? n_arms : 1); ++ai) {
+ if (bench) {
+ g_hot = arms[ai].hot;
+ g_gap_now = arms[ai].gap;
+ if (arms[ai].persist != g_persist) { g_persist = arms[ai].persist; persist_window(s, dw[0], wbytes, g_persist); }
+ }
for (int r : {0, 1, 2, 4}) {
native_mmvq_il_tune(r);
cudaMemset(yil, 0xff, ybytes);
@@ -148,45 +228,21 @@ int run_case(const Case& c, bool bench, cudaStream_t s) {
}
(void) used;
if (bench && r != 0) {
- cudaEvent_t e0, e1;
- cudaEventCreate(&e0);
- cudaEventCreate(&e1);
- const int reps = 200;
- for (int rep = 0; rep < 2; ++rep) { // rep 0 warms, rep 1 times
- cudaEventRecord(e0, s);
- for (int i = 0; i < reps; ++i) native_mmvq_il(c.type, dw[i % copies], xq, xil, yil, c.n_in, c.n_out, T, s);
- cudaEventRecord(e1, s);
- cudaEventSynchronize(e1);
- }
- float ms;
- cudaEventElapsedTime(&ms, e0, e1);
- best_il[r] = 1000.0 * ms / reps;
- cudaEventDestroy(e0);
- cudaEventDestroy(e1);
+ best_il[r] = time_us([&](int i) { native_mmvq_il(c.type, dw[g_hot ? 0 : i % copies], xq, xil, yil, c.n_in, c.n_out, T, s); }, 200, s, gap);
}
}
native_mmvq_il_tune(0);
if (bench) {
- cudaEvent_t e0, e1;
- cudaEventCreate(&e0);
- cudaEventCreate(&e1);
- const int reps = 200;
- for (int rep = 0; rep < 2; ++rep) {
- cudaEventRecord(e0, s);
- for (int i = 0; i < reps; ++i) native_mmvq(c.type, dw[i % copies], xq, yref, c.n_in, c.n_out, T, s);
- cudaEventRecord(e1, s);
- cudaEventSynchronize(e1);
- }
- float ms;
- cudaEventElapsedTime(&ms, e0, e1);
- best_ref = 1000.0 * ms / reps;
+ best_ref = time_us([&](int i) { native_mmvq(c.type, dw[g_hot ? 0 : i % copies], xq, yref, c.n_in, c.n_out, T, s); }, 200, s, gap);
(void) table_r;
g_cells.push_back({c.type, T, c.n_out, best_ref, {0, best_il[1], best_il[2], 0, best_il[4]}});
- std::printf("BENCH %-26s T=%d multi %.1f us | il rows1 %.1f rows2 %.1f rows4 %.1f us\n", c.name, T, best_ref,
- best_il[1], best_il[2], best_il[4]);
+ std::printf("BENCH %-26s T=%d %-16s multi %.1f us | il rows1 %.1f rows2 %.1f rows4 %.1f us\n", c.name, T,
+ arms[ai].name, best_ref, best_il[1], best_il[2], best_il[4]);
}
+ } // arms
cudaFree(dx); cudaFree(xq); cudaFree(xil); cudaFree(yref); cudaFree(yil);
}
+ if (bench && g_persist) persist_window(s, dw[0], wbytes, false);
for (void* p : dw) cudaFree(p);
return bad;
}
@@ -298,10 +354,43 @@ int check_tables() {
int main(int argc, char** argv) {
const bool bench = argc > 1 && std::string(argv[1]) == "--bench";
const bool emit = bench && argc > 2 && std::string(argv[2]) == "--emit-table";
+ for (int i = 2; i < argc; ++i) {
+ const std::string a = argv[i];
+ if (a == "--hot") g_hot = true;
+ else if (a == "--ab") g_ab = true;
+ else if (a == "--persist") g_persist = true;
+ else if (a == "--gap" && i + 1 < argc) g_gap_bytes = (std::size_t) std::atoi(argv[++i]) << 20;
+ }
cudaStream_t s;
cudaStreamCreate(&s);
+#if !defined(STRATA_HIP_GFX906)
+ if (g_persist || g_ab) {
+ int dev = 0, l2 = 0, maxp = 0, maxw = 0;
+ cudaGetDevice(&dev);
+ cudaDeviceGetAttribute(&l2, cudaDevAttrL2CacheSize, dev);
+ cudaDeviceGetAttribute(&maxp, cudaDevAttrMaxPersistingL2CacheSize, dev);
+ cudaDeviceGetAttribute(&maxw, cudaDevAttrMaxAccessPolicyWindowSize, dev);
+ std::size_t want = std::min<std::size_t>(20u << 20, (std::size_t) maxp), got = 0;
+ const cudaError_t e = cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, want);
+ cudaDeviceGetLimit(&got, cudaLimitPersistingL2CacheSize);
+ std::printf("--persist: L2 %d MiB, max persisting %d MiB, max window %d MiB; set-aside asked %zu MiB -> %s, granted %zu MiB\n",
+ l2 >> 20, maxp >> 20, maxw >> 20, want >> 20, cudaGetErrorString(e), got >> 20);
+ }
+#endif
+ if (bench) std::printf("bench arms: %s%s%s\n", g_hot ? "hot (one copy) " : "cold (192 MB rotation) ",
+ g_gap_bytes ? ("gap " + std::to_string(g_gap_bytes >> 20) + " MB ").c_str() : "", g_persist ? "persist" : "");
int bad = 0;
bad += check_tables();
+ if (bench) { // ~150 ms of launches so the clocks are up before the first timed case
+ void* wu = nullptr; float* yu = nullptr; void* xu = nullptr;
+ if (cudaMalloc(&wu, 6144u * 2560u) == cudaSuccess && cudaMalloc((void**) &yu, 2560 * 4) == cudaSuccess &&
+ cudaMalloc(&xu, native_q8_1_bytes(6144, 1)) == cudaSuccess) {
+ (void) cudaMemset(wu, 0, 6144u * 2560u); (void) cudaMemset(xu, 0, native_q8_1_bytes(6144, 1));
+ for (int i = 0; i < 6000; ++i) native_mmvq(12, wu, xu, yu, 6144, 2560, 1, s);
+ (void) cudaStreamSynchronize(s);
+ (void) cudaFree(wu); (void) cudaFree(yu); (void) cudaFree(xu);
+ }
+ }
const char* only = std::getenv("MMVQ_IL_ONLY"); // 143-dense: only the cases whose name contains this
for (const Case& c : CASES)
if (only == nullptr || std::strstr(c.name, only) != nullptr) bad += run_case(c, bench, s);