Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

native_expert_bench: BENCH_CAP sizes the launch for more possible groups than real ones (what the empty block rows of a verify window cost); bench only

Niko1221b7d5ea2f8286
diff --git a/src/kernels/native_expert_bench.cpp b/src/kernels/native_expert_bench.cpp
index 482346127..3263ee7ae 100644
--- a/src/kernels/native_expert_bench.cpp
+++ b/src/kernels/native_expert_bench.cpp
@@ -14,6 +14,7 @@
#include <cuda_runtime.h>
+#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdio>
@@ -83,6 +84,8 @@ int main(int argc, char** argv) {
std::vector<float> x((size_t) NTOK * H);
for (auto& v : x) v = nd(rng);
const int NE = G * T;
+ // BENCH_CAP=n: the launch is sized for n possible groups (the engine passes n_tokens * K), G of them real: what the empty block rows cost
+ const int CAP = std::getenv("BENCH_CAP") ? std::max(G, std::atoi(std::getenv("BENCH_CAP"))) : G, CAPE = std::max(NE, CAP);
std::vector<unsigned long long> ptr(G);
std::vector<int32_t> start(G + 1), dst(NE), tok(NE);
const auto L = K::native_expert_layout(f.gu_type, f.d_type, H, FF);
@@ -94,7 +97,7 @@ int main(int argc, char** argv) {
cudaMalloc((void**) &dblob, blobs.size());
cudaMalloc(&dx, x.size() * 4);
cudaMalloc(&dxq, (size_t) NTOK * H / 32 * 36);
- cudaMalloc(&dscr, K::native_expert_scratch_bytes(NE, FF));
+ cudaMalloc(&dscr, K::native_expert_scratch_bytes(CAPE, FF));
cudaMalloc((void**) &dout, (size_t) NE * H * 4);
cudaMalloc((void**) &dptr, (size_t) G * 8);
cudaMalloc((void**) &dstart, (size_t) (G + 1) * 4);
@@ -120,7 +123,7 @@ int main(int argc, char** argv) {
K::quantize_q8_1_rows((const float*) dx, NTOK, H, dxq, s);
auto run = [&](int m, int phase) {
K::native_expert_set_mode(m, phase);
- K::native_expert_grouped(L, dptr, dstart, dn, ddst, dtok, G, NE, dxq, dscr, dout, s);
+ K::native_expert_grouped(L, dptr, dstart, dn, ddst, dtok, CAP, CAPE, dxq, dscr, dout, s);
};
std::vector<float> ref((size_t) NE * H), got((size_t) NE * H);
cudaMemsetAsync(dout, 0xff, got.size() * 4, s);