Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
native_expert_bench: BENCH_CAP sizes the launch for more possible groups than real ones (what the empty block rows of a verify window cost); bench only
Niko1221b7d5ea2f8286
diff --git a/src/kernels/native_expert_bench.cpp b/src/kernels/native_expert_bench.cppindex 482346127..3263ee7ae 100644--- a/src/kernels/native_expert_bench.cpp+++ b/src/kernels/native_expert_bench.cpp@@ -14,6 +14,7 @@#include <cuda_runtime.h>+#include <algorithm>#include <cmath>#include <cstdint>#include <cstdio>@@ -83,6 +84,8 @@ int main(int argc, char** argv) {std::vector<float> x((size_t) NTOK * H);for (auto& v : x) v = nd(rng);const int NE = G * T;+ // BENCH_CAP=n: the launch is sized for n possible groups (the engine passes n_tokens * K), G of them real: what the empty block rows cost+ const int CAP = std::getenv("BENCH_CAP") ? std::max(G, std::atoi(std::getenv("BENCH_CAP"))) : G, CAPE = std::max(NE, CAP);std::vector<unsigned long long> ptr(G);std::vector<int32_t> start(G + 1), dst(NE), tok(NE);const auto L = K::native_expert_layout(f.gu_type, f.d_type, H, FF);@@ -94,7 +97,7 @@ int main(int argc, char** argv) {cudaMalloc((void**) &dblob, blobs.size());cudaMalloc(&dx, x.size() * 4);cudaMalloc(&dxq, (size_t) NTOK * H / 32 * 36);- cudaMalloc(&dscr, K::native_expert_scratch_bytes(NE, FF));+ cudaMalloc(&dscr, K::native_expert_scratch_bytes(CAPE, FF));cudaMalloc((void**) &dout, (size_t) NE * H * 4);cudaMalloc((void**) &dptr, (size_t) G * 8);cudaMalloc((void**) &dstart, (size_t) (G + 1) * 4);@@ -120,7 +123,7 @@ int main(int argc, char** argv) {K::quantize_q8_1_rows((const float*) dx, NTOK, H, dxq, s);auto run = [&](int m, int phase) {K::native_expert_set_mode(m, phase);- K::native_expert_grouped(L, dptr, dstart, dn, ddst, dtok, G, NE, dxq, dscr, dout, s);+ K::native_expert_grouped(L, dptr, dstart, dn, ddst, dtok, CAP, CAPE, dxq, dscr, dout, s);};std::vector<float> ref((size_t) NE * H), got((size_t) NE * H);cudaMemsetAsync(dout, 0xff, got.size() * 4, s);