Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: engine-kernel bench in ds4_tq_test (--bench)
Niko122188fcf2b4433a
diff --git a/tests/dsv41/tq_expert_test.cpp b/tests/dsv41/tq_expert_test.cppindex aac351178..eb8ccb718 100644--- a/tests/dsv41/tq_expert_test.cpp+++ b/tests/dsv41/tq_expert_test.cpp@@ -73,10 +73,44 @@ static double dotd(const float* w, const float* x, int k, double* sabs = nullptrdouble s = 0, sa = 0; for (int i = 0; i < k; ++i) { const double t = (double) w[i] * x[i]; s += t; sa += std::fabs(t); } if (sabs) *sabs = sa; return s;}+// one decode token of the DeepSeek shape (n_embd 5120, n_ff 2304, top-6 experts, distinct blobs) through the token path: gate/up ++// down launches, hipEvent time per layer; and a 5-entry verify window of one expert group+static void bench(int type) {+ const int K = iq::tq_kbits(type), E = 5120, FF = 2304, NE = 6;+ const size_t rb_g = (size_t) (E / 128) * (2 + 4 * K), rb_d = (size_t) (FF / 128) * (2 + 4 * K);+ auto al = [](size_t n) { return (n + 255) / 256 * 256; };+ const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);+ std::mt19937_64 rng(5);+ std::vector<uint8_t> one(blob);+ for (auto& b : one) b = (uint8_t) rng();+ std::vector<unsigned long long> plan(NE);+ for (int e = 0; e < NE; ++e) { uint8_t* d; CK(hipMalloc(&d, blob)); CK(hipMemcpy(d, one.data(), blob, hipMemcpyHostToDevice)); plan[e] = (unsigned long long) d; }+ std::vector<int8_t> xq(E, 3), hq((size_t) NE * FF, 3); std::vector<float> xd(E / 32, 0.01f), hd((size_t) NE * FF / 32, 0.01f);+ iq::Args a{};+ a.plan_ptr = up(plan); a.n_embd = E; a.n_ff = FF; a.limit = 10.0f; a.xq = up(xq); a.xd = up(xd); a.hq = up(hq); a.hd = up(hd); a.tg = a.td = type;+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];+ float *dh, *dp; CK(hipMalloc(&dh, (size_t) NE * FF * 4)); CK(hipMalloc(&dp, (size_t) NE * E * 4)); a.h = dh; a.part = dp;+ hipEvent_t e0, e1; hipEventCreate(&e0); hipEventCreate(&e1);+ for (int rep = 0; rep < 5; ++rep) { iq::gate_up(a, NE, 1, 0); iq::down(a, NE, 1, 0); }+ CK(hipDeviceSynchronize());+ float ms_gu = 0, ms_dn = 0;+ for (int part = 0; part < 2; ++part) {+ hipEventRecord(e0);+ for (int rep = 0; rep < 100; ++rep) { if (part == 0) iq::gate_up(a, NE, 1, 0); else iq::down(a, NE, 1, 0); }+ hipEventRecord(e1); hipEventSynchronize(e1);+ float ms; hipEventElapsedTime(&ms, e0, e1);+ (part == 0 ? ms_gu : ms_dn) = ms / 100;+ }+ const double bytes = (double) NE * blob;+ std::printf("bench %-6s token path, 6 experts: gate/up %.0f us, down %.0f us, layer %.0f us (%.0f GB/s, %.2f Tw/s), x40 = %.1f ms per token\n", iq::name(type), ms_gu * 1e3,+ ms_dn * 1e3, (ms_gu + ms_dn) * 1e3, bytes / ((ms_gu + ms_dn) * 1e-3) / 1e9, (double) NE * 3 * FF * E / ((ms_gu + ms_dn) * 1e-3) / 1e12, 40.0 * (ms_gu + ms_dn));+}+int main(int argc, char** argv) {std::vector<int> types = {145, 146, 224, 225, 226, 227};for (int i = 1; i < argc; ++i)- if (std::string(argv[i]) == "--type" && i + 1 < argc) { types.clear(); for (const char* p = argv[++i]; *p;) { types.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }+ if (std::string(argv[i]) == "--bench") { for (int t : {225, 224, 145, 146}) bench(t); return 0; }+ else if (std::string(argv[i]) == "--type" && i + 1 < argc) { types.clear(); for (const char* p = argv[++i]; *p;) { types.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }std::mt19937_64 rng(11);std::normal_distribution<float> nd(0.f, 1.f);int fails = 0;