Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: engine-kernel bench in ds4_tq_test (--bench)

Niko122188fcf2b4433a
diff --git a/tests/dsv41/tq_expert_test.cpp b/tests/dsv41/tq_expert_test.cpp
index aac351178..eb8ccb718 100644
--- a/tests/dsv41/tq_expert_test.cpp
+++ b/tests/dsv41/tq_expert_test.cpp
@@ -73,10 +73,44 @@ static double dotd(const float* w, const float* x, int k, double* sabs = nullptr
double s = 0, sa = 0; for (int i = 0; i < k; ++i) { const double t = (double) w[i] * x[i]; s += t; sa += std::fabs(t); } if (sabs) *sabs = sa; return s;
}
+// one decode token of the DeepSeek shape (n_embd 5120, n_ff 2304, top-6 experts, distinct blobs) through the token path: gate/up +
+// down launches, hipEvent time per layer; and a 5-entry verify window of one expert group
+static void bench(int type) {
+ const int K = iq::tq_kbits(type), E = 5120, FF = 2304, NE = 6;
+ const size_t rb_g = (size_t) (E / 128) * (2 + 4 * K), rb_d = (size_t) (FF / 128) * (2 + 4 * K);
+ auto al = [](size_t n) { return (n + 255) / 256 * 256; };
+ const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);
+ std::mt19937_64 rng(5);
+ std::vector<uint8_t> one(blob);
+ for (auto& b : one) b = (uint8_t) rng();
+ std::vector<unsigned long long> plan(NE);
+ for (int e = 0; e < NE; ++e) { uint8_t* d; CK(hipMalloc(&d, blob)); CK(hipMemcpy(d, one.data(), blob, hipMemcpyHostToDevice)); plan[e] = (unsigned long long) d; }
+ std::vector<int8_t> xq(E, 3), hq((size_t) NE * FF, 3); std::vector<float> xd(E / 32, 0.01f), hd((size_t) NE * FF / 32, 0.01f);
+ iq::Args a{};
+ a.plan_ptr = up(plan); a.n_embd = E; a.n_ff = FF; a.limit = 10.0f; a.xq = up(xq); a.xd = up(xd); a.hq = up(hq); a.hd = up(hd); a.tg = a.td = type;
+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];
+ float *dh, *dp; CK(hipMalloc(&dh, (size_t) NE * FF * 4)); CK(hipMalloc(&dp, (size_t) NE * E * 4)); a.h = dh; a.part = dp;
+ hipEvent_t e0, e1; hipEventCreate(&e0); hipEventCreate(&e1);
+ for (int rep = 0; rep < 5; ++rep) { iq::gate_up(a, NE, 1, 0); iq::down(a, NE, 1, 0); }
+ CK(hipDeviceSynchronize());
+ float ms_gu = 0, ms_dn = 0;
+ for (int part = 0; part < 2; ++part) {
+ hipEventRecord(e0);
+ for (int rep = 0; rep < 100; ++rep) { if (part == 0) iq::gate_up(a, NE, 1, 0); else iq::down(a, NE, 1, 0); }
+ hipEventRecord(e1); hipEventSynchronize(e1);
+ float ms; hipEventElapsedTime(&ms, e0, e1);
+ (part == 0 ? ms_gu : ms_dn) = ms / 100;
+ }
+ const double bytes = (double) NE * blob;
+ std::printf("bench %-6s token path, 6 experts: gate/up %.0f us, down %.0f us, layer %.0f us (%.0f GB/s, %.2f Tw/s), x40 = %.1f ms per token\n", iq::name(type), ms_gu * 1e3,
+ ms_dn * 1e3, (ms_gu + ms_dn) * 1e3, bytes / ((ms_gu + ms_dn) * 1e-3) / 1e9, (double) NE * 3 * FF * E / ((ms_gu + ms_dn) * 1e-3) / 1e12, 40.0 * (ms_gu + ms_dn));
+}
+
int main(int argc, char** argv) {
std::vector<int> types = {145, 146, 224, 225, 226, 227};
for (int i = 1; i < argc; ++i)
- if (std::string(argv[i]) == "--type" && i + 1 < argc) { types.clear(); for (const char* p = argv[++i]; *p;) { types.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
+ if (std::string(argv[i]) == "--bench") { for (int t : {225, 224, 145, 146}) bench(t); return 0; }
+ else if (std::string(argv[i]) == "--type" && i + 1 < argc) { types.clear(); for (const char* p = argv[++i]; *p;) { types.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
std::mt19937_64 rng(11);
std::normal_distribution<float> nd(0.f, 1.f);
int fails = 0;