Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 iq test: --bench-big (>=640 MiB working set, rotated experts: DRAM rates), --dump/--cmp (bit-exact golden compare incl. real shapes and grouped path)
Niko122195994c45c581
diff --git a/tests/dsv41/iq_expert_test.cpp b/tests/dsv41/iq_expert_test.cppindex 260548fc0..32d6243ea 100644--- a/tests/dsv41/iq_expert_test.cpp+++ b/tests/dsv41/iq_expert_test.cpp@@ -144,15 +144,125 @@ static void bench(int type) {(double) NE * 3 * FF * E / ((ms_gu + ms_dn) * 1e-3) / 1e12, 40.0 * (ms_gu + ms_dn));}++// --dump F / --cmp F: every device output of a run as raw bits, to a file / against one (a kernel change that keeps the sums+// exact and the float order must leave all of them identical)+static FILE* g_dump = nullptr; static FILE* g_cmp = nullptr; static long g_cmp_bad = 0, g_cmp_n = 0;+static void bits(const char* what, int type, const void* p, size_t bytes) {+ if (!g_dump && !g_cmp) return;+ if (g_dump) { std::fwrite(p, 1, bytes, g_dump); return; }+ std::vector<uint8_t> ref(bytes);+ if (std::fread(ref.data(), 1, bytes, g_cmp) != bytes) { std::printf("cmp: golden file too short at %s %s\n", iq::name(type), what); ++g_cmp_bad; return; }+ ++g_cmp_n;+ if (std::memcmp(ref.data(), p, bytes) != 0) {+ size_t bad = 0, first = 0; for (size_t i = 0; i < bytes; ++i) if (ref[i] != ((const uint8_t*) p)[i]) { if (!bad) first = i; ++bad; }+ std::printf("%-7s %-22s BITS DIFFER: %zu of %zu bytes (first at %zu)\n", iq::name(type), what, bad, bytes, first);+ ++g_cmp_bad;+ }+}+// the real shapes (n_embd 5120, n_ff 2304): the plan path (6 groups of one entry) and the grouped path (windows of 3 and 2)+static void real_shape_bits(int type, std::mt19937_64& rng) {+ const int E = 5120, FF = 2304, NG = 2, NEnt = 5;+ const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);+ auto al = [](size_t n) { return (n + 255) / 256 * 256; };+ const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);+ std::vector<uint8_t> hb((size_t) NG * blob);+ for (auto& b : hb) b = (uint8_t) rng();+ for (size_t b = 0; b < hb.size() / (size_t) iq::block_bytes(type); ++b) { // sane F16 scales (the same placement as make_mat)+ const uint16_t d = f2h(0.02f + 0.0002f * (float) (rng() % 600));+ uint8_t* p = &hb[b * (size_t) iq::block_bytes(type)];+ if (type == 29) { p[49] = (uint8_t) ((p[49] & 0x0f) | ((d & 0x000f) << 4)); p[51] = (uint8_t) ((p[51] & 0x0f) | (d & 0x00f0)); p[53] = (uint8_t) ((p[53] & 0x0f) | ((d & 0x0f00) >> 4)); p[55] = (uint8_t) ((p[55] & 0x0f) | ((d & 0xf000) >> 8)); }+ else std::memcpy(p, &d, 2);+ }+ uint8_t* dblob = up(hb);+ std::vector<int8_t> xq(E), hq((size_t) NEnt * FF); std::vector<float> xd(E / 32), hd((size_t) NEnt * FF / 32);+ for (auto& v : xq) v = (int8_t) ((int) (rng() % 255) - 127);+ for (auto& v : hq) v = (int8_t) ((int) (rng() % 255) - 127);+ for (auto& v : xd) v = 0.002f + 0.0001f * (float) (rng() % 100);+ for (auto& v : hd) v = 0.002f + 0.0001f * (float) (rng() % 100);+ iq::Args a{};+ a.n_embd = E; a.n_ff = FF; a.limit = 10.0f; a.xq = up(xq); a.xd = up(xd); a.hq = up(hq); a.hd = up(hd); a.tg = a.td = type;+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];+ float *dh, *dp; CK(hipMalloc(&dh, (size_t) NEnt * FF * 4)); CK(hipMalloc(&dp, (size_t) NEnt * E * 4)); a.h = dh; a.part = dp;+ std::vector<unsigned long long> plan = {(unsigned long long) dblob, (unsigned long long) (dblob + blob), 0ull, (unsigned long long) dblob};+ a.plan_ptr = up(plan);+ CK(hipMemset(dh, 0, (size_t) NEnt * FF * 4)); CK(hipMemset(dp, 0, (size_t) NEnt * E * 4));+ if (!iq::gate_up(a, 4, 1, 0) || !iq::down(a, 4, 1, 0)) { std::printf("real shape: refused\n"); return; }+ CK(hipDeviceSynchronize());+ { auto h = down_(dh, (size_t) NEnt * FF); bits("real plan h", type, h.data(), h.size() * 4); auto p = down_(dp, (size_t) NEnt * E); bits("real plan part", type, p.data(), p.size() * 4); }+ std::vector<unsigned long long> gp = {(unsigned long long) dblob, (unsigned long long) (dblob + blob)};+ std::vector<int> gs = {0, 3, 5}, ge = {0, 1, 2, 3, 4}, ng = {NG};+ a.plan_ptr = nullptr; a.grp_ptr = up(gp); a.grp_start = up(gs); a.grp_ent = up(ge); a.n_groups = up(ng);+ CK(hipMemset(dh, 0, (size_t) NEnt * FF * 4)); CK(hipMemset(dp, 0, (size_t) NEnt * E * 4));+ if (!iq::gate_up(a, NG, 3, 0) || !iq::down(a, NG, 3, 0)) { std::printf("real shape grouped: refused\n"); return; }+ CK(hipDeviceSynchronize());+ { auto h = down_(dh, (size_t) NEnt * FF); bits("real grp h", type, h.data(), h.size() * 4); auto p = down_(dp, (size_t) NEnt * E); bits("real grp part", type, p.data(), p.size() * 4); }+ CK(hipFree(dblob));+}++// the same DeepSeek decode token over a working set far past L2 (>= 640 MiB of distinct expert blobs, rotated launch by launch),+// so the rate is DRAM's, not L2's: gate/up and down timed apart; ne = entries a group (1: the decode token)+static void bench_big(int type, int mib) {+ const int E = 5120, FF = 2304, NE = 6;+ const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);+ auto al = [](size_t n) { return (n + 255) / 256 * 256; };+ const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);+ const int L = std::max(2, (int) (((size_t) mib << 20) / (NE * blob)) + 1); // layers rotated+ std::mt19937_64 rng(5);+ std::vector<uint8_t> one(blob);+ for (auto& b : one) b = (uint8_t) rng();+ std::vector<unsigned long long> plan((size_t) L * NE);+ for (auto& p : plan) { uint8_t* d; CK(hipMalloc(&d, blob)); CK(hipMemcpy(d, one.data(), blob, hipMemcpyHostToDevice)); p = (unsigned long long) d; }+ std::vector<int8_t> xq(E, 3), hq((size_t) NE * FF, 3); std::vector<float> xd(E / 32, 0.01f), hd((size_t) NE * FF / 32, 0.01f);+ for (int i = 0; i < E; ++i) xq[i] = (int8_t) ((int) (rng() % 255) - 127);+ for (size_t i = 0; i < hq.size(); ++i) hq[i] = (int8_t) ((int) (rng() % 255) - 127);+ unsigned long long* dplan = up(plan);+ iq::Args a{};+ a.n_embd = E; a.n_ff = FF; a.limit = 10.0f; a.xq = up(xq); a.xd = up(xd); a.hq = up(hq); a.hd = up(hd); a.tg = a.td = type;+ for (int r = 0; r < 3; ++r) a.tr[r] = off[r];+ float *dh, *dp; CK(hipMalloc(&dh, (size_t) NE * FF * 4)); CK(hipMalloc(&dp, (size_t) NE * E * 4)); a.h = dh; a.part = dp;+ hipEvent_t e0, e1; hipEventCreate(&e0); hipEventCreate(&e1);+ const int REPS = 3 * L;+ double us[2] = {0, 0};+ for (int part = 0; part < 2; ++part) {+ for (int pass = 0; pass < 2; ++pass) { // pass 0 warms up+ hipEventRecord(e0);+ for (int rep = 0; rep < REPS; ++rep) {+ a.plan_ptr = dplan + (size_t) (rep % L) * NE;+ if (part == 0) iq::gate_up(a, NE, 1, 0); else iq::down(a, NE, 1, 0);+ }+ hipEventRecord(e1); hipEventSynchronize(e1);+ float ms; hipEventElapsedTime(&ms, e0, e1);+ if (pass) us[part] = ms * 1e3 / REPS;+ }+ }+ const double bg = (double) NE * (off[2]) , bd = (double) NE * al((size_t) E * rb_d); // bytes read a launch+ std::printf("big %-7s ws %4.0f MiB: gate/up %6.1f us (%5.0f GB/s) down %6.1f us (%5.0f GB/s) layer %6.1f us -> %5.0f GB/s, %.2f Tw/s\n",+ iq::name(type), (double) L * NE * blob / 1048576.0, us[0], bg / us[0] * 1e-3, us[1], bd / us[1] * 1e-3, us[0] + us[1],+ (bg + bd) / (us[0] + us[1]) * 1e-3, (double) NE * 3 * FF * E / ((us[0] + us[1]) * 1e-6) / 1e12);+ for (auto p : plan) hipFree((void*) p);+}+int main(int argc, char** argv) {std::vector<int> types = {16, 17, 22, 18, 21, 23, 29};for (int i = 1; i < argc; ++i) {- if (std::string(argv[i]) == "--bench") {+ if (std::string(argv[i]) == "--bench-big") { // [types] [MiB]+ std::vector<int> bt; int mib = 640;+ if (i + 1 < argc && argv[i + 1][0] != '-') { for (const char* p = argv[++i]; *p;) { bt.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }+ else bt = types;+ if (i + 1 < argc && argv[i + 1][0] != '-') mib = std::atoi(argv[++i]);+ for (int t : bt) bench_big(t, mib);+ return 0;+ } else if (std::string(argv[i]) == "--bench") {std::vector<int> bt;if (i + 1 < argc && argv[i + 1][0] != '-') { for (const char* p = argv[++i]; *p;) { bt.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }else bt = types;for (int t : bt) bench(t);return 0;+ } else if (std::string(argv[i]) == "--dump" && i + 1 < argc) {+ g_dump = std::fopen(argv[++i], "wb"); if (!g_dump) { std::printf("cannot write %s\n", argv[i]); return 2; }+ } else if (std::string(argv[i]) == "--cmp" && i + 1 < argc) {+ g_cmp = std::fopen(argv[++i], "rb"); if (!g_cmp) { std::printf("cannot read %s\n", argv[i]); return 2; }} else if (std::string(argv[i]) == "--type" && i + 1 < argc) {types.clear(); for (const char* p = argv[++i]; *p;) { types.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; }} else if (std::string(argv[i]) == "--iq256" && i + 1 < argc) {@@ -218,6 +328,7 @@ int main(int argc, char** argv) {if (!iq::gate_up(a, NEXP, 1, 0)) { std::printf("gate_up refused\n"); return 1; }CK(hipDeviceSynchronize());const auto h = down_(dh, (size_t) NEXP * FF);+ bits("token h", type, h.data(), h.size() * 4);double worst = 0;for (int e = 0; e < NEXP; ++e)for (int r = 0; r < FF; ++r) {@@ -233,6 +344,7 @@ int main(int argc, char** argv) {if (!iq::down(a, NEXP, 1, 0)) { std::printf("down refused\n"); return 1; }CK(hipDeviceSynchronize());const auto part = down_(dp, (size_t) NEXP * E);+ bits("token part", type, part.data(), part.size() * 4);worst = 0;for (int e = 0; e < NEXP; ++e)for (int r = 0; r < E; ++r) {@@ -253,6 +365,7 @@ int main(int argc, char** argv) {if (!iq::set_gate_up(q, 0)) { std::printf("set_gate_up refused\n"); return 1; }CK(hipDeviceSynchronize());const auto h16 = down_(dh16, (size_t) NR * FF);+ bits("set h16", type, h16.data(), h16.size() * 2);double w2 = 0;for (int e = 0; e < 3; ++e)for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr)@@ -267,6 +380,8 @@ int main(int argc, char** argv) {if (!iq::set_down(q, 0)) { std::printf("set_down refused\n"); return 1; }CK(hipDeviceSynchronize());const auto y = down_(dy, (size_t) NR * E);+ bits("set y", type, y.data(), y.size() * 4);+ real_shape_bits(type, rng);w2 = 0;for (int e = 0; e < 3; ++e)for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr) {@@ -316,6 +431,8 @@ int main(int argc, char** argv) {}CK(hipFree(dblob));}+ if (g_cmp) { std::printf("bit-compare vs golden: %ld buffers, %ld differ -> %s\n", g_cmp_n, g_cmp_bad, g_cmp_bad ? "DIFFERENT" : "IDENTICAL"); fails += g_cmp_bad != 0; }+ if (g_dump) std::fclose(g_dump);std::printf("%s\n", fails ? "FAILED" : "ALL OK");return fails;}