Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
39.4 KB
Raw Blame
// Strata. The i-quant routed experts of the DeepSeek-V4.1 path (ds4_iq.cu on the device, ds4_iq_cpu.cpp on the CPU lane)
// against ggml's own decode of the same bytes plus a double reference: the token path (gate/up, down), the window path
// (grouped entries), the prompt sets (gate/up and down), and the CPU lane on the same experts.  Random legal blocks: any
// bit pattern with a sane F16 scale is one.  This is the H2 gate for a new type (IQ2_XS first): the kernel's integer sums
// must equal ggml's weights times the same activations.
//   iq_expert_test [--type 16,17,18,21,22,23,29] [--bench] [--iq256 0|1]
#include "../../src/dsv41/ds4_gguf_cpu.hpp"
#include "../../src/dsv41/ds4_iq.cuh"

#include "strata/dsv41/hip_or_cuda.h"   // HIP names on a HIP build, the CUDA runtime otherwise

#include "ggml.h"   // ggml_get_type_traits(type)->to_float: ggml's decode of the same blocks

#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <random>
#include <string>
#include <vector>

namespace iq = strata::ds4::gguf::iq;
namespace gg = strata::ds4::gguf;
#define CK(x) do { hipError_t e_ = (x); if (e_ != hipSuccess) { std::printf("HIP error %s at %d\n", hipGetErrorString(e_), __LINE__); std::exit(2); } } while (0)

template <typename T> static T* up(const std::vector<T>& v) {
    T* d; CK(hipMalloc(&d, std::max<size_t>(1, v.size() * sizeof(T)))); CK(hipMemcpy(d, v.data(), v.size() * sizeof(T), hipMemcpyHostToDevice)); return d;
}
template <typename T> static std::vector<T> down_(const T* d, size_t n) {
    std::vector<T> v(n); CK(hipMemcpy(v.data(), d, n * sizeof(T), hipMemcpyDeviceToHost)); return v;
}
static float h2f(uint16_t h) {
    const int s = h >> 15, e = (h >> 10) & 31, m = h & 1023;
    float f = e == 0 ? std::ldexp((float) m, -24) : std::ldexp(1.0f + m / 1024.0f, e - 15);
    return s ? -f : f;
}
static uint16_t f2h(float f) {   // round to nearest even, normal range
    uint32_t u; std::memcpy(&u, &f, 4);
    const uint32_t s = (u >> 16) & 0x8000u; int e = (int) ((u >> 23) & 0xff) - 127 + 15; uint32_t m = u & 0x7fffffu;
    uint32_t hm = m >> 13; const uint32_t rem = m & 0x1fffu;
    uint32_t v = ((uint32_t) e << 10) | hm;
    if (rem > 0x1000u || (rem == 0x1000u && (v & 1))) ++v;
    return (uint16_t) (s | v);
}

struct Q8 { std::vector<int8_t> q; std::vector<float> d; std::vector<float> xhat; };
// the engine's q8_quant: sc = max|x| / 127 per 32, q = round-half-even(x / sc)
static Q8 quant(const float* x, int k) {
    Q8 r; r.q.resize(k); r.d.resize(k / 32); r.xhat.resize(k);
    for (int b = 0; b < k / 32; ++b) {
        float am = 0; for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(x[b * 32 + j]));
        const float sc = am / 127.0f; r.d[b] = sc;
        for (int j = 0; j < 32; ++j) { const int q = am > 0 ? (int) std::nearbyint(x[b * 32 + j] / sc) : 0; r.q[b * 32 + j] = (int8_t) q; r.xhat[b * 32 + j] = (float) q * sc; }
    }
    return r;
}
static double swiglu(double g, double u, double lim) {
    if (lim > 0) { g = std::min(g, lim); u = std::min(std::max(u, -lim), lim); }
    return g / (1.0 + std::exp(-g)) * u;
}
static double dotd(const float* w, const float* x, int k, double* sabs = nullptr) {
    double s = 0, sa = 0; for (int i = 0; i < k; ++i) { const double t = (double) w[i] * x[i]; s += t; sa += std::fabs(t); } if (sabs) *sabs = sa; return s;
}
// an activation buffer of a ggml type (Q8_K for the i-quant rows) decoded by ggml itself
static std::vector<float> act_values(int type, const void* act, int k) {
    std::vector<float> v((size_t) k);
    if (type == 15) {   // Q8_K: float d, int8 qs[256], int16 bsums[16] - no to_float (an intermediate type)
        const int nb = k / 256;
        const size_t bs = ggml_row_size((ggml_type) type, k) / (size_t) nb;
        for (int b = 0; b < nb; ++b) {
            const uint8_t* p = (const uint8_t*) act + (size_t) b * bs;
            float d; std::memcpy(&d, p, 4);
            const int8_t* qs = (const int8_t*) (p + 4);
            for (int j = 0; j < 256; ++j) v[(size_t) b * 256 + j] = d * (float) qs[j];
        }
        return v;
    }
    const ggml_type_traits* t = ggml_get_type_traits((ggml_type) type);
    if (t == nullptr || t->to_float == nullptr) { std::printf("no to_float for activation type %d\n", type); std::exit(2); }
    t->to_float(act, v.data(), k);
    return v;
}

struct Mat { std::vector<uint8_t> bytes; std::vector<float> w; int rows = 0, k = 0; };
// random legal blocks of `type` for a rows x k matrix, decoded by ggml itself
static Mat make_mat(std::mt19937_64& rng, int type, int rows, int k) {
    Mat m; m.rows = rows; m.k = k;
    const int bv = iq::block_values(type), bb = iq::block_bytes(type);
    const size_t nblk = (size_t) rows * (k / bv);
    m.bytes.resize(nblk * (size_t) bb);
    for (auto& b : m.bytes) b = (uint8_t) rng();
    for (size_t b = 0; b < nblk; ++b) {   // a sane F16 scale: at offset 0 (every type here but IQ1_M has d there)
        const uint16_t d = f2h(0.02f + 0.0002f * (float) (rng() % 600));
        uint8_t* p = &m.bytes[b * (size_t) bb];
        if (type == 29) {   // block_iq1_m has no d field: its F16 scale is the top nibbles of its four u16 scales
            p[49] = (uint8_t) ((p[49] & 0x0f) | ((d & 0x000f) << 4));
            p[51] = (uint8_t) ((p[51] & 0x0f) | (d & 0x00f0));
            p[53] = (uint8_t) ((p[53] & 0x0f) | ((d & 0x0f00) >> 4));
            p[55] = (uint8_t) ((p[55] & 0x0f) | ((d & 0xf000) >> 8));
        } else {
            std::memcpy(p, &d, 2);
        }
    }
    m.w.resize((size_t) rows * k);
    const ggml_type_traits* tr = ggml_get_type_traits((ggml_type) type);
    if (tr == nullptr || tr->to_float == nullptr) { std::printf("ggml has no to_float for type %d\n", type); std::exit(2); }
    const size_t rb = (size_t) (k / bv) * (size_t) bb;
    for (int r = 0; r < rows; ++r) tr->to_float(&m.bytes[(size_t) r * rb], &m.w[(size_t) r * k], k);
    return m;
}

// one decode token of the DeepSeek shape (n_embd 5120, n_ff 2304, top-6 experts): gate/up + down launches, per-layer time
static void bench(int type) {
    const int E = 5120, FF = 2304, NE = 6;
    const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);
    auto al = [](size_t n) { return (n + 255) / 256 * 256; };
    const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);
    std::mt19937_64 rng(5);
    std::vector<uint8_t> one(blob);
    for (auto& b : one) b = (uint8_t) rng();
    std::vector<unsigned long long> plan(NE);
    for (int e = 0; e < NE; ++e) { uint8_t* d; CK(hipMalloc(&d, blob)); CK(hipMemcpy(d, one.data(), blob, hipMemcpyHostToDevice)); plan[e] = (unsigned long long) d; }
    std::vector<int8_t> xq(E, 3), hq((size_t) NE * FF, 3); std::vector<float> xd(E / 32, 0.01f), hd((size_t) NE * FF / 32, 0.01f);
    iq::Args a{};
    a.plan_ptr = up(plan); a.n_embd = E; a.n_ff = FF; a.limit = 10.0f; a.xq = up(xq); a.xd = up(xd); a.hq = up(hq); a.hd = up(hd); a.tg = a.td = type;
    for (int r = 0; r < 3; ++r) a.tr[r] = off[r];
    float *dh, *dp; CK(hipMalloc(&dh, (size_t) NE * FF * 4)); CK(hipMalloc(&dp, (size_t) NE * E * 4)); a.h = dh; a.part = dp;
    hipEvent_t e0, e1; hipEventCreate(&e0); hipEventCreate(&e1);
    for (int rep = 0; rep < 5; ++rep) { iq::gate_up(a, NE, 1, 0); iq::down(a, NE, 1, 0); }
    CK(hipDeviceSynchronize());
    float ms_gu = 0, ms_dn = 0;
    for (int part = 0; part < 2; ++part) {
        hipEventRecord(e0);
        for (int rep = 0; rep < 100; ++rep) { if (part == 0) iq::gate_up(a, NE, 1, 0); else iq::down(a, NE, 1, 0); }
        hipEventRecord(e1); hipEventSynchronize(e1);
        float ms; hipEventElapsedTime(&ms, e0, e1);
        (part == 0 ? ms_gu : ms_dn) = ms / 100;
    }
    const double bytes = (double) NE * blob;
    std::printf("bench %-7s 6 experts x 40 layers: gate/up %.0f us, down %.0f us, layer %.0f us -> %.0f GB/s, %.2f Tw/s, 40 layers %.1f ms/token\n",
                iq::name(type), ms_gu * 1e3, ms_dn * 1e3, (ms_gu + ms_dn) * 1e3,
                bytes / ((ms_gu + ms_dn) * 1e-3) / 1e9,
                (double) NE * 3 * FF * E / ((ms_gu + ms_dn) * 1e-3) / 1e12, 40.0 * (ms_gu + ms_dn));
}


// --dump F / --cmp F: every device output of a run as raw bits, to a file / against one (a kernel change that keeps the sums
// exact and the float order must leave all of them identical)
static FILE* g_dump = nullptr; static FILE* g_cmp = nullptr; static long g_cmp_bad = 0, g_cmp_n = 0;
static void bits(const char* what, int type, const void* p, size_t bytes) {
    if (!g_dump && !g_cmp) return;
    if (g_dump) { std::fwrite(p, 1, bytes, g_dump); return; }
    std::vector<uint8_t> ref(bytes);
    if (std::fread(ref.data(), 1, bytes, g_cmp) != bytes) { std::printf("cmp: golden file too short at %s %s\n", iq::name(type), what); ++g_cmp_bad; return; }
    ++g_cmp_n;
    if (std::memcmp(ref.data(), p, bytes) != 0) {
        size_t bad = 0, first = 0; for (size_t i = 0; i < bytes; ++i) if (ref[i] != ((const uint8_t*) p)[i]) { if (!bad) first = i; ++bad; }
        std::printf("%-7s %-22s BITS DIFFER: %zu of %zu bytes (first at %zu)\n", iq::name(type), what, bad, bytes, first);
        ++g_cmp_bad;
    }
}
// the real shapes (n_embd 5120, n_ff 2304): the plan path (6 groups of one entry) and the grouped path (windows of 3 and 2)
static void real_shape_bits(int type) {
    std::mt19937_64 rng(1000 + type);   // (its own stream: the main loop's random experts are the same with or without this)
    const int E = 5120, FF = 2304, NG = 2, NEnt = 12;
    const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);
    auto al = [](size_t n) { return (n + 255) / 256 * 256; };
    const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);
    std::vector<uint8_t> hb((size_t) NG * blob);
    for (auto& b : hb) b = (uint8_t) rng();
    for (size_t b = 0; b < hb.size() / (size_t) iq::block_bytes(type); ++b) {   // sane F16 scales (the same placement as make_mat)
        const uint16_t d = f2h(0.02f + 0.0002f * (float) (rng() % 600));
        uint8_t* p = &hb[b * (size_t) iq::block_bytes(type)];
        if (type == 29) { p[49] = (uint8_t) ((p[49] & 0x0f) | ((d & 0x000f) << 4)); p[51] = (uint8_t) ((p[51] & 0x0f) | (d & 0x00f0)); p[53] = (uint8_t) ((p[53] & 0x0f) | ((d & 0x0f00) >> 4)); p[55] = (uint8_t) ((p[55] & 0x0f) | ((d & 0xf000) >> 8)); }
        else std::memcpy(p, &d, 2);
    }
    uint8_t* dblob = up(hb);
    std::vector<int8_t> xq(2 * E), hq((size_t) NEnt * FF); std::vector<float> xd(2 * E / 32), hd((size_t) NEnt * FF / 32);
    for (auto& v : xq) v = (int8_t) ((int) (rng() % 255) - 127);
    for (auto& v : hq) v = (int8_t) ((int) (rng() % 255) - 127);
    for (auto& v : xd) v = 0.002f + 0.0001f * (float) (rng() % 100);
    for (auto& v : hd) v = 0.002f + 0.0001f * (float) (rng() % 100);
    iq::Args a{};
    a.n_embd = E; a.n_ff = FF; a.limit = 10.0f; a.xq = up(xq); a.xd = up(xd); a.hq = up(hq); a.hd = up(hd); a.tg = a.td = type;
    for (int r = 0; r < 3; ++r) a.tr[r] = off[r];
    float *dh, *dp; CK(hipMalloc(&dh, (size_t) NEnt * FF * 4)); CK(hipMalloc(&dp, (size_t) NEnt * E * 4)); a.h = dh; a.part = dp;
    std::vector<unsigned long long> plan = {(unsigned long long) dblob, (unsigned long long) (dblob + blob), 0ull, (unsigned long long) dblob};
    a.plan_ptr = up(plan);
    CK(hipMemset(dh, 0, (size_t) NEnt * FF * 4)); CK(hipMemset(dp, 0, (size_t) NEnt * E * 4));
    if (!iq::gate_up(a, 4, 1, 0) || !iq::down(a, 4, 1, 0)) { std::printf("real shape: refused\n"); return; }
    CK(hipDeviceSynchronize());
    { auto h = down_(dh, (size_t) NEnt * FF); bits("real plan h", type, h.data(), h.size() * 4); auto p = down_(dp, (size_t) NEnt * E); bits("real plan part", type, p.data(), p.size() * 4); }
    std::vector<unsigned long long> gp = {(unsigned long long) dblob, (unsigned long long) (dblob + blob)};
    std::vector<int> gs = {0, 3, 5}, ge = {0, 1, 2, 3, 4}, ng = {NG};
    a.plan_ptr = nullptr; a.grp_ptr = up(gp); a.grp_start = up(gs); a.grp_ent = up(ge); a.n_groups = up(ng);
    CK(hipMemset(dh, 0, (size_t) NEnt * FF * 4)); CK(hipMemset(dp, 0, (size_t) NEnt * E * 4));
    if (!iq::gate_up(a, NG, 3, 0) || !iq::down(a, NG, 3, 0)) { std::printf("real shape grouped: refused\n"); return; }
    CK(hipDeviceSynchronize());
    { auto h = down_(dh, (size_t) NEnt * FF); bits("real grp h", type, h.data(), h.size() * 4); auto p = down_(dp, (size_t) NEnt * E); bits("real grp part", type, p.data(), p.size() * 4); }
    // more windows: a bound of 7 groups with 3 live (dynamic count), 5/1/3 entries (ne_max 5: the 8-entry kernels), then 9 groups of
    // one entry with 4 live; entries index the h/hq rows, their input row is entry >> 3
    {
        std::vector<unsigned long long> gp2 = {(unsigned long long) dblob, (unsigned long long) (dblob + blob), (unsigned long long) dblob, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull};
        std::vector<int> gs2 = {0, 5, 6, 9, 9, 9, 9, 9, 9, 9}, ge2 = {3, 1, 4, 0, 2, 7, 5, 6, 8, 11, 10, 9}, ng2 = {3};
        a.grp_ptr = up(gp2); a.grp_start = up(gs2); a.grp_ent = up(ge2); a.n_groups = up(ng2);
        CK(hipMemset(dh, 0, (size_t) NEnt * FF * 4)); CK(hipMemset(dp, 0, (size_t) NEnt * E * 4));
        if (!iq::gate_up(a, 7, 5, 0) || !iq::down(a, 7, 5, 0)) { std::printf("real shape window 2: refused\n"); return; }
        CK(hipDeviceSynchronize());
        { auto h = down_(dh, (size_t) NEnt * FF); bits("real win2 h", type, h.data(), h.size() * 4); auto p = down_(dp, (size_t) NEnt * E); bits("real win2 part", type, p.data(), p.size() * 4); }
        std::vector<int> gs3 = {0, 1, 2, 3, 4, 4, 4, 4, 4, 4}, ng3 = {4};
        std::vector<unsigned long long> gp3 = {(unsigned long long) (dblob + blob), (unsigned long long) dblob, (unsigned long long) (dblob + blob), (unsigned long long) dblob, 0ull, 0ull, 0ull, 0ull, 0ull};
        a.grp_ptr = up(gp3); a.grp_start = up(gs3); a.n_groups = up(ng3);
        CK(hipMemset(dh, 0, (size_t) NEnt * FF * 4)); CK(hipMemset(dp, 0, (size_t) NEnt * E * 4));
        if (!iq::gate_up(a, 9, 1, 0) || !iq::down(a, 9, 1, 0)) { std::printf("real shape window 3: refused\n"); return; }
        CK(hipDeviceSynchronize());
        { auto h = down_(dh, (size_t) NEnt * FF); bits("real win3 h", type, h.data(), h.size() * 4); auto p = down_(dp, (size_t) NEnt * E); bits("real win3 part", type, p.data(), p.size() * 4); }
    }
    CK(hipFree(dblob));
}

// the same DeepSeek decode token over a working set far past L2 (>= 640 MiB of distinct expert blobs, rotated launch by launch),
// so the rate is DRAM's, not L2's: gate/up and down timed apart; ne = entries a group (1: the decode token)
static void bench_big(int type, int mib) {
    const int E = 5120, FF = 2304, NE = 6;
    const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);
    auto al = [](size_t n) { return (n + 255) / 256 * 256; };
    const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);
    const int L = std::max(2, (int) (((size_t) mib << 20) / (NE * blob)) + 1);   // layers rotated
    std::mt19937_64 rng(5);
    std::vector<uint8_t> one(blob);
    for (auto& b : one) b = (uint8_t) rng();
    std::vector<unsigned long long> plan((size_t) L * NE);
    for (auto& p : plan) { uint8_t* d; CK(hipMalloc(&d, blob)); CK(hipMemcpy(d, one.data(), blob, hipMemcpyHostToDevice)); p = (unsigned long long) d; }
    const int NW = std::getenv("DS4_BENCH_WIN") ? std::max(1, std::min(16, std::atoi(std::getenv("DS4_BENCH_WIN")))) : 0;   // window path: NW entries a group
    const int NEN = std::max(1, NW) * NE;   // entries
    std::vector<int8_t> xq((size_t) 8 * E, 3), hq((size_t) NEN * FF, 3); std::vector<float> xd((size_t) 8 * E / 32, 0.01f), hd((size_t) NEN * FF / 32, 0.01f);
    for (size_t i = 0; i < xq.size(); ++i) xq[i] = (int8_t) ((int) (rng() % 255) - 127);
    for (size_t i = 0; i < hq.size(); ++i) hq[i] = (int8_t) ((int) (rng() % 255) - 127);
    if (const char* e = std::getenv("DS4_BENCH_ACTIVE")) { const int na = std::atoi(e); for (int l = 0; l < L; ++l) for (int k = na; k < NE; ++k) plan[(size_t) l * NE + k] = 0; }   // (empty plan entries)
    unsigned long long* dplan = up(plan);
    iq::Args a{};
    a.n_embd = E; a.n_ff = FF; a.limit = 10.0f; a.xq = up(xq); a.xd = up(xd); a.hq = up(hq); a.hd = up(hd); a.tg = a.td = type;
    for (int r = 0; r < 3; ++r) a.tr[r] = off[r];
    float *dh, *dp; CK(hipMalloc(&dh, (size_t) NEN * FF * 4)); CK(hipMalloc(&dp, (size_t) NEN * E * 4)); a.h = dh; a.part = dp;
    std::vector<int> gstart(NE + 1), gent(NEN), ngrp(1, NE);
    for (int g = 0; g <= NE; ++g) gstart[g] = g * std::max(1, NW);
    for (int i = 0; i < NEN; ++i) gent[i] = i;
    a.grp_start = up(gstart); a.grp_ent = up(gent); a.n_groups = up(ngrp);
    hipEvent_t e0, e1; hipEventCreate(&e0); hipEventCreate(&e1);
    double us[2] = {0, 0};
    for (int part = 0; part < 2; ++part) {
        auto run = [&](int reps) {
            hipEventRecord(e0);
            for (int rep = 0; rep < reps; ++rep) {
                if (NW) a.grp_ptr = dplan + (size_t) (rep % L) * NE; else a.plan_ptr = dplan + (size_t) (rep % L) * NE;
                if (part == 0) iq::gate_up(a, NE, std::max(1, NW), 0); else iq::down(a, NE, std::max(1, NW), 0);
            }
            hipEventRecord(e1); hipEventSynchronize(e1);
            float ms; hipEventElapsedTime(&ms, e0, e1);
            return (double) ms;
        };
        double ms = 0; int reps = 2 * L;
        while (ms < 300.0) { ms = run(reps); reps *= 2; }   // warm-up: clocks ramp, ~300 ms
        const int n = std::max(L, (int) (reps / 2 * 40.0 / ms));   // ~40 ms a window
        double best = 1e30;
        for (int w = 0; w < 7; ++w) best = std::min(best, run(n) * 1e3 / n);   // the best of 7 windows (clocks/thermals wander)
        us[part] = best;
    }
    const double bg = (double) NE * (off[2]) , bd = (double) NE * al((size_t) E * rb_d);   // bytes read a launch
    std::printf("big %-7s ws %4.0f MiB: gate/up %6.1f us (%5.0f GB/s)  down %6.1f us (%5.0f GB/s)  layer %6.1f us -> %5.0f GB/s, %.2f Tw/s\n",
                iq::name(type), (double) L * NE * blob / 1048576.0, us[0], bg / us[0] * 1e-3, us[1], bd / us[1] * 1e-3, us[0] + us[1],
                (bg + bd) / (us[0] + us[1]) * 1e-3, (double) NE * 3 * FF * E / ((us[0] + us[1]) * 1e-6) / 1e12);
    for (auto p : plan) if (p) hipFree((void*) p);
}

// the light sets (experts of <= 16 rows: DSpark verify windows, short prompts): n_exp experts of `rows` rows each, gate/up and down
static void bench_light(int type, int nexp, int rows, bool tiles = false) {
    const int E = 5120, FF = 2304;
    const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);
    auto al = [](size_t n) { return (n + 255) / 256 * 256; };
    const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);
    std::mt19937_64 rng(7);
    std::vector<uint8_t> one(blob);
    for (auto& b : one) b = (uint8_t) rng();
    uint8_t* w; CK(hipMalloc(&w, blob * nexp));
    for (int e = 0; e < nexp; ++e) CK(hipMemcpy(w + (size_t) e * blob, one.data(), blob, hipMemcpyHostToDevice));
    const int NR = nexp * rows, T = 8;
    if (tiles && rows <= 16) { std::printf("tiles: rows > 16 only\n"); return; }
    std::vector<int> bounds(nexp + 1), idx(NR);
    for (int e = 0; e <= nexp; ++e) bounds[e] = e * rows;
    for (int i = 0; i < NR; ++i) idx[i] = i % T;
    std::vector<float> x((size_t) T * E);
    std::normal_distribution<float> nd(0.f, 1.f);
    for (auto& v : x) v = nd(rng);
    std::vector<uint16_t> h((size_t) NR * FF, 0x3c00);
    iq::SetArgs q{};
    q.wbase = w; q.stride = blob; for (int r = 0; r < 3; ++r) q.tr[r] = off[r];
    q.bounds = up(bounds); q.row0 = 0; q.max_rows = rows; q.skip_le = 0; q.n_exp = nexp; q.n_embd = E; q.n_ff = FF; q.limit = 7.0f;
    q.tg = q.td = type; q.xf = up(x); q.idx = up(idx); q.ldx = E;
    uint16_t* dh; CK(hipMalloc(&dh, (size_t) NR * FF * 2)); q.h = dh; q.ldo = FF;
    q.xh = up(h);
    float* dy; CK(hipMalloc(&dy, (size_t) NR * E * 4));
    hipEvent_t e0, e1; hipEventCreate(&e0); hipEventCreate(&e1);
    double us[2] = {0, 0};
    for (int part = 0; part < 2; ++part) {
        auto run = [&](int reps) {
            hipEventRecord(e0);
            for (int rep = 0; rep < reps; ++rep) {
                if (part == 0) { q.h = dh; q.ldo = FF; if (tiles) iq::set_gate_up(q, 0); else iq::light_gate_up(q, 0); }
                else { q.y = dy; q.ldo = E; q.ldx = FF; if (tiles) iq::set_down(q, 0); else iq::light_down(q, 0); }
            }
            hipEventRecord(e1); hipEventSynchronize(e1);
            float ms; hipEventElapsedTime(&ms, e0, e1);
            return (double) ms;
        };
        double ms = 0; int reps = 4;
        while (ms < 200.0) { ms = run(reps); reps *= 2; }
        const int n = std::max(2, (int) (reps / 2 * 40.0 / ms));
        double best = 1e30;
        for (int wnd = 0; wnd < 5; ++wnd) best = std::min(best, run(n) * 1e3 / n);
        us[part] = best;
    }
    const double bytes = (double) nexp * (off[2] + al((size_t) E * rb_d));
    std::printf("%s %-7s %3d experts x %2d rows: gate/up %7.1f us  down %7.1f us  -> %6.0f GB/s of weights, %.2f Tw/s, %.1f TMAC/s\n", tiles ? "tiles" : "light", iq::name(type), nexp, rows,
                us[0], us[1], bytes / ((us[0] + us[1]) * 1e-6) / 1e9, (double) nexp * 3 * FF * E / ((us[0] + us[1]) * 1e-6) / 1e12, (double) NR * 3 * FF * E / ((us[0] + us[1]) * 1e-6) / 1e12);
    hipFree(w);
}

int main(int argc, char** argv) {
    std::vector<int> types = {16, 17, 22, 18, 21, 23, 29};
    std::vector<int> devs;
    for (int i = 1; i < argc; ++i) {
        if (std::string(argv[i]) == "--bench-tiles") {   // [types] [n_exp] [rows list]: the prompt sets' tensor-core tiles
            std::vector<int> bt, rl = {32, 128};
            int nexp = 48;
            if (i + 1 < argc && argv[i + 1][0] != '-') { for (const char* p = argv[++i]; *p;) { bt.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
            else bt = types;
            if (i + 1 < argc && argv[i + 1][0] != '-') nexp = std::atoi(argv[++i]);
            if (i + 1 < argc && argv[i + 1][0] != '-') { rl.clear(); for (const char* p = argv[++i]; *p;) { rl.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
            for (int t : bt) for (int r : rl) bench_light(t, nexp, r, true);
            return 0;
        } else if (std::string(argv[i]) == "--bench-light") {   // [types] [n_exp] [rows list]
            std::vector<int> bt, rl = {1, 2, 4};
            int nexp = 24;
            if (i + 1 < argc && argv[i + 1][0] != '-') { for (const char* p = argv[++i]; *p;) { bt.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
            else bt = types;
            if (i + 1 < argc && argv[i + 1][0] != '-') nexp = std::atoi(argv[++i]);
            if (i + 1 < argc && argv[i + 1][0] != '-') { rl.clear(); for (const char* p = argv[++i]; *p;) { rl.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
            for (int t : bt) for (int r : rl) bench_light(t, nexp, r);
            return 0;
        } else if (std::string(argv[i]) == "--bench-big") {   // [types] [MiB]
            std::vector<int> bt; int mib = 640;
            if (i + 1 < argc && argv[i + 1][0] != '-') { for (const char* p = argv[++i]; *p;) { bt.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
            else bt = types;
            if (i + 1 < argc && argv[i + 1][0] != '-') mib = std::atoi(argv[++i]);
            for (int t : bt) bench_big(t, mib);
            return 0;
        } else if (std::string(argv[i]) == "--bench") {
            std::vector<int> bt;
            if (i + 1 < argc && argv[i + 1][0] != '-') { for (const char* p = argv[++i]; *p;) { bt.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
            else bt = types;
            for (int t : bt) bench(t);
            return 0;
        } else if (std::string(argv[i]) == "--devices" && i + 1 < argc) {   // every check on each device in turn, in this process;
            devs.clear(); for (const char* p = argv[++i]; *p;) { devs.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; }   // the later ones bit-compared to the first
        } else if (std::string(argv[i]) == "--dump" && i + 1 < argc) {
            g_dump = std::fopen(argv[++i], "wb"); if (!g_dump) { std::printf("cannot write %s\n", argv[i]); return 2; }
        } else if (std::string(argv[i]) == "--cmp" && i + 1 < argc) {
            g_cmp = std::fopen(argv[++i], "rb"); if (!g_cmp) { std::printf("cannot read %s\n", argv[i]); return 2; }
        } else if (std::string(argv[i]) == "--type" && i + 1 < argc) {
            types.clear(); for (const char* p = argv[++i]; *p;) { types.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; }
        } else if (std::string(argv[i]) == "--iq256" && i + 1 < argc) {
            gg::gg_cpu_set_iq256(std::atoi(argv[++i]));
        }
    }
    int fails = 0;
    if (devs.size() > 1) {   // the first device writes its outputs, the others compare to them
        if (g_dump || g_cmp) { std::printf("--devices takes neither --dump nor --cmp\n"); return 2; }
    }
    for (size_t di = 0; di < std::max<size_t>(1, devs.size()); ++di) {
    if (!devs.empty()) {
        CK(hipSetDevice(devs[di]));
        if (devs.size() > 1) {
            if (g_dump) { std::fclose(g_dump); g_dump = nullptr; }
            if (di == 0) g_dump = std::fopen("iq_devices.bin", "wb"); else { g_cmp = std::fopen("iq_devices.bin", "rb"); g_cmp_n = g_cmp_bad = 0; }
        }
        std::printf("---- device %d\n", devs[di]);
    }
    std::mt19937_64 rng(11);
    std::normal_distribution<float> nd(0.f, 1.f);
    auto report = [&](const char* what, int type, double worst, double tol) {
        const bool ok = worst <= tol;
        std::printf("%-7s %-22s worst %.2e (tol %.0e) %s\n", iq::name(type), what, worst, tol, ok ? "ok" : "FAIL");
        fails += !ok;
    };
    for (int type : types) {
        if (!iq::supported(type) && !iq::known(type)) { std::printf("%-7s not a type this engine knows\n", iq::name(type)); ++fails; continue; }
        const bool dev = iq::supported(type);   // (IQ1_S / IQ1_M / IQ4_NL: a known layout, no device kernel - CPU lane only)
        const int E = 512, FF = 256, NEXP = 3, T = 4;
        const float lim = 7.0f;
        const size_t rb_g = iq::row_bytes(type, E), rb_d = iq::row_bytes(type, FF);
        auto al = [](size_t n) { return (n + 255) / 256 * 256; };
        const size_t off[3] = {0, al(FF * rb_g), al(FF * rb_g) * 2}, blob = off[2] + al((size_t) E * rb_d);
        std::vector<Mat> G(NEXP), U(NEXP), D(NEXP);
        auto refh = [&](int e, int r, const float* xv, double& scale) {
            double sg, su;
            const double g = dotd(&G[e].w[(size_t) r * E], xv, E, &sg), u = dotd(&U[e].w[(size_t) r * E], xv, E, &su);
            scale = sg + su + 1e-30;
            return swiglu(g, u, lim);
        };
        std::vector<uint8_t> blobs((size_t) NEXP * blob, 0);
        for (int e = 0; e < NEXP; ++e) {
            G[e] = make_mat(rng, type, FF, E); U[e] = make_mat(rng, type, FF, E); D[e] = make_mat(rng, type, E, FF);
            std::memcpy(&blobs[e * blob + off[0]], G[e].bytes.data(), G[e].bytes.size());
            std::memcpy(&blobs[e * blob + off[1]], U[e].bytes.data(), U[e].bytes.size());
            std::memcpy(&blobs[e * blob + off[2]], D[e].bytes.data(), D[e].bytes.size());
        }
        uint8_t* dblob = up(blobs);
        std::vector<float> x((size_t) T * E);
        for (auto& v : x) v = nd(rng) * (1.0f + 3.0f * (rng() % 7 == 0));
        std::vector<Q8> qx(T);
        std::vector<int8_t> xq((size_t) T * E); std::vector<float> xd((size_t) T * E / 32);
        for (int t = 0; t < T; ++t) { qx[t] = quant(&x[(size_t) t * E], E); std::memcpy(&xq[(size_t) t * E], qx[t].q.data(), E); std::memcpy(&xd[(size_t) t * E / 32], qx[t].d.data(), E / 32 * 4); }
        const int8_t* dxq = up(xq); const float* dxd = up(xd);

        // ---- the reference h of every expert's rows for token 0 (the CPU lane and the device share it)
        std::vector<float> href((size_t) NEXP * FF);
        std::vector<Q8> qh(NEXP);
        for (int e = 0; e < NEXP; ++e)
            for (int r = 0; r < FF; ++r) {
                double sc;
                href[(size_t) e * FF + r] = (float) refh(e, r, qx[0].xhat.data(), sc);
            }
        for (int e = 0; e < NEXP; ++e) qh[e] = quant(&href[(size_t) e * FF], FF);

        // ---- token path (device; skipped for a type with no device kernel)
        if (dev) {
            std::vector<unsigned long long> plan(NEXP);
            for (int e = 0; e < NEXP; ++e) plan[e] = (unsigned long long) (dblob + e * blob);
            iq::Args a{};
            a.plan_ptr = up(plan); a.n_embd = E; a.n_ff = FF; a.limit = lim; a.xq = dxq; a.xd = dxd; a.tg = a.td = type;
            for (int r = 0; r < 3; ++r) a.tr[r] = off[r];
            float* dh; CK(hipMalloc(&dh, (size_t) NEXP * FF * 4)); a.h = dh;
            if (!iq::gate_up(a, NEXP, 1, 0)) { std::printf("gate_up refused\n"); return 1; }
            CK(hipDeviceSynchronize());
            const auto h = down_(dh, (size_t) NEXP * FF);
            bits("token h", type, h.data(), h.size() * 4);
            double worst = 0;
            for (int e = 0; e < NEXP; ++e)
                for (int r = 0; r < FF; ++r) {
                    double sc;
                    const double ref = refh(e, r, qx[0].xhat.data(), sc);
                    worst = std::max(worst, std::fabs(h[(size_t) e * FF + r] - ref) / sc);
                }
            report("token gate_up", type, worst, 1e-5);
            std::vector<int8_t> hq((size_t) NEXP * FF); std::vector<float> hd((size_t) NEXP * FF / 32);
            for (int e = 0; e < NEXP; ++e) { std::memcpy(&hq[(size_t) e * FF], qh[e].q.data(), FF); std::memcpy(&hd[(size_t) e * FF / 32], qh[e].d.data(), FF / 32 * 4); }
            a.hq = up(hq); a.hd = up(hd);
            float* dp; CK(hipMalloc(&dp, (size_t) NEXP * E * 4)); a.part = dp;
            if (!iq::down(a, NEXP, 1, 0)) { std::printf("down refused\n"); return 1; }
            CK(hipDeviceSynchronize());
            const auto part = down_(dp, (size_t) NEXP * E);
            bits("token part", type, part.data(), part.size() * 4);
            worst = 0;
            for (int e = 0; e < NEXP; ++e)
                for (int r = 0; r < E; ++r) {
                    double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], qh[e].xhat.data(), FF, &sa);
                    worst = std::max(worst, std::fabs(part[(size_t) e * E + r] - ref) / (sa + 1e-30));
                }
            report("token down", type, worst, 1e-6);

            // ---- the prompt sets: expert 0 owns rows [0, 37), expert 2 [37, 42)
            const int NR = 42;
            std::vector<int> bounds = {0, 37, 37, 42}, idx(NR);
            for (int i = 0; i < NR; ++i) idx[i] = i % T;
            iq::SetArgs q{};
            q.wbase = dblob; q.stride = blob; for (int r = 0; r < 3; ++r) q.tr[r] = off[r];
            q.bounds = up(bounds); q.row0 = 0; q.max_rows = 37; q.n_exp = 3; q.n_embd = E; q.n_ff = FF; q.limit = lim;
            q.tg = q.td = type; q.xf = up(x); q.idx = up(idx); q.ldx = E;
            uint16_t* dh16; CK(hipMalloc(&dh16, (size_t) NR * FF * 2)); q.h = dh16; q.ldo = FF;
            if (!iq::set_gate_up(q, 0)) { std::printf("set_gate_up refused\n"); return 1; }
            CK(hipDeviceSynchronize());
            const auto h16 = down_(dh16, (size_t) NR * FF);
            bits("set h16", type, h16.data(), h16.size() * 2);
            double w2 = 0;
            for (int e = 0; e < 3; ++e)
                for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr)
                    for (int r = 0; r < FF; ++r) {
                        double sc;
                        const double ref = refh(e, r, &x[(size_t) idx[lr] * E], sc);
                        w2 = std::max(w2, std::fabs((double) h2f(h16[(size_t) lr * FF + r]) - ref) / sc);
                    }
            // (the prompt sets quantise the activations, so this worst case is data dependent: IQ1_S measured 7.1e-3 .. 1.08e-2 over
            // different random-data orders, the same on the older build, with every other check far inside; IQ1_S gets 1.5e-2)
            report("set gate_up (f16)", type, w2, type == 19 ? 1.5e-2 : 1e-2);
            q.xh = dh16; q.ldx = FF;
            float* dy; CK(hipMalloc(&dy, (size_t) NR * E * 4)); q.y = dy; q.ldo = E;
            if (!iq::set_down(q, 0)) { std::printf("set_down refused\n"); return 1; }
            CK(hipDeviceSynchronize());
            const auto y = down_(dy, (size_t) NR * E);
            bits("set y", type, y.data(), y.size() * 4);
            real_shape_bits(type);
            w2 = 0;
            for (int e = 0; e < 3; ++e)
                for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr) {
                    std::vector<float> hf(FF);
                    for (int c = 0; c < FF; ++c) hf[c] = h2f(h16[(size_t) lr * FF + c]);
                    for (int r = 0; r < E; ++r) {
                        double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], hf.data(), FF, &sa);
                        w2 = std::max(w2, std::fabs(y[(size_t) lr * E + r] - ref) / (sa + 1e-30));
                    }
                }
            // (the prompt sets' tiles take F16 operands for the i-quants: the tile's own rounding, not the token path's)
            report("set down", type, w2, 3e-3);

            // ---- the matrix-core tiles (HIP gfx11 / gfx12) forced on, against the scalar tiles forced on the same inputs: each within the
            // tolerances above of the CPU reference, the two within the same tolerance of each other, and the forced-on run must really
            // have launched them (STRATA_DS4_HIPK is not needed)
            if (iq::hipk_force(1)) {
                const long long n0 = iq::hipk_launches();
                iq::SetArgs qm = q;
                uint16_t* mh; CK(hipMalloc(&mh, (size_t) NR * FF * 2)); float* my; CK(hipMalloc(&my, (size_t) NR * E * 4));
                qm.h = mh; qm.ldx = E; qm.ldo = FF; qm.xh = nullptr; qm.y = nullptr;
                if (!iq::set_gate_up(qm, 0)) { std::printf("matrix-core set_gate_up refused\n"); return 1; }
                qm.xh = mh; qm.ldx = FF; qm.y = my; qm.ldo = E;
                if (!iq::set_down(qm, 0)) { std::printf("matrix-core set_down refused\n"); return 1; }
                CK(hipDeviceSynchronize());
                const long long taken = iq::hipk_launches() - n0;
                iq::hipk_force(0);
                const long long n1 = iq::hipk_launches();
                uint16_t* sh; CK(hipMalloc(&sh, (size_t) NR * FF * 2)); float* sy; CK(hipMalloc(&sy, (size_t) NR * E * 4));
                iq::SetArgs qs = q;
                qs.h = sh; qs.ldx = E; qs.ldo = FF; qs.xh = nullptr; qs.y = nullptr;
                if (!iq::set_gate_up(qs, 0)) { std::printf("scalar set_gate_up refused\n"); return 1; }
                qs.xh = sh; qs.ldx = FF; qs.y = sy; qs.ldo = E;
                if (!iq::set_down(qs, 0)) { std::printf("scalar set_down refused\n"); return 1; }
                CK(hipDeviceSynchronize());
                const long long scal = iq::hipk_launches() - n1;
                iq::hipk_force(-1);
                const auto hm = down_(mh, (size_t) NR * FF), hs = down_(sh, (size_t) NR * FF);
                const auto ym = down_(my, (size_t) NR * E), ys = down_(sy, (size_t) NR * E);
                double wg = 0, wd = 0, wx = 0, wy = 0;
                for (int e = 0; e < 3; ++e)
                    for (int lr = bounds[e]; lr < bounds[e + 1]; ++lr) {
                        std::vector<float> hf(FF);
                        for (int c = 0; c < FF; ++c) hf[c] = h2f(hm[(size_t) lr * FF + c]);
                        for (int r = 0; r < FF; ++r) {
                            double sc;
                            const double ref = refh(e, r, &x[(size_t) idx[lr] * E], sc);
                            wg = std::max(wg, std::fabs((double) h2f(hm[(size_t) lr * FF + r]) - ref) / sc);
                            wx = std::max(wx, std::fabs((double) h2f(hm[(size_t) lr * FF + r]) - (double) h2f(hs[(size_t) lr * FF + r])) / sc);
                        }
                        for (int r = 0; r < E; ++r) {
                            double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], hf.data(), FF, &sa);
                            wd = std::max(wd, std::fabs(ym[(size_t) lr * E + r] - ref) / (sa + 1e-30));
                            wy = std::max(wy, std::fabs((double) ym[(size_t) lr * E + r] - (double) ys[(size_t) lr * E + r]) / (sa + 1e-30));
                        }
                    }
                std::printf("%-7s matrix-core tiles taken %lld (scalar run: %lld)\n", iq::name(type), taken, scal);
                if (taken < 2 || scal != 0) { std::printf("%-7s FAIL: the matrix-core tiles did not run as forced\n", iq::name(type)); ++fails; }
                report("wmma gate_up vs CPU", type, wg, type == 19 ? 1.5e-2 : 1e-2);
                report("wmma down vs CPU", type, wd, 3e-3);
                report("wmma vs scalar h", type, wx, type == 19 ? 1.5e-2 : 1e-2);
                report("wmma vs scalar y", type, wy, 3e-3);
            }
        }

        // ---- the CPU lane on the same experts and inputs (ggml-cpu's rows, or Strata's AVX2 ones with DS4_GG_IQ256=1)
        {
            gg::IqCpu fm; std::string err;
            if (!gg::iq_cpu_fmt(type, type, E, FF, fm, err)) { std::printf("%-7s cpu lane: %s\n", iq::name(type), err.c_str()); ++fails; }
            else {
                std::vector<uint8_t> act(fm.act_bytes), hact(fm.h_bytes);
                gg::iq_cpu_quant(fm.gu_act, x.data(), E, act.data());
                // (the CPU lane rounds the activation to its own type - Q8_K, one F32 scale per 256 values - so the
                //  reference is that quantized activation, not the device's per-32 one)
                const std::vector<float> xhat = act_values(fm.gu_act, act.data(), E);
                double wg = 0, wd = 0;
                for (int e = 0; e < NEXP; ++e) {
                    std::vector<float> hc(FF), yc(E);
                    float* hp[1] = {hc.data()}; float* yp[1] = {yc.data()};
                    const void* ap[1] = {act.data()};
                    gg::iq_cpu_gate_up(fm, &blobs[e * blob + off[0]], &blobs[e * blob + off[1]], ap, 1, hp, 0, FF, lim);
                    for (int r = 0; r < FF; ++r) {
                        double sg, su;
                        const double g = dotd(&G[e].w[(size_t) r * E], xhat.data(), E, &sg);
                        const double u = dotd(&U[e].w[(size_t) r * E], xhat.data(), E, &su);
                        wg = std::max(wg, std::fabs(hc[r] - swiglu(g, u, lim)) / (sg + su + 1e-30));
                    }
                    gg::iq_cpu_quant(fm.d_act, &href[(size_t) e * FF], FF, hact.data());
                    const std::vector<float> hhat = act_values(fm.d_act, hact.data(), FF);
                    const void* hpp[1] = {hact.data()};
                    gg::iq_cpu_down(fm, &blobs[e * blob + off[2]], hpp, 1, yp, 0, E);
                    for (int r = 0; r < E; ++r) { double sa; const double ref = dotd(&D[e].w[(size_t) r * FF], hhat.data(), FF, &sa); wd = std::max(wd, std::fabs(yc[r] - ref) / (sa + 1e-30)); }
                }
                report("cpu gate_up vs ref", type, wg, 1e-5);
                report("cpu down vs ref", type, wd, 1e-5);
                std::printf("%-7s cpu rows: %s\n", iq::name(type), gg::iq_cpu_own_rows(type) ? "Strata AVX2" : "ggml-cpu");
            }
        }
        CK(hipFree(dblob));
    }
    if (g_cmp && devs.size() > 1) { std::fclose(g_cmp); g_cmp = nullptr; std::printf("device %d vs device %d: %ld buffers, %ld differ\n", devs[di], devs[0], g_cmp_n, g_cmp_bad); fails += g_cmp_bad != 0; }
    }
    if (g_cmp) { std::printf("bit-compare vs golden: %ld buffers, %ld differ -> %s\n", g_cmp_n, g_cmp_bad, g_cmp_bad ? "DIFFERENT" : "IDENTICAL"); fails += g_cmp_bad != 0; }
    if (g_dump) std::fclose(g_dump);
    std::printf("%s\n", fails ? "FAILED" : "ALL OK");
    return fails;
}