Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// src/kernels/cpu/tq_cpu_parity.cpp - the CPU trellis kernels (tq_cpu.hpp: TQ2_T / TQK6 / TQK7) against their scalar
// reference, against ggml-cpu's own vec_dot and to_float, and their speed. CPU only (no GPU runtime needed).
//
// tq_cpu_parity every check below for tqk6/tqk7 and tq2_t/tq2_t, plain and Hadamard
// tq_cpu_parity --synthetic GU/DOWN one expert pair (types by ggml name, e.g. tqk6/tqk7)
// tq_cpu_parity --synthetic-hadamard GU/DOWN the same with the block-128 Hadamard rotation of the activations
// tq_cpu_parity --bench [--quick] [--only K] the micro-benchmark (run it pinned: taskset -c <cores>);
// --only: the kernels whose name contains K
//
// Checks, per matrix (random stream bytes and scales, so every state is reached, and ggml-quantized Gaussian rows):
// (1) AVX2 and AVX-512 kernels == the scalar kernel, bit for bit, for groups of 1..8 tokens
// (2) a token's row in a group of k == the same token alone, bit for bit (no group-size dependence)
// (3) vs ggml-cpu's vec_dot on the same Q8_0 activations: the same integers summed in another float order, so
// ~1e-7 relative
// (4) vs ggml's to_float weights . the dequantized Q8_0 activations in double: the int16 state table's rounding
// (exact for |codebook| >= 2^-3, within 2^-14 below) plus float rounding, ~1e-6 relative
// and per expert (native_gu_rows -> native_quant_h -> native_down_rows, the engine's entry points, which now take
// these kernels) against the same expert computed with ggml-cpu's vec_dot: float rounding plus the odd Q8_0
// rounding flip of the intermediate.
#include "strata/kernels/cpu/tq_cpu.hpp"
#include "strata/kernels/cpu/native_expert.hpp"
#include "strata/kernels/cpu/expert.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
#include "strata/kernels/cpu/hadamard.hpp"
#include "ggml.h"
#include "ggml-cpu.h"
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <functional>
#include <random>
#include <string>
#include <thread>
#include <vector>
using namespace strata::kernels::cpu;
namespace {
constexpr int H = 2560, FF = 640;
int type_by_name(const std::string& s) {
for (int t = 0; t < GGML_TYPE_COUNT; ++t) {
const char* n = ggml_type_name((ggml_type) t);
if (n != nullptr && s == n) return t;
}
return -1;
}
uint16_t f2h(float f) { return ggml_fp32_to_fp16(f); }
// rows x n weights of `type`: random stream bytes and scales (every state reachable, both scale signs), and the
// last rows (up to 32, ggml's trellis encoder is slow) ggml's own encoding of Gaussian rows
std::vector<uint8_t> make_matrix(int type, int rows, int n, uint32_t seed) {
const size_t rb = ggml_row_size((ggml_type) type, n);
std::vector<uint8_t> m(rb * rows);
std::mt19937 rng(seed);
std::uniform_int_distribution<int> byte(0, 255);
std::uniform_real_distribution<float> scale(0.002f, 0.05f);
const size_t bsz = rb / (size_t) (n / 128);
const int half = rows - std::min(rows / 2, 32);
for (int r = 0; r < half; ++r)
for (int b = 0; b < n / 128; ++b) {
uint8_t* blk = m.data() + (size_t) r * rb + (size_t) b * bsz;
const uint16_t d = f2h((byte(rng) & 1 ? -1.f : 1.f) * scale(rng));
std::memcpy(blk, &d, 2);
for (size_t j = 2; j < bsz; ++j) blk[j] = (uint8_t) byte(rng);
}
std::normal_distribution<float> nd(0.f, 0.02f);
std::vector<float> wf((size_t) (rows - half) * n);
for (auto& v : wf) v = nd(rng);
ggml_quantize_chunk((ggml_type) type, wf.data(), m.data() + (size_t) half * rb, 0, rows - half, n, nullptr);
return m;
}
std::vector<float> make_x(int n, uint32_t seed) {
std::mt19937 rng(seed);
std::normal_distribution<float> nd(0.f, 1.f);
std::vector<float> x(n);
for (auto& v : x) v = nd(rng);
x[(size_t) (seed % (uint32_t) n)] = 40.f; // a massive channel, as the residual stream has
return x;
}
std::vector<uint8_t> q8_0(const std::vector<float>& x) {
std::vector<uint8_t> q(ggml_row_size(GGML_TYPE_Q8_0, (int64_t) x.size()));
ggml_get_type_traits_cpu(GGML_TYPE_Q8_0)->from_float(x.data(), q.data(), (int64_t) x.size());
return q;
}
bool same_bits(const float* a, const float* b, size_t n) { return std::memcmp(a, b, n * sizeof(float)) == 0; }
// one matrix: checks (1) - (4); returns failures
int check_matrix(int type, int rows, int n, uint32_t seed, bool hadamard) {
int fails = 0;
const size_t rb = ggml_row_size((ggml_type) type, n);
const auto m = make_matrix(type, rows, n, seed);
constexpr int NT = kTqMaxTokens;
std::vector<std::vector<uint8_t>> act(NT);
std::vector<std::vector<float>> xr(NT);
std::vector<float> signs(n);
for (int i = 0; i < n; ++i) signs[i] = ((i * 2654435761u) >> 7) & 1 ? -1.f : 1.f;
const void* ap[NT];
for (int t = 0; t < NT; ++t) {
xr[t] = make_x(n, seed * 31 + t);
if (hadamard) hadamard_rotate(xr[t].data(), n, 128, signs.data());
act[t] = q8_0(xr[t]);
ap[t] = act[t].data();
}
auto run = [&](TqIsa isa, int nt, const void* const* a, std::vector<float>& o) {
o.assign((size_t) nt * rows, 0.f);
float* op[NT];
for (int t = 0; t < nt; ++t) op[t] = o.data() + (size_t) t * rows;
// two row ranges, as the pool splits them
tq_rows(isa, type, m.data(), rb, n, a, nt, op, 0, rows / 3);
tq_rows(isa, type, m.data(), rb, n, a, nt, op, rows / 3, rows);
};
std::vector<float> ref, got;
run(TqIsa::scalar, NT, ap, ref);
// (1) every ISA this CPU has, every group size, == scalar bit for bit; (2) group-size independence
for (TqIsa isa : {TqIsa::scalar, TqIsa::avx2, TqIsa::avx512}) {
if (!tq_isa_ok(isa)) { std::printf(" %-6s not on this CPU: skipped\n", tq_isa_name(isa)); continue; }
bool ok = true;
for (int nt = 1; nt <= NT; ++nt)
for (int t0 = 0; t0 + nt <= NT; t0 += nt) {
run(isa, nt, ap + t0, got);
ok = ok && same_bits(got.data(), ref.data() + (size_t) t0 * rows, (size_t) nt * rows);
}
std::printf(" %-6s groups of 1..%d tokens vs the scalar kernel's 8-token run: %s\n", tq_isa_name(isa), NT,
ok ? "bitwise equal" : "DIFFERENT");
fails += !ok;
}
// (3) ggml-cpu's vec_dot, (4) to_float in double
const auto* tc = ggml_get_type_traits_cpu((ggml_type) type);
const auto* tt = ggml_get_type_traits((ggml_type) type);
const auto* q8t = ggml_get_type_traits(GGML_TYPE_Q8_0);
double num3 = 0, num4 = 0, den = 0, max3 = 0;
std::vector<float> wf(n), af(n);
for (int t = 0; t < 2; ++t) {
q8t->to_float(act[t].data(), af.data(), n);
for (int r = 0; r < rows; ++r) {
float g = 0;
tc->vec_dot(n, &g, 0, m.data() + (size_t) r * rb, 0, act[t].data(), 0, 1);
tt->to_float(m.data() + (size_t) r * rb, wf.data(), n);
double d = 0, mag = 0;
for (int i = 0; i < n; ++i) { d += (double) wf[i] * af[i]; mag += std::fabs((double) wf[i] * af[i]); }
const double ours = ref[(size_t) t * rows + r];
num3 += (ours - g) * (ours - g);
num4 += (ours - d) * (ours - d);
den += d * d;
max3 = std::max(max3, std::fabs(ours - g) / (mag + 1e-30));
}
}
const double r3 = std::sqrt(num3 / den), r4 = std::sqrt(num4 / den);
const bool ok3 = r3 < 1e-5 && max3 < 1e-5, ok4 = r4 < 1e-4;
std::printf(" vs ggml-cpu vec_dot: rel %.2e (worst row %.2e of sum|w a|) %s | vs to_float in double: rel %.2e %s\n",
r3, max3, ok3 ? "ok" : "FAILED", r4, ok4 ? "ok" : "FAILED");
return fails + !ok3 + !ok4;
}
// a whole expert through the engine's entry points vs the same expert on ggml-cpu's vec_dot
int check_expert(int gu, int dn, bool hadamard) {
NativeFmt f;
std::string err;
if (!native_fmt(gu, dn, H, FF, f, err)) { std::printf("native_fmt: %s\n", err.c_str()); return 1; }
std::vector<float> sh(H), sf(FF);
for (int i = 0; i < H; ++i) sh[i] = (i * 7 % 3) ? 1.f : -1.f;
for (int i = 0; i < FF; ++i) sf[i] = (i * 5 % 3) ? -1.f : 1.f;
if (hadamard) {
f.had_block = 128;
f.had_gu = f.had_d = true;
f.gu_signs = sh.data();
f.d_signs = sf.data();
}
std::vector<uint8_t> blob(f.bytes);
const auto g = make_matrix(gu, FF, H, 101), u = make_matrix(gu, FF, H, 202), d = make_matrix(dn, H, FF, 303);
std::memcpy(blob.data(), g.data(), g.size());
std::memcpy(blob.data() + f.up_off, u.data(), u.size());
std::memcpy(blob.data() + f.down_off, d.data(), d.size());
constexpr int NT = 3;
double worst = 0;
bool bitwise_group = true;
std::vector<std::vector<float>> outs(NT, std::vector<float>(H));
std::vector<std::vector<uint8_t>> acts(NT, std::vector<uint8_t>(kNativeActBytes));
std::vector<std::vector<float>> ffs(NT, std::vector<float>(FF));
std::vector<std::vector<uint8_t>> hqs(NT, std::vector<uint8_t>(kNativeHBytes));
const void* a[NT];
float* ffp[NT];
const void* hp[NT];
float* op[NT];
for (int t = 0; t < NT; ++t) {
const auto x = make_x(H, 7 + t);
native_quant_act(f, x.data(), acts[t].data());
a[t] = acts[t].data();
ffp[t] = ffs[t].data();
hp[t] = hqs[t].data();
op[t] = outs[t].data();
}
native_gu_rows(f, blob.data(), a, NT, ffp, 0, FF);
for (int t = 0; t < NT; ++t) native_quant_h(f, ffs[t].data(), hqs[t].data());
native_down_rows(f, blob.data(), hp, NT, op, 0, H);
const auto* tg = ggml_get_type_traits_cpu((ggml_type) gu);
const auto* td = ggml_get_type_traits_cpu((ggml_type) dn);
for (int t = 0; t < NT; ++t) {
std::vector<float> ff(FF), out(H);
for (int r = 0; r < FF; ++r) {
float gg = 0, uu = 0;
tg->vec_dot(H, &gg, 0, blob.data() + (size_t) r * f.gu_row, 0, a[t], 0, 1);
tg->vec_dot(H, &uu, 0, blob.data() + f.up_off + (size_t) r * f.gu_row, 0, a[t], 0, 1);
ff[r] = (gg / (1.f + std::exp(-gg))) * uu;
}
std::vector<uint8_t> hq(kNativeHBytes);
native_quant_h(f, ff.data(), hq.data());
for (int r = 0; r < H; ++r)
td->vec_dot(FF, &out[r], 0, blob.data() + f.down_off + (size_t) r * f.d_row, 0, hq.data(), 0, 1);
double num = 0, den = 0;
for (int r = 0; r < H; ++r) { num += std::pow((double) outs[t][r] - out[r], 2); den += (double) out[r] * out[r]; }
worst = std::max(worst, std::sqrt(num / den));
// the same token alone through the engine path: the same bits as in the group of three
std::vector<float> ff1(FF), o1(H);
float* f1[1] = {ff1.data()};
float* o1p[1] = {o1.data()};
const void* a1[1] = {a[t]};
native_gu_rows(f, blob.data(), a1, 1, f1, 0, FF);
std::vector<uint8_t> h1(kNativeHBytes);
native_quant_h(f, ff1.data(), h1.data());
const void* h1p[1] = {h1.data()};
native_down_rows(f, blob.data(), h1p, 1, o1p, 0, H);
bitwise_group = bitwise_group && same_bits(o1.data(), outs[t].data(), H);
}
const bool ok = worst < 2e-3 && bitwise_group;
std::printf(" expert %s/%s%s through native_gu_rows/native_down_rows (%s, %s): vs ggml-cpu vec_dot rel %.2e, "
"alone vs in a group of %d: %s %s\n",
ggml_type_name((ggml_type) gu), ggml_type_name((ggml_type) dn), hadamard ? " +hadamard" : "",
tq_kernels_enabled() ? "tq kernels" : "ggml-cpu", tq_isa_name(tq_isa()), worst, NT,
bitwise_group ? "bitwise equal" : "DIFFERENT", ok ? "ok" : "FAILED");
return ok ? 0 : 1;
}
int synthetic(int gu, int dn, bool hadamard) {
int fails = 0;
std::printf("%s/%s%s (tq kernels: %s, best ISA here %s)\n", ggml_type_name((ggml_type) gu),
ggml_type_name((ggml_type) dn), hadamard ? " +hadamard" : "", tq_kernels_enabled() ? "on" : "off",
tq_isa_name(tq_isa()));
std::printf(" %s, 96 rows of %d (gate/up shape)\n", ggml_type_name((ggml_type) gu), H);
fails += check_matrix(gu, 96, H, 1000 + gu, hadamard);
std::printf(" %s, 256 rows of %d (down shape)\n", ggml_type_name((ggml_type) dn), FF);
fails += check_matrix(dn, 256, FF, 2000 + dn, hadamard);
fails += check_expert(gu, dn, hadamard);
return fails;
}
// ------------------------------------------------------------------------------------------------ the benchmark
double now_us() {
return std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now().time_since_epoch()).count();
}
struct BenchExpert {
std::vector<uint8_t> gu, dn; // gate rows then up rows; down rows
size_t gu_row, d_row;
};
// one variant: compute one expert (gate+up rows, then down rows) for `nt` tokens
using ExpertFn = std::function<void(const BenchExpert&, int nt)>;
// the core clock right now: a chain of dependent adds runs at one per cycle (GCC / Clang on x86-64; 0 elsewhere)
double core_ghz() {
#if defined(__GNUC__) && defined(__x86_64__)
uint64_t x = 0;
const long n = 1000000;
const double t0 = now_us();
for (long i = 0; i < n; ++i) __asm__ volatile("add $1, %0\n\tadd $1, %0\n\tadd $1, %0\n\tadd $1, %0" : "+r"(x));
return 4.0 * n / ((now_us() - t0) * 1e3);
#else
return 0.0;
#endif
}
constexpr double kBlocksPerExpert = 2.0 * FF * (H / 128) + (double) H * (FF / 128); // 128-weight blocks
// one slice of `ms` milliseconds cycling over the experts: ns per expert call and the clock around it (negated when
// the clock moved by more than 5% during the slice: such samples count only when a variant has no other)
std::pair<double, double> time_slice(const std::vector<BenchExpert>& ex, const ExpertFn& fn, int nt, double ms,
size_t& next) {
const double g0 = core_ghz();
int it = 0;
const double t0 = now_us();
double t1 = t0;
do {
fn(ex[next++ % ex.size()], nt);
++it;
t1 = now_us();
} while (t1 - t0 < ms * 1000.0 || it < 2);
const double g1 = core_ghz();
const double ghz = 0.5 * (g0 + g1);
return {(t1 - t0) * 1e3 / it, ghz > 0 && std::fabs(g1 - g0) > 0.05 * g0 ? -ghz : ghz};
}
double median(std::vector<double> v) {
if (v.empty()) return 0.0;
std::sort(v.begin(), v.end());
return v[v.size() / 2];
}
// `threads` threads, each cycling over its own experts: aggregate weight GB/s
double time_parallel(const std::vector<BenchExpert>& ex, const ExpertFn& fn, int threads, double ms) {
std::atomic<long> done{0};
std::atomic<bool> stop{false};
std::vector<std::thread> th;
const double t0 = now_us();
for (int k = 0; k < threads; ++k)
th.emplace_back([&, k] {
long c = 0;
for (size_t i = (size_t) k; !stop.load(std::memory_order_relaxed); i += (size_t) threads, ++c)
fn(ex[i % ex.size()], 1);
done += c;
});
std::this_thread::sleep_for(std::chrono::milliseconds((int) ms));
stop = true;
for (auto& t : th) t.join();
const double us = now_us() - t0;
const double bytes = (double) (ex[0].gu_row * 2 * FF + ex[0].dn.size()) * (double) done.load();
return bytes / (us * 1e3);
}
int bench(bool quick, const std::string& only) {
const int ncold = quick ? 48 : 96; // ~50-130 MB of weights: past the last-level cache
std::printf("CPU: %s | AVX-512 kernels: %s, AVX2: %s | ggml-cpu built for this CPU (-march=native unless "
"STRATA_PORTABLE)\n", cpu_name().c_str(), cpu_avx512_ok() ? "yes" : "no", cpu_avx2_ok() ? "yes" : "no");
std::printf("expert = gate+up %d x %d and down %d x %d of one type, %d distinct experts cycled (their weights come "
"from DRAM, as a cache miss's do)\n\n", FF, H, H, FF, ncold);
struct Variant { std::string name; int type; ExpertFn fn; bool ok; };
std::vector<Variant> vars;
auto add_tq = [&](int type) {
for (TqIsa isa : {TqIsa::scalar, TqIsa::avx2, TqIsa::avx512}) {
vars.push_back({std::string("strata ") + tq_isa_name(isa), type,
[type, isa](const BenchExpert& e, int nt) {
thread_local std::vector<uint8_t> a, h;
thread_local std::vector<float> ff, out;
a.resize(ggml_row_size(GGML_TYPE_Q8_0, H) * kTqMaxTokens);
h.resize(ggml_row_size(GGML_TYPE_Q8_0, FF) * kTqMaxTokens);
ff.resize((size_t) FF * kTqMaxTokens);
out.resize((size_t) H * kTqMaxTokens);
const void* ap[kTqMaxTokens];
const void* hp[kTqMaxTokens];
float* fp[kTqMaxTokens];
float* op[kTqMaxTokens];
for (int t = 0; t < nt; ++t) {
ap[t] = a.data() + t * ggml_row_size(GGML_TYPE_Q8_0, H);
hp[t] = h.data() + t * ggml_row_size(GGML_TYPE_Q8_0, FF);
fp[t] = ff.data() + (size_t) t * FF;
op[t] = out.data() + (size_t) t * H;
}
tq_gu_rows(isa, type, e.gu.data(), e.gu_row, e.gu_row * FF, H, ap, nt, fp, 0, FF);
tq_rows(isa, type, e.dn.data(), e.d_row, FF, hp, nt, op, 0, H);
},
tq_isa_ok(isa)});
}
vars.push_back({"ggml-cpu vec_dot", type,
[type](const BenchExpert& e, int nt) {
const auto* tc = ggml_get_type_traits_cpu((ggml_type) type);
thread_local std::vector<uint8_t> a(ggml_row_size(GGML_TYPE_Q8_0, H)),
h(ggml_row_size(GGML_TYPE_Q8_0, FF));
thread_local std::vector<float> ff(FF), out(H);
for (int t = 0; t < nt; ++t) {
for (int r = 0; r < FF; ++r) {
float g, u;
tc->vec_dot(H, &g, 0, e.gu.data() + (size_t) r * e.gu_row, 0, a.data(), 0, 1);
tc->vec_dot(H, &u, 0, e.gu.data() + (size_t) (FF + r) * e.gu_row, 0, a.data(), 0, 1);
ff[r] = (g / (1.f + std::exp(-g))) * u;
}
for (int r = 0; r < H; ++r)
tc->vec_dot(FF, &out[r], 0, e.dn.data() + (size_t) r * e.d_row, 0, h.data(), 0, 1);
}
},
true});
};
auto add_q2 = [&](bool avx512) {
vars.push_back({avx512 ? "strata Q2_0 avx512" : "strata Q2_0 avx2", 42,
[avx512](const BenchExpert& e, int nt) {
thread_local ActQ a1[MAXT], a2[MAXT];
thread_local bool init = false;
if (!init) { // real scales and codes (the kernel reads every field)
const auto x = make_x(H, 3), h = make_x(FF, 4);
for (int t = 0; t < MAXT; ++t) { act_quant_q8_1_avx2(x.data(), H, a1[t]); act_quant_q8_1_avx2(h.data(), FF, a2[t]); }
init = true;
}
thread_local float g[MAXT][FF], u[MAXT][FF], out[MAXT][H];
const ActQ* ap[MAXT];
const ActQ* hp[MAXT];
float* gp[MAXT];
float* up[MAXT];
float* op[MAXT];
for (int t = 0; t < nt; ++t) { ap[t] = &a1[t]; hp[t] = &a2[t]; gp[t] = g[t]; up[t] = u[t]; op[t] = out[t]; }
auto rows = avx512 ? q2_0_gguf_rows_multi : q2_0_gguf_rows_multi_avx2;
rows(e.gu.data(), e.gu_row, H / 64, ap, nt, gp, 0, FF);
rows(e.gu.data() + e.gu_row * FF, e.gu_row, H / 64, ap, nt, up, 0, FF);
for (int t = 0; t < nt; ++t)
for (int r = 0; r < FF; ++r) g[t][r] = (g[t][r] / (1.f + std::exp(-g[t][r]))) * u[t][r];
rows(e.dn.data(), e.d_row, FF / 64, hp, nt, op, 0, H);
},
avx512 ? cpu_avx512_ok() : cpu_avx2_ok()});
};
for (int type : {145, 146, 144}) add_tq(type);
add_q2(false);
add_q2(true);
// the Q2_0 pack's own AVX-512 kernel (the repacked 1,382,400-byte blob that setup makes on AVX-512 CPUs)
vars.push_back({"strata Q2_0 avx512 pack", 42,
[](const BenchExpert& e, int nt) {
thread_local ActQ a1, a2;
thread_local bool init = false;
if (!init) {
const auto x = make_x(H, 3), h = make_x(FF, 4);
act_quant_q8_1(x.data(), H, a1);
act_quant_q8_1(h.data(), FF, a2);
init = true;
}
thread_local float ff[FF], out[H];
for (int t = 0; t < nt; ++t) { // its multi-token kernels are the same per token
s2_expert_gu_rows(e.gu.data(), a1, ff, 0, FF);
s2_expert_down_rows(e.gu.data(), a2, out, 0, H);
}
},
cpu_avx512_ok()});
auto make_experts = [&](int type, int count) {
std::vector<BenchExpert> v((size_t) count);
for (int k = 0; k < count; ++k) {
BenchExpert& e = v[(size_t) k];
e.gu_row = ggml_row_size((ggml_type) type, H);
e.d_row = ggml_row_size((ggml_type) type, FF);
// (Q2_0: room for one repacked blob too, which the pack kernel reads from the same buffer)
e.gu.resize(type == 42 ? std::max(e.gu_row * 2 * FF, (size_t) BLOB) : e.gu_row * 2 * FF);
e.dn.resize(e.d_row * H);
std::mt19937 rng(77 + k);
for (auto& b : e.gu) b = (uint8_t) rng();
for (auto& b : e.dn) b = (uint8_t) rng();
// sane fp16 scales at every block start (Q2_0: 18-byte blocks, TQ: 26/30/34)
const size_t bsz = type == 42 ? 18 : ggml_row_size((ggml_type) type, 128);
const uint16_t d = f2h(0.01f);
for (size_t o = 0; o + bsz <= e.gu.size(); o += bsz) std::memcpy(e.gu.data() + o, &d, 2);
for (size_t o = 0; o + bsz <= e.dn.size(); o += bsz) std::memcpy(e.dn.data() + o, &d, 2);
if (type == 42) { // the same bytes also hold one repacked Q2_0 blob (codes, then fp16 scales)
for (size_t o = O_GU_SCALES; o + 2 <= BLOB; o += 2) std::memcpy(e.gu.data() + o, &d, 2);
}
}
return v;
};
// Single thread, the variants of a type interleaved in short slices over several rounds (this machine's clock
// moves; a slice whose clock moved is dropped), medians reported: ns and core cycles per 128-weight block, us
// per expert, weight GB/s; and per token in a group of three (a verify window's tokens routed to one expert).
// The activations are zero-filled (the kernels do not branch on values) except Q2_0's, which are real.
const int rounds = quick ? 5 : 15;
const double slice_ms = quick ? 40 : 60;
std::printf("%-6s %-24s %9s %8s %8s %8s | %10s %8s\n", "type", "kernel", "us/expert", "ns/blk", "cyc/blk", "GB/s",
"x3: us/tok", "cyc/blk");
for (int type : {145, 146, 144, 42}) {
const auto cold = make_experts(type, ncold);
const double eb = (double) (cold[0].gu_row * 2 * FF + cold[0].dn.size()); // weight bytes per expert
std::vector<const Variant*> vs;
for (const auto& v : vars)
if (v.type == type && v.name.find(only) != std::string::npos) {
if (v.ok) vs.push_back(&v);
else std::printf("%-6s %-24s (not on this CPU)\n", ggml_type_name((ggml_type) type), v.name.c_str());
}
std::vector<std::vector<double>> ns1(vs.size()), cy1(vs.size()), ns3(vs.size()), cy3(vs.size());
std::vector<std::vector<double>> ns1m(vs.size()), cy1m(vs.size()), ns3m(vs.size()), cy3m(vs.size());
size_t next = 0;
for (auto* v : vs) v->fn(cold[0], 3); // warm-up: tables, buffers
for (int r = 0; r < rounds; ++r)
for (size_t k = 0; k < vs.size(); ++k) {
const bool slow = vs[k]->name.find("scalar") != std::string::npos;
const auto a = time_slice(cold, vs[k]->fn, 1, slow ? slice_ms / 2 : slice_ms, next);
const auto b = time_slice(cold, vs[k]->fn, 3, slow ? slice_ms / 2 : slice_ms, next);
if (a.second >= 0) { ns1[k].push_back(a.first); cy1[k].push_back(a.first * a.second); }
else { ns1m[k].push_back(a.first); cy1m[k].push_back(-a.first * a.second); }
if (b.second >= 0) { ns3[k].push_back(b.first); cy3[k].push_back(b.first * b.second); }
else { ns3m[k].push_back(b.first); cy3m[k].push_back(-b.first * b.second); }
}
std::vector<size_t> steady(vs.size());
for (size_t k = 0; k < vs.size(); ++k) {
steady[k] = ns1[k].size();
if (ns1[k].empty()) { ns1[k] = ns1m[k]; cy1[k] = cy1m[k]; }
if (ns3[k].empty()) { ns3[k] = ns3m[k]; cy3[k] = cy3m[k]; }
}
for (size_t k = 0; k < vs.size(); ++k) {
const double n1 = median(ns1[k]), c1 = median(cy1[k]), n3 = median(ns3[k]), c3 = median(cy3[k]);
std::printf("%-6s %-24s %9.1f %8.1f %8.1f %8.2f | %10.1f %8.1f (%zu/%d steady)\n",
ggml_type_name((ggml_type) type), vs[k]->name.c_str(), n1 / 1e3, n1 / kBlocksPerExpert,
c1 / kBlocksPerExpert, eb / n1, n3 / 3e3, c3 / kBlocksPerExpert, steady[k], rounds);
}
std::fflush(stdout);
}
// several workers at once (as the pool runs a layer's misses): aggregate GB/s on the cold set
const int nthr = (int) std::max(1u, std::min(4u, std::thread::hardware_concurrency()));
std::printf("\n%d threads at once, cold experts, one token: aggregate weight GB/s (us per expert per thread)\n", nthr);
for (int type : {145, 146, 144, 42}) {
const auto cold = make_experts(type, ncold);
const double eb = (double) (cold[0].gu_row * 2 * FF + cold[0].dn.size());
std::vector<const Variant*> vs;
for (const auto& v : vars)
if (v.type == type && v.ok && v.name.find("scalar") == std::string::npos && v.name.find(only) != std::string::npos)
vs.push_back(&v);
std::vector<std::vector<double>> g(vs.size());
for (int r = 0; r < (quick ? 2 : 4); ++r)
for (size_t k = 0; k < vs.size(); ++k) g[k].push_back(time_parallel(cold, vs[k]->fn, nthr, quick ? 150 : 250));
for (size_t k = 0; k < vs.size(); ++k) {
const double gbs = median(g[k]);
std::printf("%-6s %-24s %8.2f GB/s (%7.1f us per expert per thread)\n", ggml_type_name((ggml_type) type),
vs[k]->name.c_str(), gbs, eb / (gbs * 1e3) * nthr);
}
std::fflush(stdout);
}
// the per-token Hadamard rotation of one layer's activation (once per token per layer, shared by its experts)
{
std::vector<float> x = make_x(H, 5), s(H, 1.f);
const int it = 20000;
const double t0 = now_us();
for (int i = 0; i < it; ++i) hadamard_rotate(x.data(), H, 128, s.data());
std::printf("\nHadamard rotation of one %d-wide activation (block 128): %.2f us\n", H, (now_us() - t0) / it);
}
return 0;
}
} // namespace
int main(int argc, char** argv) {
ggml_cpu_init();
if (GGML_TYPE_COUNT <= 146) {
std::fprintf(stderr, "this ggml has no trellis types (build with STRATA_GGML_DIR = the agention llama.cpp fork): skipped\n");
return 77; // ctest SKIP_RETURN_CODE: a plain ggml is not a failure (#3358)
}
if (argc >= 2 && std::strcmp(argv[1], "--bench") == 0) { // --bench [--quick] [--only <kernel name part>]
bool quick = false;
std::string only;
for (int i = 2; i < argc; ++i) {
if (std::strcmp(argv[i], "--quick") == 0) quick = true;
else if (std::strcmp(argv[i], "--only") == 0 && i + 1 < argc) only = argv[++i];
}
return bench(quick, only);
}
if (argc >= 3 && (std::strcmp(argv[1], "--synthetic") == 0 || std::strcmp(argv[1], "--synthetic-hadamard") == 0)) {
const std::string pair = argv[2];
const size_t sl = pair.find('/');
const int gu = type_by_name(pair.substr(0, sl)), dn = sl == std::string::npos ? gu : type_by_name(pair.substr(sl + 1));
if (!tq_supported(gu) || !tq_supported(dn)) { std::fprintf(stderr, "not a trellis pair: %s\n", argv[2]); return 2; }
const int f = synthetic(gu, dn, std::strcmp(argv[1], "--synthetic-hadamard") == 0);
std::printf("%d failures\n", f);
return f ? 1 : 0;
}
int f = 0;
for (bool had : {false, true}) {
f += synthetic(145, 146, had); // Gyro-S
f += synthetic(144, 144, had); // Gyro-M
}
std::printf("%d failures\n", f);
return f ? 1 : 0;
}