Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

kq_cpu_bench: i-quant pairs no longer crash, and Q2_0 is timed on the CPU pool's path (#3871)

Heinz N. Gies5bc1e246965c
diff --git a/src/kernels/cpu/kq_cpu_bench.cpp b/src/kernels/cpu/kq_cpu_bench.cpp
index 0655bc0ea..3a2f9b8d3 100644
--- a/src/kernels/cpu/kq_cpu_bench.cpp
+++ b/src/kernels/cpu/kq_cpu_bench.cpp
@@ -53,19 +53,23 @@ std::vector<std::string> split(const std::string& s) {
}
// one expert blob in the GGUF form: gate rows, up rows (n_ff x n_embd each), down rows (n_embd x n_ff), from random
-// weights quantized by ggml-cpu (valid scales, as a real file has)
+// weights quantized by ggml (valid scales, as a real file has). ggml_quantize_chunk, not the CPU traits' from_float:
+// ggml-cpu has no from_float for the i-quants (IQ3_S called a null pointer: an access violation on Windows)
void make_blob(const cpu::NativeFmt& f, uint8_t* blob, std::mt19937& rng) {
std::normal_distribution<float> nd(0.f, 0.02f);
std::vector<float> row((size_t) std::max(kH, kFF));
- const auto* tg = ggml_get_type_traits_cpu((ggml_type) f.gu_type);
- const auto* td = ggml_get_type_traits_cpu((ggml_type) f.d_type);
+ const std::vector<float> imatrix((size_t) std::max(kH, kFF), 1.0f);
+ auto quant = [&](int type, uint8_t* dst, int64_t n) {
+ ggml_quantize_chunk((ggml_type) type, row.data(), dst, 0, 1, n,
+ ggml_quantize_requires_imatrix((ggml_type) type) ? imatrix.data() : nullptr);
+ };
for (int64_t r = 0; r < 2 * kFF; ++r) {
for (int64_t i = 0; i < kH; ++i) row[(size_t) i] = nd(rng);
- tg->from_float(row.data(), blob + (size_t) r * f.gu_row, kH);
+ quant(f.gu_type, blob + (size_t) r * f.gu_row, kH);
}
for (int64_t r = 0; r < kH; ++r) {
for (int64_t i = 0; i < kFF; ++i) row[(size_t) i] = nd(rng);
- td->from_float(row.data(), blob + f.down_off + (size_t) r * f.d_row, kFF);
+ quant(f.d_type, blob + f.down_off + (size_t) r * f.d_row, kFF);
}
}
@@ -123,16 +127,36 @@ int main(int argc, char** argv) {
ha(kMaxT, std::vector<uint8_t>(cpu::kNativeHBytes));
const void* act[kMaxT];
const void* hq[kMaxT];
+ // a Q2_0 down takes the pool's own path (pool.cpp, run_split_multi_native and drain mode 6): the activation
+ // as an ActQ (act_quant_any) and Strata's Q2_0 rows (q2_rows_any), not ggml-cpu's Q2_0 dot
+ const bool q2_down = cpu::q2_native_kernels(f.d_type);
+ std::vector<cpu::ActQ> h2(q2_down ? kMaxT : 0);
+ const cpu::ActQ* a2[kMaxT] = {};
+ // a Q2_0 gate/up the same (pool.cpp mode 5): x as an ActQ, gate and up on q2_rows_any, then SwiGLU
+ const bool q2_gu = cpu::q2_native_kernels(f.gu_type);
+ std::vector<cpu::ActQ> x2(q2_gu ? kMaxT : 0);
+ const cpu::ActQ* ax[kMaxT] = {};
for (int t = 0; t < kMaxT; ++t) {
std::vector<float> x(kH), h(kFF);
for (auto& v : x) v = nd(rng);
for (auto& v : h) v = 0.1f * nd(rng);
- cpu::native_quant_act(f, x.data(), xa[t].data());
- cpu::native_quant_h(f, h.data(), ha[t].data());
+ if (q2_gu) {
+ cpu::act_quant_any(x.data(), (int) kH, x2[(size_t) t]);
+ ax[t] = &x2[(size_t) t];
+ } else {
+ cpu::native_quant_act(f, x.data(), xa[t].data());
+ }
+ if (q2_down) {
+ cpu::act_quant_any(h.data(), (int) kFF, h2[(size_t) t]);
+ a2[t] = &h2[(size_t) t];
+ } else {
+ cpu::native_quant_h(f, h.data(), ha[t].data());
+ }
act[t] = xa[t].data();
hq[t] = ha[t].data();
}
- std::printf("\n%s: %zu experts of %.3f MB (%.0f MB)\n", pair.c_str(), E, f.bytes / 1e6, E * (double) f.bytes / 1e6);
+ std::printf("\n%s: %zu experts of %.3f MB (%.0f MB)%s%s\n", pair.c_str(), E, f.bytes / 1e6, E * (double) f.bytes / 1e6,
+ q2_gu ? ", Q2_0 gate/up" : "", q2_down ? ", Q2_0 down on Strata's rows (the pool's path)" : "");
for (int nt : nts) {
for (int T : threads) {
if (T < 1) continue;
@@ -141,6 +165,11 @@ int main(int argc, char** argv) {
std::atomic<size_t> next{0};
auto work = [&] {
std::vector<float> ffb((size_t) kMaxT * kFF), outb((size_t) kMaxT * kH);
+ std::vector<float> gb(q2_gu ? (size_t) kMaxT * kFF : 0), ub(q2_gu ? (size_t) kMaxT * kFF : 0);
+ float* gp[kMaxT] = {};
+ float* up[kMaxT] = {};
+ if (q2_gu)
+ for (int t = 0; t < kMaxT; ++t) { gp[t] = &gb[(size_t) t * kFF]; up[t] = &ub[(size_t) t * kFF]; }
float* ff[kMaxT];
float* out[kMaxT];
for (int t = 0; t < kMaxT; ++t) { ff[t] = &ffb[(size_t) t * kFF]; out[t] = &outb[(size_t) t * kH]; }
@@ -149,8 +178,19 @@ int main(int argc, char** argv) {
const size_t e = next.fetch_add(1);
if (e >= E) break;
const uint8_t* blob = blobs + e * f.bytes;
- cpu::native_gu_rows(f, blob, act, nt, ff, 0, (int) kFF);
- cpu::native_down_rows(f, blob, hq, nt, out, 0, (int) kH);
+ if (q2_gu) {
+ cpu::q2_rows_any(blob, f.gu_row, (int) (kH / 64), ax, nt, gp, 0, (int) kFF);
+ cpu::q2_rows_any(blob + f.up_off, f.gu_row, (int) (kH / 64), ax, nt, up, 0, (int) kFF);
+ for (int t = 0; t < nt; ++t)
+ for (int64_t r = 0; r < kFF; ++r)
+ ff[t][r] = (gp[t][r] / (1.f + std::exp(-gp[t][r]))) * up[t][r];
+ } else {
+ cpu::native_gu_rows(f, blob, act, nt, ff, 0, (int) kFF);
+ }
+ if (q2_down)
+ cpu::q2_rows_any(blob + f.down_off, f.d_row, (int) (f.n_ff / 64), a2, nt, out, 0, (int) kH);
+ else
+ cpu::native_down_rows(f, blob, hq, nt, out, 0, (int) kH);
}
volatile float sink = outb[0] + ffb[0];
(void) sink;