Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

ds4_gg_bench: --k512 calls the AVX-512 rows (iq512x_rows) directly against ggml-cpu's dot, bit for bit, and says whether the CPU and type allow them

Niko1221e752c65455e7
diff --git a/tests/dsv41/gg_dot_bench.cpp b/tests/dsv41/gg_dot_bench.cpp
index ba9dd4eee..75dd3df97 100644
--- a/tests/dsv41/gg_dot_bench.cpp
+++ b/tests/dsv41/gg_dot_bench.cpp
@@ -8,6 +8,8 @@
#include "ds4_gguf_cpu.hpp"
#include "ggml-cpu.h"
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/iq_avx512.hpp"
#include "ggml.h"
#include <algorithm>
@@ -122,7 +124,7 @@ static int run_expert(int tg, int td, int threads, long mb, int nt, int reps) {
int main(int argc, char** argv) {
int type = 18, threads = 32, nt = 1, reps = 3;
long mb = 3072;
- bool exact = false, expert = false;
+ bool exact = false, expert = false, k512 = false;
int dtype = -1;
std::string dump; // --dump FILE: ggml-cpu's results (nt x rows floats) written raw - compare two builds with cmp
for (int i = 1; i < argc; ++i) {
@@ -135,6 +137,7 @@ int main(int argc, char** argv) {
else if (a == "--reps") reps = (int) val();
else if (a == "--exact") exact = true;
else if (a == "--expert") expert = true;
+ else if (a == "--k512") k512 = true;
else if (a == "--dtype") dtype = (int) val();
else if (a == "--dump" && i + 1 < argc) dump = argv[++i];
}
@@ -173,6 +176,28 @@ int main(int argc, char** argv) {
gg_cpu_quant(f.gu_act, x.data(), n, a[(size_t) t].data());
ap[(size_t) t] = a[(size_t) t].data();
}
+ if (k512) { // --k512: Strata's AVX-512 rows (iq512x_rows) called directly against ggml-cpu's dot, bit for bit: no dispatcher in between
+ const bool cpu_ok = strata::kernels::cpu::cpu_avx512_ok(), sup = strata::kernels::cpu::iq512x_supported(type);
+ std::printf("k512: type %d nt %d: cpu_avx512_ok %d, iq512x_supported %d%s
+", type, nt, (int) cpu_ok, (int) sup,
+ cpu_ok && sup ? "" : " -> the AVX-512 rows cannot run here");
+ if (!cpu_ok || !sup) return 3;
+ const ggml_type_traits_cpu* tc = ggml_get_type_traits_cpu((ggml_type) type);
+ std::vector<std::vector<float>> r0v((size_t) nt, std::vector<float>((size_t) rows)), r1v = r0v;
+ std::vector<float*> q1;
+ for (int t = 0; t < nt; ++t) q1.push_back(r1v[(size_t) t].data());
+ for (long r = 0; r < rows; ++r)
+ for (int t = 0; t < nt; ++t) tc->vec_dot(n, &r0v[(size_t) t][(size_t) r], 0, w.data() + (size_t) r * rb, 0, ap[(size_t) t], 0, 1);
+ strata::kernels::cpu::iq512x_rows(type, w.data(), rb, n, ap.data(), nt, q1.data(), 0, (int) rows);
+ long dif = 0;
+ for (int t = 0; t < nt; ++t)
+ for (long r = 0; r < rows; ++r)
+ if (std::memcmp(&r0v[(size_t) t][(size_t) r], &r1v[(size_t) t][(size_t) r], 4) != 0) ++dif;
+ std::printf("k512: iq512x_rows against ggml-cpu's dot: %ld of %ld differ in bits
+%s
+", dif, (long) nt * rows, dif == 0 ? "OK" : "FAIL");
+ return dif == 0 ? 0 : 1;
+ }
std::vector<std::vector<float>> o0((size_t) nt, std::vector<float>((size_t) rows)), o1 = o0;
std::vector<float*> p0, p1;
for (int t = 0; t < nt; ++t) { p0.push_back(o0[(size_t) t].data()); p1.push_back(o1[(size_t) t].data()); }