Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

APR port, stage 3: Hadamard-folded experts on the CPU path

Laurent Zuijdwijkf47b0385f320
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 72a13be81..8295506ad 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1245,7 +1245,7 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)
endif()
add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp
- src/kernels/cpu/expert_layout.cpp src/kernels/cpu/q2_avx2.cpp)
+ src/kernels/cpu/expert_layout.cpp src/kernels/cpu/q2_avx2.cpp src/kernels/cpu/hadamard.cpp)
target_include_directories(strata_kernels_cpu PUBLIC include)
# Plan v0.3 P6: only the kernel files get the AVX-512 / AVX2 flags; the pool and the probe run on any x86-64,
# and the engine picks the kernels at run time (cpu_avx512_ok).
@@ -1786,6 +1786,15 @@ if(STRATA_BUILD_TESTS)
set_tests_properties(q2_bitplane_parity PROPERTIES ENVIRONMENT "STRATA_Q2_BITPLANE=1")
endif()
+ if(STRATA_NATIVE_EXPERTS)
+ # APR port: a Hadamard-folded trellis expert (TQ2_T / TQK6 / TQK7) on the CPU path vs llama.cpp's graph for the
+ # same weights (synthetic; needs STRATA_GGML_DIR = a ggml with the trellis types, else it reports and fails)
+ add_executable(native_hadamard_test src/kernels/cpu/native_hadamard_test.cpp)
+ target_link_libraries(native_hadamard_test PRIVATE strata_kernels_cpu strata_artifact)
+ add_test(NAME native_hadamard_test COMMAND native_hadamard_test)
+ set_tests_properties(native_hadamard_test PROPERTIES TIMEOUT 300)
+ endif()
+
add_executable(expert_parity src/kernels/cpu/expert_parity.cpp)
target_link_libraries(expert_parity PRIVATE strata_kernels_cpu)
# The pack is a REQUIRED input, not an optional one: this test compares against a real expert blob, and a
diff --git a/include/strata/kernels/cpu/expert_layout.hpp b/include/strata/kernels/cpu/expert_layout.hpp
index 6b3fddb66..b78528c77 100644
--- a/include/strata/kernels/cpu/expert_layout.hpp
+++ b/include/strata/kernels/cpu/expert_layout.hpp
@@ -96,6 +96,8 @@ const ExpertLayout& expert_layout();
bool expert_layout_load(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err);
/// The newest native_experts.txt this engine reads. v4 = v3 plus the per-role shard column `gate,up,down`,
/// written only when some layer's roles are in different shards (every other pack stays v3, byte for byte).
-inline constexpr int kExpertLayoutVersion = 4;
+/// v5 = v4 plus `hadamard.txt` beside it: Hadamard-folded experts (hadamard.hpp), written only for such models,
+/// so an engine that cannot rotate the activations refuses the pack instead of running it wrong.
+inline constexpr int kExpertLayoutVersion = 5;
} // namespace strata::kernels::cpu
diff --git a/include/strata/kernels/cpu/hadamard.hpp b/include/strata/kernels/cpu/hadamard.hpp
new file mode 100644
index 000000000..43b3fcfce
--- /dev/null
+++ b/include/strata/kernels/cpu/hadamard.hpp
@@ -0,0 +1,51 @@
+// include/strata/kernels/cpu/hadamard.hpp - the activation-side transform of Hadamard-folded experts.
+//
+// Some GGUFs (Agention's APR "Gyro" files) store their routed experts in a block-Hadamard-rotated basis: the
+// encoder replaced each expert weight W by W' = W * diag(s) * H_b^T per block of b input columns, where H_b is
+// the normalized Sylvester-Walsh-Hadamard matrix (entry (-1)^popcount(r & c) / sqrt(b)) and s a +/-1 sign
+// vector of the input width. The GGUF says so in its `prism.hadamard.*` metadata. Since H_b is orthonormal and
+// symmetric, W x = W' (H_b (s * x)): the runtime must apply y = H_b (s * x), per block of b values, to every
+// activation that enters a folded weight - the gate/up input x and the down input h - before quantizing it.
+// Skipping it on any path (CPU or GPU, decode or prompt) gives plausible-looking garbage, so a pack that carries
+// rotated experts is refused by an engine path that cannot apply it.
+//
+// tools/iq_pack.py reads the metadata and writes `<pack>/hadamard.txt` (and native_experts.txt v5, so an engine
+// older than this refuses the pack). The format:
+//
+// # strata hadamard v1 ... (comment lines start with #)
+// block 128 the rotation block (a power of two)
+// signs <width> <width x +1/-1> one line per sign-vector width (none = identity signs)
+// layer <l> <gate 0|1> <up 0|1> <down 0|1> which roles of layer l are folded (gate == up: one input)
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace strata::kernels::cpu {
+
+struct HadamardSpec {
+ int block = 0; ///< 0: no Hadamard-folded experts
+ std::map<int, std::vector<float>> signs; ///< width -> +1/-1 per input value (empty: identity signs)
+ std::vector<uint8_t> roles; ///< per layer: bit 0 gate, bit 1 up, bit 2 down
+ bool any() const { return block > 0; }
+ bool gu(int64_t layer) const { return block > 0 && (roles[(size_t) layer] & 3u) != 0; }
+ bool down(int64_t layer) const { return block > 0 && (roles[(size_t) layer] & 4u) != 0; }
+};
+
+/// Parses `path` (hadamard.txt) for a model of `n_layers` layers. False with a reason when it is malformed.
+bool hadamard_parse(const std::string& path, int64_t n_layers, HadamardSpec& h, std::string& err);
+
+/// The process-wide spec (set by expert_layout_load from the pack; empty for every other pack).
+const HadamardSpec& hadamard_spec();
+void hadamard_set(HadamardSpec h);
+/// The sign vector of `width` in the process-wide spec, or nullptr (identity signs). The pointer stays valid
+/// until the next hadamard_set.
+const float* hadamard_signs(int width);
+
+/// In place: x[i] *= signs[i] (when signs is not null), then each block of `block` values -> H_block x / sqrt(block).
+/// `n` must be a multiple of `block`.
+void hadamard_rotate(float* x, int n, int block, const float* signs);
+
+} // namespace strata::kernels::cpu
diff --git a/include/strata/kernels/cpu/native_expert.hpp b/include/strata/kernels/cpu/native_expert.hpp
index 895f90c20..fbac35052 100644
--- a/include/strata/kernels/cpu/native_expert.hpp
+++ b/include/strata/kernels/cpu/native_expert.hpp
@@ -32,6 +32,12 @@ struct NativeFmt {
/// projection's FP32 output: gate * s_gate, up * s_up, down * s_down. Do not fold s_down into up: the hidden
/// drops to ~1e-5 and its q8 block scale becomes an FP16 subnormal (expert error 2-12% instead of 1.1%).
size_t tail_off = 0;
+ /// prism.hadamard (hadamard.hpp): the experts of this layer are stored in a rotated basis, and
+ /// native_quant_act / native_quant_h apply y = H_block (signs * x) before quantizing. had_block 0 = none.
+ int had_block = 0;
+ bool had_gu = false, had_d = false;
+ const float* gu_signs = nullptr; ///< n_embd signs for x (nullptr: identity)
+ const float* d_signs = nullptr; ///< n_ff signs for h (nullptr: identity)
};
constexpr int kNvfp4Type = 40;
constexpr size_t kNvfp4Tail = 16;
@@ -41,9 +47,9 @@ bool native_experts_available() noexcept;
/// Fills `f` for a layer; false (with a reason) when ggml-cpu has no dot product for a type.
bool native_fmt(int gu_type, int d_type, int64_t n_embd, int64_t n_ff, NativeFmt& f, std::string& err);
-/// x (n_embd floats) -> the gate/up activation (act_bytes).
+/// x (n_embd floats) -> the gate/up activation (act_bytes), rotated first when the layer is Hadamard-folded.
void native_quant_act(const NativeFmt& f, const float* x, void* dst);
-/// h (n_ff floats) -> the down activation (h_bytes).
+/// h (n_ff floats) -> the down activation (h_bytes), rotated first when the layer is Hadamard-folded.
void native_quant_h(const NativeFmt& f, const float* h, void* dst);
/// From how many tokens native_gu_rows gives this gate/up type to a multi-token kernel (#152; ggml-cpu's per-token dot
diff --git a/src/kernels/cpu/expert_layout.cpp b/src/kernels/cpu/expert_layout.cpp
index 001fa5d47..b6924950d 100644
--- a/src/kernels/cpu/expert_layout.cpp
+++ b/src/kernels/cpu/expert_layout.cpp
@@ -1,5 +1,6 @@
// src/kernels/cpu/expert_layout.cpp - plan v0.3 P6: the per-layer expert table. See the header.
#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/hadamard.hpp"
#include <atomic>
#include <cstdio>
@@ -391,6 +392,7 @@ bool expert_layout_load(const std::string& pack_dir, int64_t n_layers, int64_t n
std::ifstream in(pack_dir + "/native_experts.txt");
if (!in) {
L.total = (uint64_t) n_layers * (uint64_t) n_expert * (uint64_t) BLOB;
+ hadamard_set({});
g_layout = L;
return true;
}
@@ -487,6 +489,48 @@ bool expert_layout_load(const std::string& pack_dir, int64_t n_layers, int64_t n
at += L.bytes[(size_t) l] * (uint64_t) L.n_expert;
}
L.total = at;
+ // v5: Agention's Hadamard-folded experts (prism.hadamard, hadamard.hpp). The pack must say which layers are
+ // rotated and with what signs; a v5 pack without hadamard.txt (or the reverse) is refused, never run unrotated.
+ {
+ std::ifstream probe(pack_dir + "/hadamard.txt");
+ const bool have = (bool) probe;
+ if (L.version >= 5 && !have) {
+ err = "native_experts.txt v5 declares Hadamard-folded experts, but " + pack_dir + "/hadamard.txt is missing";
+ return false;
+ }
+ if (have && L.version < 5) {
+ err = pack_dir + "/hadamard.txt exists but native_experts.txt is v" + std::to_string(L.version) +
+ ": the pack is inconsistent (repack with tools/iq_pack.py)";
+ return false;
+ }
+ HadamardSpec h;
+ if (have && !hadamard_parse(pack_dir + "/hadamard.txt", n_layers, h, err)) return false;
+ hadamard_set(std::move(h));
+ const HadamardSpec& hs = hadamard_spec();
+ for (int64_t l = 0; hs.any() && l < n_layers; ++l) {
+ NativeFmt& f = L.fmt[(size_t) l];
+ f.had_block = hs.block;
+ f.had_gu = hs.gu(l);
+ f.had_d = hs.down(l);
+ if ((f.had_gu && f.n_embd % hs.block) || (f.had_d && f.n_ff % hs.block)) {
+ err = "hadamard.txt: block " + std::to_string(hs.block) + " does not divide layer " +
+ std::to_string(l) + "'s expert widths";
+ return false;
+ }
+ // the Q2_0 rows quantize their activations with Strata's own act_quant_any, which does not rotate
+ if ((f.had_gu && f.gu_type == 42) || (f.had_d && f.d_type == 42)) {
+ err = "hadamard.txt: layer " + std::to_string(l) + " folds a Q2_0 expert, which this engine cannot rotate";
+ return false;
+ }
+ f.gu_signs = f.had_gu ? hadamard_signs((int) f.n_embd) : nullptr;
+ f.d_signs = f.had_d ? hadamard_signs((int) f.n_ff) : nullptr;
+ // explicit signs are per width: a width without a vector would silently mean identity signs
+ if (!hs.signs.empty() && ((f.had_gu && !f.gu_signs) || (f.had_d && !f.d_signs))) {
+ err = "hadamard.txt: no sign vector for layer " + std::to_string(l) + "'s expert width";
+ return false;
+ }
+ }
+ }
g_layout = L;
return true;
#endif
diff --git a/src/kernels/cpu/hadamard.cpp b/src/kernels/cpu/hadamard.cpp
new file mode 100644
index 000000000..3a9fc0c7f
--- /dev/null
+++ b/src/kernels/cpu/hadamard.cpp
@@ -0,0 +1,110 @@
+// src/kernels/cpu/hadamard.cpp - see include/strata/kernels/cpu/hadamard.hpp.
+#include "strata/kernels/cpu/hadamard.hpp"
+
+#include <cmath>
+#include <cstdlib>
+#include <fstream>
+#include <sstream>
+
+namespace strata::kernels::cpu {
+namespace {
+HadamardSpec g_spec;
+}
+
+const HadamardSpec& hadamard_spec() { return g_spec; }
+void hadamard_set(HadamardSpec h) { g_spec = std::move(h); }
+
+const float* hadamard_signs(int width) {
+ const auto it = g_spec.signs.find(width);
+ return it == g_spec.signs.end() ? nullptr : it->second.data();
+}
+
+bool hadamard_parse(const std::string& path, int64_t n_layers, HadamardSpec& h, std::string& err) {
+ std::ifstream in(path);
+ if (!in) { err = "cannot read " + path; return false; }
+ h = HadamardSpec{};
+ h.roles.assign((size_t) n_layers, 0);
+ std::vector<uint8_t> seen((size_t) n_layers, 0);
+ std::string line;
+ bool versioned = false;
+ while (std::getline(in, line)) {
+ if (line.empty()) continue;
+ if (line[0] == '#') {
+ static const char tag[] = "# strata hadamard v";
+ if (line.compare(0, sizeof tag - 1, tag) == 0) {
+ if (std::atoi(line.c_str() + sizeof tag - 1) != 1) {
+ err = path + ": an unknown hadamard.txt version (this engine reads v1)";
+ return false;
+ }
+ versioned = true;
+ }
+ continue;
+ }
+ std::istringstream ss(line);
+ std::string key;
+ ss >> key;
+ if (key == "block") {
+ long long b = 0;
+ if (!(ss >> b) || b < 2 || b > 4096 || (b & (b - 1)) != 0 || h.block) {
+ err = path + ": a bad or repeated block line: " + line;
+ return false;
+ }
+ h.block = (int) b;
+ } else if (key == "signs") {
+ long long w = 0;
+ if (!(ss >> w) || w <= 0 || w > (1 << 20) || h.signs.count((int) w)) {
+ err = path + ": a bad or repeated signs line";
+ return false;
+ }
+ std::vector<float> v((size_t) w);
+ for (auto& x : v) {
+ int s = 0;
+ if (!(ss >> s) || (s != 1 && s != -1)) { err = path + ": sign values must be +1 or -1"; return false; }
+ x = (float) s;
+ }
+ std::string extra;
+ if (ss >> extra) { err = path + ": a signs line longer than its width"; return false; }
+ h.signs.emplace((int) w, std::move(v));
+ } else if (key == "layer") {
+ long long l = -1;
+ int g = -1, u = -1, d = -1;
+ if (!(ss >> l >> g >> u >> d) || l < 0 || l >= n_layers || (g | u | d) & ~1 || seen[(size_t) l]) {
+ err = path + ": a bad or repeated layer line: " + line;
+ return false;
+ }
+ if (g != u) { // gate and up read one activation: a transform for one of them only is not runnable
+ err = path + ": layer " + std::to_string(l) + " folds only one of gate/up";
+ return false;
+ }
+ seen[(size_t) l] = 1;
+ h.roles[(size_t) l] = (uint8_t) (g | (u << 1) | (d << 2));
+ } else {
+ err = path + ": an unknown line: " + line;
+ return false;
+ }
+ }
+ if (!versioned || h.block == 0) { err = path + ": no version header or no block line"; return false; }
+ for (const auto& [w, v] : h.signs)
+ if (w % h.block) { err = path + ": sign width " + std::to_string(w) + " is not whole blocks"; return false; }
+ return true;
+}
+
+void hadamard_rotate(float* x, int n, int block, const float* signs) {
+ if (signs)
+ for (int i = 0; i < n; ++i) x[i] *= signs[i];
+ const float scale = 1.0f / std::sqrt((float) block);
+ for (int b0 = 0; b0 < n; b0 += block) {
+ float* v = x + b0;
+ // the in-place fast Walsh-Hadamard transform in natural (Sylvester) order: entry (-1)^popcount(r & c)
+ for (int len = 1; len < block; len <<= 1)
+ for (int i = 0; i < block; i += len << 1)
+ for (int j = i; j < i + len; ++j) {
+ const float a = v[j], c = v[j + len];
+ v[j] = a + c;
+ v[j + len] = a - c;
+ }
+ for (int i = 0; i < block; ++i) v[i] *= scale;
+ }
+}
+
+} // namespace strata::kernels::cpu
diff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cpp
index 1ab75b642..7ee0e36aa 100644
--- a/src/kernels/cpu/native_expert.cpp
+++ b/src/kernels/cpu/native_expert.cpp
@@ -10,6 +10,7 @@
#include "strata/kernels/cpu/nvfp4_avx2.hpp"
#include "strata/kernels/cpu/nvfp4_avx512.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/hadamard.hpp"
#include "ggml.h"
#include "ggml-cpu.h"
@@ -18,6 +19,7 @@
#include <cstdlib>
#include <cstring>
#include <mutex>
+#include <vector>
namespace strata::kernels::cpu {
namespace {
@@ -110,12 +112,24 @@ bool q8k_avx1(int type) {
} // namespace
void native_quant_act(const NativeFmt& f, const float* x, void* dst) {
+ if (f.had_gu) { // Hadamard-folded gate/up: quantize H (s * x), never x itself (hadamard.hpp)
+ thread_local std::vector<float> tmp;
+ tmp.assign(x, x + f.n_embd);
+ hadamard_rotate(tmp.data(), (int) f.n_embd, f.had_block, f.gu_signs);
+ x = tmp.data();
+ }
if (q8k_avx2(f.gu_act)) { q8k_quant_avx2(x, dst, f.n_embd); return; }
if (q8k_avx1(f.gu_act)) { q8k_quant_avx1(x, dst, f.n_embd); return; }
traits(f.gu_act)->from_float(x, dst, f.n_embd);
}
void native_quant_h(const NativeFmt& f, const float* h, void* dst) {
+ if (f.had_d) {
+ thread_local std::vector<float> tmp;
+ tmp.assign(h, h + f.n_ff);
+ hadamard_rotate(tmp.data(), (int) f.n_ff, f.had_block, f.d_signs);
+ h = tmp.data();
+ }
if (q8k_avx2(f.d_act)) { q8k_quant_avx2(h, dst, f.n_ff); return; }
if (q8k_avx1(f.d_act)) { q8k_quant_avx1(h, dst, f.n_ff); return; }
traits(f.d_act)->from_float(h, dst, f.n_ff);
diff --git a/src/kernels/cpu/native_hadamard_test.cpp b/src/kernels/cpu/native_hadamard_test.cpp
new file mode 100644
index 000000000..de5b923f1
--- /dev/null
+++ b/src/kernels/cpu/native_hadamard_test.cpp
@@ -0,0 +1,311 @@
+// src/kernels/cpu/native_hadamard_test.cpp - APR port, stage 3: one Hadamard-folded trellis expert (TQ2_T / TQK6 /
+// TQK7, ggml types 144-146) on Strata's CPU path against llama.cpp's own graph for the same weights and input.
+//
+// Strata's path: native_quant_act (signs, FWHT, Q8_0) -> native_gu_rows (ggml-cpu vec_dot + SwiGLU)
+// -> native_quant_h (signs, FWHT, Q8_0) -> native_down_rows
+// The reference: the ops llama.cpp's build_lora_mm_id builds for a prism.hadamard weight - ggml_mul(x, signs),
+// llama_mul_mat_hadamard (reshape to [128, n/128], ggml_mul_mat(rot, .)), ggml_mul_mat(W', .) -
+// computed by ggml-cpu from the same checkout (STRATA_GGML_DIR = the agention llama.cpp fork).
+// The rotation itself is computed two different ways (FWHT here, a dense 128x128 matmul there), so the two outputs
+// agree to float rounding plus the odd Q8_0 rounding flip, not bit for bit; the test bounds the relative error.
+//
+// native_hadamard_test synthetic: random W, rotated and quantized here; also checks the
+// folded expert against the UNROTATED float expert (W x) and that
+// skipping the rotation is caught
+// native_hadamard_test --pack <dir> --gguf <file> [layer expert]... real experts of a prism.hadamard GGUF
+// (Gyro-S / Gyro-M) through the pack tools/iq_pack.py made of it
+#include "strata/kernels/cpu/hadamard.hpp"
+#include "strata/kernels/cpu/native_expert.hpp"
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/pool.hpp"
+#include "strata/artifact/gguf_reader.hpp"
+
+#include "ggml.h"
+#include "ggml-cpu.h"
+
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <algorithm>
+#include <random>
+#include <string>
+#include <vector>
+
+using namespace strata::kernels::cpu;
+
+namespace {
+
+constexpr int H = 2560, FF = 640, B = 128;
+
+struct Expert {
+ int gu_type = 0, d_type = 0;
+ std::vector<uint8_t> gate, up, down; // ggml rows: gate/up [FF rows of H], down [H rows of FF]
+ std::vector<float> s_h, s_ff; // sign vectors (empty: identity)
+};
+
+double rel(const std::vector<float>& a, const std::vector<float>& b) {
+ double num = 0, den = 0;
+ for (size_t i = 0; i < a.size(); ++i) {
+ num += (double) (a[i] - b[i]) * (a[i] - b[i]);
+ den += (double) b[i] * b[i];
+ }
+ return std::sqrt(num / (den > 0 ? den : 1));
+}
+
+float silu(float g) { return g / (1.f + std::exp(-g)); }
+
+// Strata's CPU path for one token; `loaded` = the layer's NativeFmt as expert_layout_load made it from a pack
+std::vector<float> strata_expert(const Expert& e, const std::vector<float>& x, bool rotate,
+ const NativeFmt* loaded = nullptr) {
+ NativeFmt f;
+ std::string err;
+ if (!native_fmt(e.gu_type, e.d_type, H, FF, f, err)) { std::fprintf(stderr, "native_fmt: %s\n", err.c_str()); std::exit(1); }
+ if (rotate && loaded) {
+ f = *loaded;
+ } else if (rotate) {
+ f.had_block = B;
+ f.had_gu = f.had_d = true;
+ f.gu_signs = e.s_h.empty() ? nullptr : e.s_h.data();
+ f.d_signs = e.s_ff.empty() ? nullptr : e.s_ff.data();
+ }
+ std::vector<uint8_t> blob(f.bytes);
+ std::memcpy(blob.data(), e.gate.data(), e.gate.size());
+ std::memcpy(blob.data() + f.up_off, e.up.data(), e.up.size());
+ std::memcpy(blob.data() + f.down_off, e.down.data(), e.down.size());
+ std::vector<uint8_t> act(kNativeActBytes), hq(kNativeHBytes);
+ std::vector<float> ff(FF), out(H);
+ native_quant_act(f, x.data(), act.data());
+ const void* a[1] = {act.data()};
+ float* ffp[1] = {ff.data()};
+ native_gu_rows(f, blob.data(), a, 1, ffp, 0, FF);
+ native_quant_h(f, ff.data(), hq.data());
+ const void* hp[1] = {hq.data()};
+ float* op[1] = {out.data()};
+ native_down_rows(f, blob.data(), hp, 1, op, 0, H);
+ return out;
+}
+
+// llama.cpp's graph for the same expert (the fork's build_lora_mm + llama_mul_mat_hadamard), one token
+std::vector<float> llama_expert(const Expert& e, const std::vector<float>& x) {
+ ggml_init_params ip = {(size_t) 256 << 20, nullptr, false};
+ ggml_context* ctx = ggml_init(ip);
+ // the normalized Sylvester-Walsh-Hadamard matrix exactly as llama_model_base builds prism.hadamard.128
+ ggml_tensor* rot = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, B, B);
+ float* R = (float*) rot->data;
+ const float scale = 1.0f / sqrtf((float) B);
+ for (uint32_t r = 0; r < (uint32_t) B; ++r)
+ for (uint32_t c = 0; c < (uint32_t) B; ++c) {
+ uint32_t p = r & c;
+ p ^= p >> 16; p ^= p >> 8; p ^= p >> 4; p ^= p >> 2; p ^= p >> 1;
+ R[r * B + c] = (p & 1) ? -scale : scale;
+ }
+ auto hadamard = [&](ggml_tensor* cur, const std::vector<float>& signs) {
+ if (!signs.empty()) {
+ ggml_tensor* s = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, (int64_t) signs.size());
+ std::memcpy(s->data, signs.data(), signs.size() * sizeof(float));
+ cur = ggml_mul(ctx, cur, s);
+ }
+ ggml_tensor* res = ggml_reshape_2d(ctx, cur, B, ggml_nelements(cur) / B);
+ res = ggml_mul_mat(ctx, rot, res);
+ return ggml_reshape_4d(ctx, res, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3]);
+ };
+ ggml_tensor* wg = ggml_new_tensor_2d(ctx, (ggml_type) e.gu_type, H, FF);
+ ggml_tensor* wu = ggml_new_tensor_2d(ctx, (ggml_type) e.gu_type, H, FF);
+ ggml_tensor* wd = ggml_new_tensor_2d(ctx, (ggml_type) e.d_type, FF, H);
+ std::memcpy(wg->data, e.gate.data(), e.gate.size());
+ std::memcpy(wu->data, e.up.data(), e.up.size());
+ std::memcpy(wd->data, e.down.data(), e.down.size());
+ ggml_tensor* xt = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, H);
+ std::memcpy(xt->data, x.data(), H * sizeof(float));
+ ggml_tensor* xr = hadamard(xt, e.s_h);
+ ggml_tensor* g = ggml_mul_mat(ctx, wg, xr);
+ ggml_tensor* u = ggml_mul_mat(ctx, wu, xr);
+ ggml_tensor* h = ggml_mul(ctx, ggml_silu(ctx, g), u);
+ ggml_tensor* out = ggml_mul_mat(ctx, wd, hadamard(h, e.s_ff));
+ ggml_cgraph* gf = ggml_new_graph(ctx);
+ ggml_build_forward_expand(gf, out);
+ ggml_graph_compute_with_ctx(ctx, gf, 4);
+ std::vector<float> y(H);
+ std::memcpy(y.data(), out->data, H * sizeof(float));
+ ggml_free(ctx);
+ return y;
+}
+
+// W' = W diag(s) H^T per 128-column block (what the APR encoder folds), then quantized with ggml
+std::vector<uint8_t> fold_and_quantize(const std::vector<float>& w, int rows, int cols, const std::vector<float>& s,
+ int type, std::vector<float>& folded) {
+ folded.assign(w.begin(), w.end());
+ for (int r = 0; r < rows; ++r) hadamard_rotate(folded.data() + (size_t) r * cols, cols, B, s.data());
+ // (H symmetric and orthonormal: row-wise H (s * w_r) is exactly the fold W diag(s) H^T)
+ std::vector<uint8_t> q(ggml_row_size((ggml_type) type, cols) * rows);
+ ggml_quantize_chunk((ggml_type) type, folded.data(), q.data(), 0, rows, cols, nullptr);
+ return q;
+}
+
+int synthetic(int gu_type, int d_type) {
+ std::mt19937 rng(1234 + gu_type * 7 + d_type);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ std::uniform_int_distribution<int> coin(0, 1);
+ Expert e;
+ e.gu_type = gu_type;
+ e.d_type = d_type;
+ std::vector<float> wg((size_t) FF * H), wu((size_t) FF * H), wd((size_t) H * FF), x(H);
+ for (auto& v : wg) v = nd(rng) * 0.02f;
+ for (auto& v : wu) v = nd(rng) * 0.02f;
+ for (auto& v : wd) v = nd(rng) * 0.02f;
+ for (auto& v : x) v = nd(rng);
+ x[17] = 40.f; // a massive channel, as the real residual stream has: the case the rotation exists for
+ e.s_h.resize(H);
+ e.s_ff.resize(FF);
+ for (auto& v : e.s_h) v = coin(rng) ? 1.f : -1.f;
+ for (auto& v : e.s_ff) v = coin(rng) ? 1.f : -1.f;
+ std::vector<float> fg, fu, fd;
+ e.gate = fold_and_quantize(wg, FF, H, e.s_h, gu_type, fg);
+ e.up = fold_and_quantize(wu, FF, H, e.s_h, gu_type, fu);
+ e.down = fold_and_quantize(wd, H, FF, e.s_ff, d_type, fd);
+
+ // the unrotated float expert W x: what the folded, quantized expert must approximate
+ std::vector<float> h(FF), ref(H);
+ for (int r = 0; r < FF; ++r) {
+ double g = 0, u = 0;
+ for (int c = 0; c < H; ++c) { g += (double) wg[(size_t) r * H + c] * x[c]; u += (double) wu[(size_t) r * H + c] * x[c]; }
+ h[r] = silu((float) g) * (float) u;
+ }
+ for (int r = 0; r < H; ++r) {
+ double o = 0;
+ for (int c = 0; c < FF; ++c) o += (double) wd[(size_t) r * FF + c] * h[c];
+ ref[r] = (float) o;
+ }
+ const auto ours = strata_expert(e, x, true);
+ const auto llama = llama_expert(e, x);
+ const auto unrot = strata_expert(e, x, false);
+ const double e_ll = rel(ours, llama), e_ref = rel(ours, ref), e_llref = rel(llama, ref), e_un = rel(unrot, ref);
+ std::printf("synthetic %s/%s: strata vs llama.cpp graph %.2e | vs unrotated float expert: strata %.4f, llama.cpp "
+ "%.4f | strata WITHOUT the rotation %.4f\n", ggml_type_name((ggml_type) gu_type),
+ ggml_type_name((ggml_type) d_type), e_ll, e_ref, e_llref, e_un);
+ // strata == llama.cpp to rounding, and both approximate W x equally well. With Q8_0 weights the quantization
+ // error is ~1%, so the fold's convention itself is checked: the rotated path must reproduce the unrotated float
+ // expert and the unrotated path must not. (The trellis types' own error at 1.6-2.1 bpw on Gaussian weights,
+ // compounded through SwiGLU, is large with this test's placeholder ggml encoder; there only agreement counts.)
+ const bool hi = gu_type == 8 && d_type == 8;
+ const bool ok = e_ll < 2e-3 && std::fabs(e_ref - e_llref) < 0.01 && (!hi || (e_ref < 0.03 && e_un > 0.8));
+ std::printf(" %s\n", ok ? "ok" : "FAILED");
+ return ok ? 0 : 1;
+}
+
+// A real expert, the pack's loader path: expert_layout_load(<pack>) reads native_experts.txt v5 + hadamard.txt (from
+// tools/iq_pack.py), and the layer's NativeFmt is what the engine's CPU pool would use. (The signs come from the
+// pack: strata::GgufFile keeps only the first 64 items of a metadata array, so it cannot read sign_values.)
+int real(const char* pack, const char* path, int layer, int expert) {
+ std::string err;
+ if (!expert_layout_load(pack, 48, 512, err)) { std::fprintf(stderr, "expert_layout_load: %s\n", err.c_str()); return 1; }
+ const ExpertLayout& lay = expert_layout();
+ const HadamardSpec& hs = hadamard_spec();
+ if (!lay.native || !hs.any() || lay.version != 5) { std::fprintf(stderr, "%s: not a v5 (Hadamard) native pack\n", pack); return 1; }
+ const NativeFmt& lf = lay.fmt[(size_t) layer];
+ if (lf.had_block != B || !lf.had_gu || !lf.had_d) { std::fprintf(stderr, "layer %d is not folded in the pack\n", layer); return 1; }
+ strata::GgufFile g(path);
+ Expert e;
+ if (const float* sh = hadamard_signs(H)) e.s_h.assign(sh, sh + H);
+ if (const float* sf = hadamard_signs(FF)) e.s_ff.assign(sf, sf + FF);
+ auto slice = [&](const char* role, std::vector<uint8_t>& dst, int& type) {
+ const std::string name = "blk." + std::to_string(layer) + ".ffn_" + role + "_exps.weight";
+ const strata::TensorInfo* t = g.find(name);
+ if (!t || t->shape.size() != 3) { std::fprintf(stderr, "missing %s\n", name.c_str()); std::exit(1); }
+ type = (int) t->type;
+ const size_t per = ggml_row_size((ggml_type) type, (int64_t) t->shape[0]) * t->shape[1];
+ const uint8_t* p = g.tensor_data(*t) + per * (size_t) expert;
+ dst.assign(p, p + per);
+ };
+ int dt = 0, ut = 0;
+ slice("gate", e.gate, e.gu_type);
+ slice("up", e.up, ut);
+ slice("down", e.down, dt);
+ e.d_type = dt;
+ if (e.gu_type != lf.gu_type || e.d_type != lf.d_type) { std::fprintf(stderr, "the pack and the GGUF disagree\n"); return 1; }
+ std::mt19937 rng(99);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ std::vector<float> x(H);
+ for (auto& v : x) v = nd(rng);
+ x[17] = 40.f;
+ const auto ours = strata_expert(e, x, true, &lf);
+ const auto llama = llama_expert(e, x);
+ const auto unrot = strata_expert(e, x, false);
+ const double e_ll = rel(ours, llama), e_un = rel(unrot, llama);
+ std::printf("%s layer %d expert %d (%s/%s, signs %s): strata vs llama.cpp graph %.2e | without the rotation %.3f\n",
+ path, layer, expert, ggml_type_name((ggml_type) e.gu_type), ggml_type_name((ggml_type) e.d_type),
+ e.s_h.empty() ? "identity" : "explicit", e_ll, e_un);
+ bool ok = e_ll < 2e-3 && e_un > 0.3;
+ std::printf(" %s\n", ok ? "ok" : "FAILED");
+
+ // the engine's CPU entry point: ExpertPool::run_split_multi_native with the pack's NativeFmt, three tokens
+ // routed to this expert (x quantized by native_quant_act, as expert_source.cpp does for the doorbell copy)
+ {
+ constexpr int NT = 3;
+ ExpertPool pool(4, false, false);
+ std::vector<uint8_t> blob(lf.bytes);
+ std::memcpy(blob.data(), e.gate.data(), e.gate.size());
+ std::memcpy(blob.data() + lf.up_off, e.up.data(), e.up.size());
+ std::memcpy(blob.data() + lf.down_off, e.down.data(), e.down.size());
+ std::vector<std::vector<float>> xs(NT, std::vector<float>(H)), outs(NT, std::vector<float>(H));
+ std::vector<std::vector<uint8_t>> acts(NT, std::vector<uint8_t>(kNativeActBytes));
+ ExpertJobMulti job;
+ job.blob = blob.data();
+ job.nt = NT;
+ for (int t = 0; t < NT; ++t) {
+ for (auto& v : xs[t]) v = nd(rng);
+ xs[t][(size_t) (101 * t + 5)] = 30.f;
+ native_quant_act(lf, xs[t].data(), acts[t].data());
+ job.nact[t] = acts[t].data();
+ job.out[t] = outs[t].data();
+ }
+ pool.run_split_multi_native(lf, &job, 1);
+ double worst = 0;
+ for (int t = 0; t < NT; ++t) worst = std::max(worst, rel(outs[t], llama_expert(e, xs[t])));
+ const bool pok = worst < 2e-3;
+ std::printf(" ExpertPool::run_split_multi_native, %d tokens: worst vs llama.cpp graph %.2e %s\n", NT, worst,
+ pok ? "ok" : "FAILED");
+ ok = ok && pok;
+ }
+ return ok ? 0 : 1;
+}
+
+// the FWHT against the dense matrix llama.cpp builds, on one block
+int fwht_check() {
+ std::mt19937 rng(7);
+ std::normal_distribution<float> nd(0.f, 1.f);
+ std::vector<float> x(B), y(B), s(B);
+ for (auto& v : x) v = nd(rng);
+ for (int i = 0; i < B; ++i) s[i] = (i * 37 % 5) < 2 ? -1.f : 1.f;
+ y = x;
+ hadamard_rotate(y.data(), B, B, s.data());
+ double worst = 0;
+ for (int r = 0; r < B; ++r) {
+ double acc = 0;
+ for (int c = 0; c < B; ++c) acc += ((__builtin_popcount(r & c) & 1) ? -1.0 : 1.0) * s[c] * x[c];
+ worst = std::max(worst, std::fabs(acc / std::sqrt((double) B) - y[r]));
+ }
+ std::printf("FWHT vs dense normalized Sylvester matrix: max abs diff %.2e %s\n", worst, worst < 1e-5 ? "ok" : "FAILED");
+ return worst < 1e-5 ? 0 : 1;
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+ if (!native_experts_available()) { std::fprintf(stderr, "built without native experts\n"); return 1; }
+ int rc = fwht_check();
+ if (argc >= 5 && std::strcmp(argv[1], "--pack") == 0 && std::strcmp(argv[3], "--gguf") == 0) {
+ for (int i = 5; i + 1 < argc || i == 5; i += 2) { // [layer expert]...
+ const int layer = argc > i ? std::atoi(argv[i]) : 0, expert = argc > i + 1 ? std::atoi(argv[i + 1]) : 0;
+ rc |= real(argv[2], argv[4], layer, expert);
+ }
+ return rc;
+ }
+ // ggml type ids of the agention fork: 144 TQ2_T, 145 TQK6, 146 TQK7
+ if (GGML_TYPE_COUNT <= 146) { std::fprintf(stderr, "this ggml has no trellis types (build with STRATA_GGML_DIR = the agention fork)\n"); return 1; }
+ rc |= synthetic(8, 8); // Q8_0: the fold's convention, against the unrotated float expert
+ rc |= synthetic(145, 146); // Gyro-S
+ rc |= synthetic(144, 144); // Gyro-M
+ return rc;
+}
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index fa8eb17c1..0ebbe9ab3 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -43,6 +43,7 @@
#include "strata/kernels/cpu/pool.hpp"
#include "strata/platform/aux_cpus.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/hadamard.hpp"
#include "strata/kernels/ngram.hpp"
#include "strata/kernels/iq_kernels.hpp"
#include "strata/kernels/s2_expert_grouped.hpp"
@@ -3015,6 +3016,14 @@ int main(int argc, char** argv) {
(void) strata::prefill::mmq::supported(f.d_type);
#endif
}
+ // APR (prism.hadamard) packs: the CPU pool rotates the expert activations (native_quant_act / _h), the GPU
+ // expert kernels, the remote GPUs and the prompt path do not yet. Refused here
+ // rather than run with unrotated activations, which would produce plausible-looking garbage.
+ if (strata::kernels::cpu::hadamard_spec().any()) {
+ std::fprintf(stderr, "strata generate: this pack's experts are Hadamard-folded (hadamard.txt, an APR "
+ "model); the GPU expert paths cannot apply the activation rotation yet\n");
+ return 1;
+ }
}
const bool native_pack = strata::kernels::cpu::expert_layout().native;
// STRATA_MMQ_DIRECT: the room each sized expert slot keeps after its blob (0 unless it is on: the slot costs and