Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
APR port, stage 3: Hadamard-folded experts on the CPU path
Laurent Zuijdwijkf47b0385f320
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 72a13be81..8295506ad 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1245,7 +1245,7 @@ if(STRATA_BUILD_TESTS OR STRATA_BUILD_CONVERSATION_TESTS)endif()add_library(strata_kernels_cpu STATIC src/kernels/cpu/expert.cpp src/kernels/cpu/pool.cpp- src/kernels/cpu/expert_layout.cpp src/kernels/cpu/q2_avx2.cpp)+ src/kernels/cpu/expert_layout.cpp src/kernels/cpu/q2_avx2.cpp src/kernels/cpu/hadamard.cpp)target_include_directories(strata_kernels_cpu PUBLIC include)# Plan v0.3 P6: only the kernel files get the AVX-512 / AVX2 flags; the pool and the probe run on any x86-64,# and the engine picks the kernels at run time (cpu_avx512_ok).@@ -1786,6 +1786,15 @@ if(STRATA_BUILD_TESTS)set_tests_properties(q2_bitplane_parity PROPERTIES ENVIRONMENT "STRATA_Q2_BITPLANE=1")endif()+ if(STRATA_NATIVE_EXPERTS)+ # APR port: a Hadamard-folded trellis expert (TQ2_T / TQK6 / TQK7) on the CPU path vs llama.cpp's graph for the+ # same weights (synthetic; needs STRATA_GGML_DIR = a ggml with the trellis types, else it reports and fails)+ add_executable(native_hadamard_test src/kernels/cpu/native_hadamard_test.cpp)+ target_link_libraries(native_hadamard_test PRIVATE strata_kernels_cpu strata_artifact)+ add_test(NAME native_hadamard_test COMMAND native_hadamard_test)+ set_tests_properties(native_hadamard_test PROPERTIES TIMEOUT 300)+ endif()+add_executable(expert_parity src/kernels/cpu/expert_parity.cpp)target_link_libraries(expert_parity PRIVATE strata_kernels_cpu)# The pack is a REQUIRED input, not an optional one: this test compares against a real expert blob, and adiff --git a/include/strata/kernels/cpu/expert_layout.hpp b/include/strata/kernels/cpu/expert_layout.hppindex 6b3fddb66..b78528c77 100644--- a/include/strata/kernels/cpu/expert_layout.hpp+++ b/include/strata/kernels/cpu/expert_layout.hpp@@ -96,6 +96,8 @@ const ExpertLayout& expert_layout();bool expert_layout_load(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err);/// The newest native_experts.txt this engine reads. v4 = v3 plus the per-role shard column `gate,up,down`,/// written only when some layer's roles are in different shards (every other pack stays v3, byte for byte).-inline constexpr int kExpertLayoutVersion = 4;+/// v5 = v4 plus `hadamard.txt` beside it: Hadamard-folded experts (hadamard.hpp), written only for such models,+/// so an engine that cannot rotate the activations refuses the pack instead of running it wrong.+inline constexpr int kExpertLayoutVersion = 5;} // namespace strata::kernels::cpudiff --git a/include/strata/kernels/cpu/hadamard.hpp b/include/strata/kernels/cpu/hadamard.hppnew file mode 100644index 000000000..43b3fcfce--- /dev/null+++ b/include/strata/kernels/cpu/hadamard.hpp@@ -0,0 +1,51 @@+// include/strata/kernels/cpu/hadamard.hpp - the activation-side transform of Hadamard-folded experts.+//+// Some GGUFs (Agention's APR "Gyro" files) store their routed experts in a block-Hadamard-rotated basis: the+// encoder replaced each expert weight W by W' = W * diag(s) * H_b^T per block of b input columns, where H_b is+// the normalized Sylvester-Walsh-Hadamard matrix (entry (-1)^popcount(r & c) / sqrt(b)) and s a +/-1 sign+// vector of the input width. The GGUF says so in its `prism.hadamard.*` metadata. Since H_b is orthonormal and+// symmetric, W x = W' (H_b (s * x)): the runtime must apply y = H_b (s * x), per block of b values, to every+// activation that enters a folded weight - the gate/up input x and the down input h - before quantizing it.+// Skipping it on any path (CPU or GPU, decode or prompt) gives plausible-looking garbage, so a pack that carries+// rotated experts is refused by an engine path that cannot apply it.+//+// tools/iq_pack.py reads the metadata and writes `<pack>/hadamard.txt` (and native_experts.txt v5, so an engine+// older than this refuses the pack). The format:+//+// # strata hadamard v1 ... (comment lines start with #)+// block 128 the rotation block (a power of two)+// signs <width> <width x +1/-1> one line per sign-vector width (none = identity signs)+// layer <l> <gate 0|1> <up 0|1> <down 0|1> which roles of layer l are folded (gate == up: one input)+#pragma once++#include <cstdint>+#include <map>+#include <string>+#include <vector>++namespace strata::kernels::cpu {++struct HadamardSpec {+ int block = 0; ///< 0: no Hadamard-folded experts+ std::map<int, std::vector<float>> signs; ///< width -> +1/-1 per input value (empty: identity signs)+ std::vector<uint8_t> roles; ///< per layer: bit 0 gate, bit 1 up, bit 2 down+ bool any() const { return block > 0; }+ bool gu(int64_t layer) const { return block > 0 && (roles[(size_t) layer] & 3u) != 0; }+ bool down(int64_t layer) const { return block > 0 && (roles[(size_t) layer] & 4u) != 0; }+};++/// Parses `path` (hadamard.txt) for a model of `n_layers` layers. False with a reason when it is malformed.+bool hadamard_parse(const std::string& path, int64_t n_layers, HadamardSpec& h, std::string& err);++/// The process-wide spec (set by expert_layout_load from the pack; empty for every other pack).+const HadamardSpec& hadamard_spec();+void hadamard_set(HadamardSpec h);+/// The sign vector of `width` in the process-wide spec, or nullptr (identity signs). The pointer stays valid+/// until the next hadamard_set.+const float* hadamard_signs(int width);++/// In place: x[i] *= signs[i] (when signs is not null), then each block of `block` values -> H_block x / sqrt(block).+/// `n` must be a multiple of `block`.+void hadamard_rotate(float* x, int n, int block, const float* signs);++} // namespace strata::kernels::cpudiff --git a/include/strata/kernels/cpu/native_expert.hpp b/include/strata/kernels/cpu/native_expert.hppindex 895f90c20..fbac35052 100644--- a/include/strata/kernels/cpu/native_expert.hpp+++ b/include/strata/kernels/cpu/native_expert.hpp@@ -32,6 +32,12 @@ struct NativeFmt {/// projection's FP32 output: gate * s_gate, up * s_up, down * s_down. Do not fold s_down into up: the hidden/// drops to ~1e-5 and its q8 block scale becomes an FP16 subnormal (expert error 2-12% instead of 1.1%).size_t tail_off = 0;+ /// prism.hadamard (hadamard.hpp): the experts of this layer are stored in a rotated basis, and+ /// native_quant_act / native_quant_h apply y = H_block (signs * x) before quantizing. had_block 0 = none.+ int had_block = 0;+ bool had_gu = false, had_d = false;+ const float* gu_signs = nullptr; ///< n_embd signs for x (nullptr: identity)+ const float* d_signs = nullptr; ///< n_ff signs for h (nullptr: identity)};constexpr int kNvfp4Type = 40;constexpr size_t kNvfp4Tail = 16;@@ -41,9 +47,9 @@ bool native_experts_available() noexcept;/// Fills `f` for a layer; false (with a reason) when ggml-cpu has no dot product for a type.bool native_fmt(int gu_type, int d_type, int64_t n_embd, int64_t n_ff, NativeFmt& f, std::string& err);-/// x (n_embd floats) -> the gate/up activation (act_bytes).+/// x (n_embd floats) -> the gate/up activation (act_bytes), rotated first when the layer is Hadamard-folded.void native_quant_act(const NativeFmt& f, const float* x, void* dst);-/// h (n_ff floats) -> the down activation (h_bytes).+/// h (n_ff floats) -> the down activation (h_bytes), rotated first when the layer is Hadamard-folded.void native_quant_h(const NativeFmt& f, const float* h, void* dst);/// From how many tokens native_gu_rows gives this gate/up type to a multi-token kernel (#152; ggml-cpu's per-token dotdiff --git a/src/kernels/cpu/expert_layout.cpp b/src/kernels/cpu/expert_layout.cppindex 001fa5d47..b6924950d 100644--- a/src/kernels/cpu/expert_layout.cpp+++ b/src/kernels/cpu/expert_layout.cpp@@ -1,5 +1,6 @@// src/kernels/cpu/expert_layout.cpp - plan v0.3 P6: the per-layer expert table. See the header.#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/hadamard.hpp"#include <atomic>#include <cstdio>@@ -391,6 +392,7 @@ bool expert_layout_load(const std::string& pack_dir, int64_t n_layers, int64_t nstd::ifstream in(pack_dir + "/native_experts.txt");if (!in) {L.total = (uint64_t) n_layers * (uint64_t) n_expert * (uint64_t) BLOB;+ hadamard_set({});g_layout = L;return true;}@@ -487,6 +489,48 @@ bool expert_layout_load(const std::string& pack_dir, int64_t n_layers, int64_t nat += L.bytes[(size_t) l] * (uint64_t) L.n_expert;}L.total = at;+ // v5: Agention's Hadamard-folded experts (prism.hadamard, hadamard.hpp). The pack must say which layers are+ // rotated and with what signs; a v5 pack without hadamard.txt (or the reverse) is refused, never run unrotated.+ {+ std::ifstream probe(pack_dir + "/hadamard.txt");+ const bool have = (bool) probe;+ if (L.version >= 5 && !have) {+ err = "native_experts.txt v5 declares Hadamard-folded experts, but " + pack_dir + "/hadamard.txt is missing";+ return false;+ }+ if (have && L.version < 5) {+ err = pack_dir + "/hadamard.txt exists but native_experts.txt is v" + std::to_string(L.version) ++ ": the pack is inconsistent (repack with tools/iq_pack.py)";+ return false;+ }+ HadamardSpec h;+ if (have && !hadamard_parse(pack_dir + "/hadamard.txt", n_layers, h, err)) return false;+ hadamard_set(std::move(h));+ const HadamardSpec& hs = hadamard_spec();+ for (int64_t l = 0; hs.any() && l < n_layers; ++l) {+ NativeFmt& f = L.fmt[(size_t) l];+ f.had_block = hs.block;+ f.had_gu = hs.gu(l);+ f.had_d = hs.down(l);+ if ((f.had_gu && f.n_embd % hs.block) || (f.had_d && f.n_ff % hs.block)) {+ err = "hadamard.txt: block " + std::to_string(hs.block) + " does not divide layer " ++ std::to_string(l) + "'s expert widths";+ return false;+ }+ // the Q2_0 rows quantize their activations with Strata's own act_quant_any, which does not rotate+ if ((f.had_gu && f.gu_type == 42) || (f.had_d && f.d_type == 42)) {+ err = "hadamard.txt: layer " + std::to_string(l) + " folds a Q2_0 expert, which this engine cannot rotate";+ return false;+ }+ f.gu_signs = f.had_gu ? hadamard_signs((int) f.n_embd) : nullptr;+ f.d_signs = f.had_d ? hadamard_signs((int) f.n_ff) : nullptr;+ // explicit signs are per width: a width without a vector would silently mean identity signs+ if (!hs.signs.empty() && ((f.had_gu && !f.gu_signs) || (f.had_d && !f.d_signs))) {+ err = "hadamard.txt: no sign vector for layer " + std::to_string(l) + "'s expert width";+ return false;+ }+ }+ }g_layout = L;return true;#endifdiff --git a/src/kernels/cpu/hadamard.cpp b/src/kernels/cpu/hadamard.cppnew file mode 100644index 000000000..3a9fc0c7f--- /dev/null+++ b/src/kernels/cpu/hadamard.cpp@@ -0,0 +1,110 @@+// src/kernels/cpu/hadamard.cpp - see include/strata/kernels/cpu/hadamard.hpp.+#include "strata/kernels/cpu/hadamard.hpp"++#include <cmath>+#include <cstdlib>+#include <fstream>+#include <sstream>++namespace strata::kernels::cpu {+namespace {+HadamardSpec g_spec;+}++const HadamardSpec& hadamard_spec() { return g_spec; }+void hadamard_set(HadamardSpec h) { g_spec = std::move(h); }++const float* hadamard_signs(int width) {+ const auto it = g_spec.signs.find(width);+ return it == g_spec.signs.end() ? nullptr : it->second.data();+}++bool hadamard_parse(const std::string& path, int64_t n_layers, HadamardSpec& h, std::string& err) {+ std::ifstream in(path);+ if (!in) { err = "cannot read " + path; return false; }+ h = HadamardSpec{};+ h.roles.assign((size_t) n_layers, 0);+ std::vector<uint8_t> seen((size_t) n_layers, 0);+ std::string line;+ bool versioned = false;+ while (std::getline(in, line)) {+ if (line.empty()) continue;+ if (line[0] == '#') {+ static const char tag[] = "# strata hadamard v";+ if (line.compare(0, sizeof tag - 1, tag) == 0) {+ if (std::atoi(line.c_str() + sizeof tag - 1) != 1) {+ err = path + ": an unknown hadamard.txt version (this engine reads v1)";+ return false;+ }+ versioned = true;+ }+ continue;+ }+ std::istringstream ss(line);+ std::string key;+ ss >> key;+ if (key == "block") {+ long long b = 0;+ if (!(ss >> b) || b < 2 || b > 4096 || (b & (b - 1)) != 0 || h.block) {+ err = path + ": a bad or repeated block line: " + line;+ return false;+ }+ h.block = (int) b;+ } else if (key == "signs") {+ long long w = 0;+ if (!(ss >> w) || w <= 0 || w > (1 << 20) || h.signs.count((int) w)) {+ err = path + ": a bad or repeated signs line";+ return false;+ }+ std::vector<float> v((size_t) w);+ for (auto& x : v) {+ int s = 0;+ if (!(ss >> s) || (s != 1 && s != -1)) { err = path + ": sign values must be +1 or -1"; return false; }+ x = (float) s;+ }+ std::string extra;+ if (ss >> extra) { err = path + ": a signs line longer than its width"; return false; }+ h.signs.emplace((int) w, std::move(v));+ } else if (key == "layer") {+ long long l = -1;+ int g = -1, u = -1, d = -1;+ if (!(ss >> l >> g >> u >> d) || l < 0 || l >= n_layers || (g | u | d) & ~1 || seen[(size_t) l]) {+ err = path + ": a bad or repeated layer line: " + line;+ return false;+ }+ if (g != u) { // gate and up read one activation: a transform for one of them only is not runnable+ err = path + ": layer " + std::to_string(l) + " folds only one of gate/up";+ return false;+ }+ seen[(size_t) l] = 1;+ h.roles[(size_t) l] = (uint8_t) (g | (u << 1) | (d << 2));+ } else {+ err = path + ": an unknown line: " + line;+ return false;+ }+ }+ if (!versioned || h.block == 0) { err = path + ": no version header or no block line"; return false; }+ for (const auto& [w, v] : h.signs)+ if (w % h.block) { err = path + ": sign width " + std::to_string(w) + " is not whole blocks"; return false; }+ return true;+}++void hadamard_rotate(float* x, int n, int block, const float* signs) {+ if (signs)+ for (int i = 0; i < n; ++i) x[i] *= signs[i];+ const float scale = 1.0f / std::sqrt((float) block);+ for (int b0 = 0; b0 < n; b0 += block) {+ float* v = x + b0;+ // the in-place fast Walsh-Hadamard transform in natural (Sylvester) order: entry (-1)^popcount(r & c)+ for (int len = 1; len < block; len <<= 1)+ for (int i = 0; i < block; i += len << 1)+ for (int j = i; j < i + len; ++j) {+ const float a = v[j], c = v[j + len];+ v[j] = a + c;+ v[j + len] = a - c;+ }+ for (int i = 0; i < block; ++i) v[i] *= scale;+ }+}++} // namespace strata::kernels::cpudiff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cppindex 1ab75b642..7ee0e36aa 100644--- a/src/kernels/cpu/native_expert.cpp+++ b/src/kernels/cpu/native_expert.cpp@@ -10,6 +10,7 @@#include "strata/kernels/cpu/nvfp4_avx2.hpp"#include "strata/kernels/cpu/nvfp4_avx512.hpp"#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/hadamard.hpp"#include "ggml.h"#include "ggml-cpu.h"@@ -18,6 +19,7 @@#include <cstdlib>#include <cstring>#include <mutex>+#include <vector>namespace strata::kernels::cpu {namespace {@@ -110,12 +112,24 @@ bool q8k_avx1(int type) {} // namespacevoid native_quant_act(const NativeFmt& f, const float* x, void* dst) {+ if (f.had_gu) { // Hadamard-folded gate/up: quantize H (s * x), never x itself (hadamard.hpp)+ thread_local std::vector<float> tmp;+ tmp.assign(x, x + f.n_embd);+ hadamard_rotate(tmp.data(), (int) f.n_embd, f.had_block, f.gu_signs);+ x = tmp.data();+ }if (q8k_avx2(f.gu_act)) { q8k_quant_avx2(x, dst, f.n_embd); return; }if (q8k_avx1(f.gu_act)) { q8k_quant_avx1(x, dst, f.n_embd); return; }traits(f.gu_act)->from_float(x, dst, f.n_embd);}void native_quant_h(const NativeFmt& f, const float* h, void* dst) {+ if (f.had_d) {+ thread_local std::vector<float> tmp;+ tmp.assign(h, h + f.n_ff);+ hadamard_rotate(tmp.data(), (int) f.n_ff, f.had_block, f.d_signs);+ h = tmp.data();+ }if (q8k_avx2(f.d_act)) { q8k_quant_avx2(h, dst, f.n_ff); return; }if (q8k_avx1(f.d_act)) { q8k_quant_avx1(h, dst, f.n_ff); return; }traits(f.d_act)->from_float(h, dst, f.n_ff);diff --git a/src/kernels/cpu/native_hadamard_test.cpp b/src/kernels/cpu/native_hadamard_test.cppnew file mode 100644index 000000000..de5b923f1--- /dev/null+++ b/src/kernels/cpu/native_hadamard_test.cpp@@ -0,0 +1,311 @@+// src/kernels/cpu/native_hadamard_test.cpp - APR port, stage 3: one Hadamard-folded trellis expert (TQ2_T / TQK6 /+// TQK7, ggml types 144-146) on Strata's CPU path against llama.cpp's own graph for the same weights and input.+//+// Strata's path: native_quant_act (signs, FWHT, Q8_0) -> native_gu_rows (ggml-cpu vec_dot + SwiGLU)+// -> native_quant_h (signs, FWHT, Q8_0) -> native_down_rows+// The reference: the ops llama.cpp's build_lora_mm_id builds for a prism.hadamard weight - ggml_mul(x, signs),+// llama_mul_mat_hadamard (reshape to [128, n/128], ggml_mul_mat(rot, .)), ggml_mul_mat(W', .) -+// computed by ggml-cpu from the same checkout (STRATA_GGML_DIR = the agention llama.cpp fork).+// The rotation itself is computed two different ways (FWHT here, a dense 128x128 matmul there), so the two outputs+// agree to float rounding plus the odd Q8_0 rounding flip, not bit for bit; the test bounds the relative error.+//+// native_hadamard_test synthetic: random W, rotated and quantized here; also checks the+// folded expert against the UNROTATED float expert (W x) and that+// skipping the rotation is caught+// native_hadamard_test --pack <dir> --gguf <file> [layer expert]... real experts of a prism.hadamard GGUF+// (Gyro-S / Gyro-M) through the pack tools/iq_pack.py made of it+#include "strata/kernels/cpu/hadamard.hpp"+#include "strata/kernels/cpu/native_expert.hpp"+#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/pool.hpp"+#include "strata/artifact/gguf_reader.hpp"++#include "ggml.h"+#include "ggml-cpu.h"++#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <algorithm>+#include <random>+#include <string>+#include <vector>++using namespace strata::kernels::cpu;++namespace {++constexpr int H = 2560, FF = 640, B = 128;++struct Expert {+ int gu_type = 0, d_type = 0;+ std::vector<uint8_t> gate, up, down; // ggml rows: gate/up [FF rows of H], down [H rows of FF]+ std::vector<float> s_h, s_ff; // sign vectors (empty: identity)+};++double rel(const std::vector<float>& a, const std::vector<float>& b) {+ double num = 0, den = 0;+ for (size_t i = 0; i < a.size(); ++i) {+ num += (double) (a[i] - b[i]) * (a[i] - b[i]);+ den += (double) b[i] * b[i];+ }+ return std::sqrt(num / (den > 0 ? den : 1));+}++float silu(float g) { return g / (1.f + std::exp(-g)); }++// Strata's CPU path for one token; `loaded` = the layer's NativeFmt as expert_layout_load made it from a pack+std::vector<float> strata_expert(const Expert& e, const std::vector<float>& x, bool rotate,+ const NativeFmt* loaded = nullptr) {+ NativeFmt f;+ std::string err;+ if (!native_fmt(e.gu_type, e.d_type, H, FF, f, err)) { std::fprintf(stderr, "native_fmt: %s\n", err.c_str()); std::exit(1); }+ if (rotate && loaded) {+ f = *loaded;+ } else if (rotate) {+ f.had_block = B;+ f.had_gu = f.had_d = true;+ f.gu_signs = e.s_h.empty() ? nullptr : e.s_h.data();+ f.d_signs = e.s_ff.empty() ? nullptr : e.s_ff.data();+ }+ std::vector<uint8_t> blob(f.bytes);+ std::memcpy(blob.data(), e.gate.data(), e.gate.size());+ std::memcpy(blob.data() + f.up_off, e.up.data(), e.up.size());+ std::memcpy(blob.data() + f.down_off, e.down.data(), e.down.size());+ std::vector<uint8_t> act(kNativeActBytes), hq(kNativeHBytes);+ std::vector<float> ff(FF), out(H);+ native_quant_act(f, x.data(), act.data());+ const void* a[1] = {act.data()};+ float* ffp[1] = {ff.data()};+ native_gu_rows(f, blob.data(), a, 1, ffp, 0, FF);+ native_quant_h(f, ff.data(), hq.data());+ const void* hp[1] = {hq.data()};+ float* op[1] = {out.data()};+ native_down_rows(f, blob.data(), hp, 1, op, 0, H);+ return out;+}++// llama.cpp's graph for the same expert (the fork's build_lora_mm + llama_mul_mat_hadamard), one token+std::vector<float> llama_expert(const Expert& e, const std::vector<float>& x) {+ ggml_init_params ip = {(size_t) 256 << 20, nullptr, false};+ ggml_context* ctx = ggml_init(ip);+ // the normalized Sylvester-Walsh-Hadamard matrix exactly as llama_model_base builds prism.hadamard.128+ ggml_tensor* rot = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, B, B);+ float* R = (float*) rot->data;+ const float scale = 1.0f / sqrtf((float) B);+ for (uint32_t r = 0; r < (uint32_t) B; ++r)+ for (uint32_t c = 0; c < (uint32_t) B; ++c) {+ uint32_t p = r & c;+ p ^= p >> 16; p ^= p >> 8; p ^= p >> 4; p ^= p >> 2; p ^= p >> 1;+ R[r * B + c] = (p & 1) ? -scale : scale;+ }+ auto hadamard = [&](ggml_tensor* cur, const std::vector<float>& signs) {+ if (!signs.empty()) {+ ggml_tensor* s = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, (int64_t) signs.size());+ std::memcpy(s->data, signs.data(), signs.size() * sizeof(float));+ cur = ggml_mul(ctx, cur, s);+ }+ ggml_tensor* res = ggml_reshape_2d(ctx, cur, B, ggml_nelements(cur) / B);+ res = ggml_mul_mat(ctx, rot, res);+ return ggml_reshape_4d(ctx, res, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3]);+ };+ ggml_tensor* wg = ggml_new_tensor_2d(ctx, (ggml_type) e.gu_type, H, FF);+ ggml_tensor* wu = ggml_new_tensor_2d(ctx, (ggml_type) e.gu_type, H, FF);+ ggml_tensor* wd = ggml_new_tensor_2d(ctx, (ggml_type) e.d_type, FF, H);+ std::memcpy(wg->data, e.gate.data(), e.gate.size());+ std::memcpy(wu->data, e.up.data(), e.up.size());+ std::memcpy(wd->data, e.down.data(), e.down.size());+ ggml_tensor* xt = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, H);+ std::memcpy(xt->data, x.data(), H * sizeof(float));+ ggml_tensor* xr = hadamard(xt, e.s_h);+ ggml_tensor* g = ggml_mul_mat(ctx, wg, xr);+ ggml_tensor* u = ggml_mul_mat(ctx, wu, xr);+ ggml_tensor* h = ggml_mul(ctx, ggml_silu(ctx, g), u);+ ggml_tensor* out = ggml_mul_mat(ctx, wd, hadamard(h, e.s_ff));+ ggml_cgraph* gf = ggml_new_graph(ctx);+ ggml_build_forward_expand(gf, out);+ ggml_graph_compute_with_ctx(ctx, gf, 4);+ std::vector<float> y(H);+ std::memcpy(y.data(), out->data, H * sizeof(float));+ ggml_free(ctx);+ return y;+}++// W' = W diag(s) H^T per 128-column block (what the APR encoder folds), then quantized with ggml+std::vector<uint8_t> fold_and_quantize(const std::vector<float>& w, int rows, int cols, const std::vector<float>& s,+ int type, std::vector<float>& folded) {+ folded.assign(w.begin(), w.end());+ for (int r = 0; r < rows; ++r) hadamard_rotate(folded.data() + (size_t) r * cols, cols, B, s.data());+ // (H symmetric and orthonormal: row-wise H (s * w_r) is exactly the fold W diag(s) H^T)+ std::vector<uint8_t> q(ggml_row_size((ggml_type) type, cols) * rows);+ ggml_quantize_chunk((ggml_type) type, folded.data(), q.data(), 0, rows, cols, nullptr);+ return q;+}++int synthetic(int gu_type, int d_type) {+ std::mt19937 rng(1234 + gu_type * 7 + d_type);+ std::normal_distribution<float> nd(0.f, 1.f);+ std::uniform_int_distribution<int> coin(0, 1);+ Expert e;+ e.gu_type = gu_type;+ e.d_type = d_type;+ std::vector<float> wg((size_t) FF * H), wu((size_t) FF * H), wd((size_t) H * FF), x(H);+ for (auto& v : wg) v = nd(rng) * 0.02f;+ for (auto& v : wu) v = nd(rng) * 0.02f;+ for (auto& v : wd) v = nd(rng) * 0.02f;+ for (auto& v : x) v = nd(rng);+ x[17] = 40.f; // a massive channel, as the real residual stream has: the case the rotation exists for+ e.s_h.resize(H);+ e.s_ff.resize(FF);+ for (auto& v : e.s_h) v = coin(rng) ? 1.f : -1.f;+ for (auto& v : e.s_ff) v = coin(rng) ? 1.f : -1.f;+ std::vector<float> fg, fu, fd;+ e.gate = fold_and_quantize(wg, FF, H, e.s_h, gu_type, fg);+ e.up = fold_and_quantize(wu, FF, H, e.s_h, gu_type, fu);+ e.down = fold_and_quantize(wd, H, FF, e.s_ff, d_type, fd);++ // the unrotated float expert W x: what the folded, quantized expert must approximate+ std::vector<float> h(FF), ref(H);+ for (int r = 0; r < FF; ++r) {+ double g = 0, u = 0;+ for (int c = 0; c < H; ++c) { g += (double) wg[(size_t) r * H + c] * x[c]; u += (double) wu[(size_t) r * H + c] * x[c]; }+ h[r] = silu((float) g) * (float) u;+ }+ for (int r = 0; r < H; ++r) {+ double o = 0;+ for (int c = 0; c < FF; ++c) o += (double) wd[(size_t) r * FF + c] * h[c];+ ref[r] = (float) o;+ }+ const auto ours = strata_expert(e, x, true);+ const auto llama = llama_expert(e, x);+ const auto unrot = strata_expert(e, x, false);+ const double e_ll = rel(ours, llama), e_ref = rel(ours, ref), e_llref = rel(llama, ref), e_un = rel(unrot, ref);+ std::printf("synthetic %s/%s: strata vs llama.cpp graph %.2e | vs unrotated float expert: strata %.4f, llama.cpp "+ "%.4f | strata WITHOUT the rotation %.4f\n", ggml_type_name((ggml_type) gu_type),+ ggml_type_name((ggml_type) d_type), e_ll, e_ref, e_llref, e_un);+ // strata == llama.cpp to rounding, and both approximate W x equally well. With Q8_0 weights the quantization+ // error is ~1%, so the fold's convention itself is checked: the rotated path must reproduce the unrotated float+ // expert and the unrotated path must not. (The trellis types' own error at 1.6-2.1 bpw on Gaussian weights,+ // compounded through SwiGLU, is large with this test's placeholder ggml encoder; there only agreement counts.)+ const bool hi = gu_type == 8 && d_type == 8;+ const bool ok = e_ll < 2e-3 && std::fabs(e_ref - e_llref) < 0.01 && (!hi || (e_ref < 0.03 && e_un > 0.8));+ std::printf(" %s\n", ok ? "ok" : "FAILED");+ return ok ? 0 : 1;+}++// A real expert, the pack's loader path: expert_layout_load(<pack>) reads native_experts.txt v5 + hadamard.txt (from+// tools/iq_pack.py), and the layer's NativeFmt is what the engine's CPU pool would use. (The signs come from the+// pack: strata::GgufFile keeps only the first 64 items of a metadata array, so it cannot read sign_values.)+int real(const char* pack, const char* path, int layer, int expert) {+ std::string err;+ if (!expert_layout_load(pack, 48, 512, err)) { std::fprintf(stderr, "expert_layout_load: %s\n", err.c_str()); return 1; }+ const ExpertLayout& lay = expert_layout();+ const HadamardSpec& hs = hadamard_spec();+ if (!lay.native || !hs.any() || lay.version != 5) { std::fprintf(stderr, "%s: not a v5 (Hadamard) native pack\n", pack); return 1; }+ const NativeFmt& lf = lay.fmt[(size_t) layer];+ if (lf.had_block != B || !lf.had_gu || !lf.had_d) { std::fprintf(stderr, "layer %d is not folded in the pack\n", layer); return 1; }+ strata::GgufFile g(path);+ Expert e;+ if (const float* sh = hadamard_signs(H)) e.s_h.assign(sh, sh + H);+ if (const float* sf = hadamard_signs(FF)) e.s_ff.assign(sf, sf + FF);+ auto slice = [&](const char* role, std::vector<uint8_t>& dst, int& type) {+ const std::string name = "blk." + std::to_string(layer) + ".ffn_" + role + "_exps.weight";+ const strata::TensorInfo* t = g.find(name);+ if (!t || t->shape.size() != 3) { std::fprintf(stderr, "missing %s\n", name.c_str()); std::exit(1); }+ type = (int) t->type;+ const size_t per = ggml_row_size((ggml_type) type, (int64_t) t->shape[0]) * t->shape[1];+ const uint8_t* p = g.tensor_data(*t) + per * (size_t) expert;+ dst.assign(p, p + per);+ };+ int dt = 0, ut = 0;+ slice("gate", e.gate, e.gu_type);+ slice("up", e.up, ut);+ slice("down", e.down, dt);+ e.d_type = dt;+ if (e.gu_type != lf.gu_type || e.d_type != lf.d_type) { std::fprintf(stderr, "the pack and the GGUF disagree\n"); return 1; }+ std::mt19937 rng(99);+ std::normal_distribution<float> nd(0.f, 1.f);+ std::vector<float> x(H);+ for (auto& v : x) v = nd(rng);+ x[17] = 40.f;+ const auto ours = strata_expert(e, x, true, &lf);+ const auto llama = llama_expert(e, x);+ const auto unrot = strata_expert(e, x, false);+ const double e_ll = rel(ours, llama), e_un = rel(unrot, llama);+ std::printf("%s layer %d expert %d (%s/%s, signs %s): strata vs llama.cpp graph %.2e | without the rotation %.3f\n",+ path, layer, expert, ggml_type_name((ggml_type) e.gu_type), ggml_type_name((ggml_type) e.d_type),+ e.s_h.empty() ? "identity" : "explicit", e_ll, e_un);+ bool ok = e_ll < 2e-3 && e_un > 0.3;+ std::printf(" %s\n", ok ? "ok" : "FAILED");++ // the engine's CPU entry point: ExpertPool::run_split_multi_native with the pack's NativeFmt, three tokens+ // routed to this expert (x quantized by native_quant_act, as expert_source.cpp does for the doorbell copy)+ {+ constexpr int NT = 3;+ ExpertPool pool(4, false, false);+ std::vector<uint8_t> blob(lf.bytes);+ std::memcpy(blob.data(), e.gate.data(), e.gate.size());+ std::memcpy(blob.data() + lf.up_off, e.up.data(), e.up.size());+ std::memcpy(blob.data() + lf.down_off, e.down.data(), e.down.size());+ std::vector<std::vector<float>> xs(NT, std::vector<float>(H)), outs(NT, std::vector<float>(H));+ std::vector<std::vector<uint8_t>> acts(NT, std::vector<uint8_t>(kNativeActBytes));+ ExpertJobMulti job;+ job.blob = blob.data();+ job.nt = NT;+ for (int t = 0; t < NT; ++t) {+ for (auto& v : xs[t]) v = nd(rng);+ xs[t][(size_t) (101 * t + 5)] = 30.f;+ native_quant_act(lf, xs[t].data(), acts[t].data());+ job.nact[t] = acts[t].data();+ job.out[t] = outs[t].data();+ }+ pool.run_split_multi_native(lf, &job, 1);+ double worst = 0;+ for (int t = 0; t < NT; ++t) worst = std::max(worst, rel(outs[t], llama_expert(e, xs[t])));+ const bool pok = worst < 2e-3;+ std::printf(" ExpertPool::run_split_multi_native, %d tokens: worst vs llama.cpp graph %.2e %s\n", NT, worst,+ pok ? "ok" : "FAILED");+ ok = ok && pok;+ }+ return ok ? 0 : 1;+}++// the FWHT against the dense matrix llama.cpp builds, on one block+int fwht_check() {+ std::mt19937 rng(7);+ std::normal_distribution<float> nd(0.f, 1.f);+ std::vector<float> x(B), y(B), s(B);+ for (auto& v : x) v = nd(rng);+ for (int i = 0; i < B; ++i) s[i] = (i * 37 % 5) < 2 ? -1.f : 1.f;+ y = x;+ hadamard_rotate(y.data(), B, B, s.data());+ double worst = 0;+ for (int r = 0; r < B; ++r) {+ double acc = 0;+ for (int c = 0; c < B; ++c) acc += ((__builtin_popcount(r & c) & 1) ? -1.0 : 1.0) * s[c] * x[c];+ worst = std::max(worst, std::fabs(acc / std::sqrt((double) B) - y[r]));+ }+ std::printf("FWHT vs dense normalized Sylvester matrix: max abs diff %.2e %s\n", worst, worst < 1e-5 ? "ok" : "FAILED");+ return worst < 1e-5 ? 0 : 1;+}++} // namespace++int main(int argc, char** argv) {+ if (!native_experts_available()) { std::fprintf(stderr, "built without native experts\n"); return 1; }+ int rc = fwht_check();+ if (argc >= 5 && std::strcmp(argv[1], "--pack") == 0 && std::strcmp(argv[3], "--gguf") == 0) {+ for (int i = 5; i + 1 < argc || i == 5; i += 2) { // [layer expert]...+ const int layer = argc > i ? std::atoi(argv[i]) : 0, expert = argc > i + 1 ? std::atoi(argv[i + 1]) : 0;+ rc |= real(argv[2], argv[4], layer, expert);+ }+ return rc;+ }+ // ggml type ids of the agention fork: 144 TQ2_T, 145 TQK6, 146 TQK7+ if (GGML_TYPE_COUNT <= 146) { std::fprintf(stderr, "this ggml has no trellis types (build with STRATA_GGML_DIR = the agention fork)\n"); return 1; }+ rc |= synthetic(8, 8); // Q8_0: the fold's convention, against the unrotated float expert+ rc |= synthetic(145, 146); // Gyro-S+ rc |= synthetic(144, 144); // Gyro-M+ return rc;+}diff --git a/src/program/generate.cpp b/src/program/generate.cppindex fa8eb17c1..0ebbe9ab3 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -43,6 +43,7 @@#include "strata/kernels/cpu/pool.hpp"#include "strata/platform/aux_cpus.hpp"#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/hadamard.hpp"#include "strata/kernels/ngram.hpp"#include "strata/kernels/iq_kernels.hpp"#include "strata/kernels/s2_expert_grouped.hpp"@@ -3015,6 +3016,14 @@ int main(int argc, char** argv) {(void) strata::prefill::mmq::supported(f.d_type);#endif}+ // APR (prism.hadamard) packs: the CPU pool rotates the expert activations (native_quant_act / _h), the GPU+ // expert kernels, the remote GPUs and the prompt path do not yet. Refused here+ // rather than run with unrotated activations, which would produce plausible-looking garbage.+ if (strata::kernels::cpu::hadamard_spec().any()) {+ std::fprintf(stderr, "strata generate: this pack's experts are Hadamard-folded (hadamard.txt, an APR "+ "model); the GPU expert paths cannot apply the activation rotation yet\n");+ return 1;+ }}const bool native_pack = strata::kernels::cpu::expert_layout().native;// STRATA_MMQ_DIRECT: the room each sized expert slot keeps after its blob (0 unless it is on: the slot costs and