Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

native experts: STRATA_ACT_DUMP, per-layer second moments of the CPU expert inputs (research, env-gated, off by default)

Niko1221a775b0469593
diff --git a/include/strata/kernels/cpu/native_expert.hpp b/include/strata/kernels/cpu/native_expert.hpp
index fbac35052..e0cad9065 100644
--- a/include/strata/kernels/cpu/native_expert.hpp
+++ b/include/strata/kernels/cpu/native_expert.hpp
@@ -36,6 +36,7 @@ struct NativeFmt {
/// native_quant_act / native_quant_h apply y = H_block (signs * x) before quantizing. had_block 0 = none.
int had_block = 0;
bool had_gu = false, had_d = false;
+ int layer = -1; ///< the layer this format belongs to (set by expert_layout_load; STRATA_ACT_DUMP only)
const float* gu_signs = nullptr; ///< n_embd signs for x (nullptr: identity)
const float* d_signs = nullptr; ///< n_ff signs for h (nullptr: identity)
};
diff --git a/src/kernels/cpu/expert_layout.cpp b/src/kernels/cpu/expert_layout.cpp
index b6924950d..9d296c976 100644
--- a/src/kernels/cpu/expert_layout.cpp
+++ b/src/kernels/cpu/expert_layout.cpp
@@ -503,6 +503,7 @@ bool expert_layout_load(const std::string& pack_dir, int64_t n_layers, int64_t n
": the pack is inconsistent (repack with tools/iq_pack.py)";
return false;
}
+ for (int64_t l = 0; l < n_layers; ++l) L.fmt[(size_t) l].layer = (int) l;
HadamardSpec h;
if (have && !hadamard_parse(pack_dir + "/hadamard.txt", n_layers, h, err)) return false;
hadamard_set(std::move(h));
diff --git a/src/kernels/cpu/native_expert.cpp b/src/kernels/cpu/native_expert.cpp
index 57cb6dfa9..66ec2db08 100644
--- a/src/kernels/cpu/native_expert.cpp
+++ b/src/kernels/cpu/native_expert.cpp
@@ -16,11 +16,14 @@
#include "ggml.h"
#include "ggml-cpu.h"
+#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
+#include <map>
#include <mutex>
+#include <string>
#include <vector>
namespace strata::kernels::cpu {
@@ -129,6 +132,88 @@ bool q8k_avx1(int type) {
}
} // namespace
+// STRATA_ACT_DUMP=<prefix> (research, off by default): the second moments of the vectors that enter the CPU experts, in the
+// basis the experts use (after the Hadamard rotation of a folded layer): per layer the gate/up input (the token's hidden
+// state, once per token) and the down input (the SwiGLU output, once per routed expert). STRATA_ACT_DUMP_LAYERS=10,20 picks
+// the layers (default none), STRATA_ACT_DUMP_COV=1 adds the full covariance. Files <prefix>_L<l>_<gu|dn>.bin: int64 n,
+// int64 count, double sum x^2 [n], (double sum x x^T [n*n]); rewritten every 2000 records and at exit.
+namespace {
+struct ActAcc {
+ int64_t n = 0, count = 0;
+ std::vector<double> sq, cov;
+};
+struct ActDump {
+ bool on = false, cov = false;
+ std::string prefix;
+ std::vector<int> layers;
+ std::map<std::pair<int, int>, ActAcc> acc;
+ std::mutex mu;
+ int64_t records = 0;
+ ActDump() {
+ const char* p = std::getenv("STRATA_ACT_DUMP");
+ if (p == nullptr || *p == 0) return;
+ prefix = p;
+ cov = std::getenv("STRATA_ACT_DUMP_COV") != nullptr && std::atoi(std::getenv("STRATA_ACT_DUMP_COV")) != 0;
+ if (const char* l = std::getenv("STRATA_ACT_DUMP_LAYERS")) {
+ std::string s(l);
+ size_t at = 0;
+ while (at < s.size()) {
+ layers.push_back(std::atoi(s.c_str() + at));
+ const size_t c = s.find(',', at);
+ if (c == std::string::npos) break;
+ at = c + 1;
+ }
+ }
+ on = !layers.empty();
+ if (on) std::atexit([] { act_dump_flush(); });
+ }
+ static void act_dump_flush();
+};
+ActDump& act_dump() {
+ static ActDump d;
+ return d;
+}
+void ActDump::act_dump_flush() {
+ ActDump& d = act_dump();
+ std::lock_guard<std::mutex> lk(d.mu);
+ for (auto& kv : d.acc) {
+ const std::string path = d.prefix + "_L" + std::to_string(kv.first.first) + (kv.first.second ? "_dn.bin" : "_gu.bin");
+ FILE* f = std::fopen((path + ".tmp").c_str(), "wb");
+ if (!f) continue;
+ std::fwrite(&kv.second.n, 8, 1, f);
+ std::fwrite(&kv.second.count, 8, 1, f);
+ std::fwrite(kv.second.sq.data(), 8, kv.second.sq.size(), f);
+ if (!kv.second.cov.empty()) std::fwrite(kv.second.cov.data(), 8, kv.second.cov.size(), f);
+ std::fclose(f);
+ std::rename((path + ".tmp").c_str(), path.c_str());
+ }
+}
+void act_dump_record(int layer, int kind, const float* v, int64_t n) {
+ ActDump& d = act_dump();
+ if (std::find(d.layers.begin(), d.layers.end(), layer) == d.layers.end()) return;
+ std::lock_guard<std::mutex> lk(d.mu);
+ ActAcc& a = d.acc[{layer, kind}];
+ if (a.n == 0) {
+ a.n = n;
+ a.sq.assign((size_t) n, 0.0);
+ if (d.cov) a.cov.assign((size_t) n * (size_t) n, 0.0);
+ }
+ for (int64_t i = 0; i < n; ++i) a.sq[(size_t) i] += (double) v[i] * v[i];
+ if (d.cov)
+ for (int64_t i = 0; i < n; ++i) {
+ double* row = a.cov.data() + (size_t) i * (size_t) n;
+ const double vi = v[i];
+ for (int64_t j = 0; j < n; ++j) row[j] += vi * v[j];
+ }
+ ++a.count;
+ if (++d.records % 2000 == 0) {
+ d.mu.unlock();
+ ActDump::act_dump_flush();
+ d.mu.lock();
+ }
+}
+} // namespace
+
void native_quant_act(const NativeFmt& f, const float* x, void* dst) {
if (f.had_gu) { // Hadamard-folded gate/up: quantize H (s * x), never x itself (hadamard.hpp)
thread_local std::vector<float> tmp;
@@ -136,6 +221,7 @@ void native_quant_act(const NativeFmt& f, const float* x, void* dst) {
hadamard_rotate(tmp.data(), (int) f.n_embd, f.had_block, f.gu_signs);
x = tmp.data();
}
+ if (act_dump().on) act_dump_record(f.layer, 0, x, f.n_embd);
if (q8k_avx2(f.gu_act)) { q8k_quant_avx2(x, dst, f.n_embd); return; }
if (q8k_avx1(f.gu_act)) { q8k_quant_avx1(x, dst, f.n_embd); return; }
traits(f.gu_act)->from_float(x, dst, f.n_embd);
@@ -148,6 +234,7 @@ void native_quant_h(const NativeFmt& f, const float* h, void* dst) {
hadamard_rotate(tmp.data(), (int) f.n_ff, f.had_block, f.d_signs);
h = tmp.data();
}
+ if (act_dump().on) act_dump_record(f.layer, 1, h, f.n_ff);
if (q8k_avx2(f.d_act)) { q8k_quant_avx2(h, dst, f.n_ff); return; }
if (q8k_avx1(f.d_act)) { q8k_quant_avx1(h, dst, f.n_ff); return; }
traits(f.d_act)->from_float(h, dst, f.n_ff);