Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
KVA: optional approximate late-layer prefill (split 24, chunk-aligned suffix, capture/oracle modes)
Niko1221f4012d63690c
diff --git a/include/strata/prefill/kva.hpp b/include/strata/prefill/kva.hppnew file mode 100644index 000000000..560effd45--- /dev/null+++ b/include/strata/prefill/kva.hpp@@ -0,0 +1,97 @@+// include/strata/prefill/kva.hpp - STRATA_KVA: the optional approximate-prefill plan (research, opt-in).+//+// After layer `split` the prompt's late layers need only their recurrent state (GDN recurrence + conv history)+// and their KV/indexer caches; the attention output, the output projection, the MoE half and its expert+// transfers have no remaining consumer for the approximate positions. The plan is:+//+// positions [0, n - suffix) approximated: late layers run state-only on a predicted block input+// positions [n - suffix, n) fully computed (the suffix), request-global+//+// v1 keeps the suffix on a CHUNK boundary (a chunk is approximated only when its last position is before the+// frontier), so no layer ever has to split one chunk into two regimes. The late layers' block input is the+// hyper-connection residual R (10,240 floats = hc * n_embd) that the chunk's layer loop carries; it comes from+// identity the residual entering layer `split` (no projector at all, the zero-cost baseline),+// oracle a captured real input per late layer (STRATA_KVA_ORACLE=<dir>; the plumbing/parity mode),+// linear a fitted low-rank projector (STRATA_KVA_PROJECTOR=<file>; not implemented in this step).+//+// Everything here is read from the environment once per process and is off unless STRATA_KVA is set, so a+// default build behaves exactly as before.+#pragma once++#include <cstdint>+#include <cstdlib>+#include <string>++namespace strata::prefill {++struct KvaConfig {+ enum class Mode { Identity, Oracle, Linear };++ bool on = false; ///< STRATA_KVA=1+ bool capture = false; ///< STRATA_KVA_CAPTURE=<dir>: dump every late layer's real input rows+ bool capture_fp32 = false; ///< STRATA_KVA_CAPTURE_FP32=1: fp32 records (else bf16; fp32 is for parity runs)+ bool oracle_fp32 = false; ///< STRATA_KVA_ORACLE_FP32=1: the oracle files are fp32 (else bf16)+ bool parity = false; ///< STRATA_KVA_PARITY=1: report the state-only path against the full path+ int64_t split = 24; ///< STRATA_KVA_SPLIT: first approximated layer (zero-based)+ int64_t suffix = 2048; ///< STRATA_KVA_SUFFIX: fully computed tail positions (rounded up to a chunk)+ int64_t min_prompt = 4096; ///< STRATA_KVA_MIN_PROMPT: shorter prompts stay on the full path+ Mode mode = Mode::Identity; ///< STRATA_KVA_MODE=identity|oracle|linear+ std::string oracle_dir; ///< STRATA_KVA_ORACLE+ std::string capture_dir; ///< STRATA_KVA_CAPTURE+ std::string projector; ///< STRATA_KVA_PROJECTOR++ /// Does this run qualify at all? A cold read of one prompt, on one stage, long enough for a suffix to+ /// leave approximated positions behind it.+ bool applies(int64_t pos0, int64_t n, int64_t lb, int64_t le, int64_t n_layers) const {+ if (!on || capture) return false; // capture runs are ordinary full reads+ if (pos0 != 0 || lb != 0 || (le >= 0 && le != n_layers)) return false;+ return n >= min_prompt && n > suffix + 64;+ }+};++inline int64_t kva_env_i64(const char* name, int64_t dflt) {+ const char* v = std::getenv(name);+ if (v == nullptr || *v == '\0') return dflt;+ return (int64_t) std::atoll(v);+}++inline bool kva_env_flag(const char* name, bool dflt) {+ const char* v = std::getenv(name);+ if (v == nullptr || *v == '\0') return dflt;+ return v[0] != '0';+}++/// The process's KVA settings, parsed once (the engine's own options are parsed once too).+inline const KvaConfig& kva_config() {+ static const KvaConfig c = [] {+ KvaConfig k;+ k.on = kva_env_flag("STRATA_KVA", false);+ k.split = kva_env_i64("STRATA_KVA_SPLIT", 24);+ k.suffix = kva_env_i64("STRATA_KVA_SUFFIX", 2048);+ k.min_prompt = kva_env_i64("STRATA_KVA_MIN_PROMPT", 4096);+ k.parity = kva_env_flag("STRATA_KVA_PARITY", false);+ if (const char* d = std::getenv("STRATA_KVA_CAPTURE"); d != nullptr && *d != '\0') {+ k.capture = true;+ k.capture_dir = d;+ k.capture_fp32 = kva_env_flag("STRATA_KVA_CAPTURE_FP32", false);+ }+ if (const char* d = std::getenv("STRATA_KVA_ORACLE"); d != nullptr && *d != '\0') {+ k.oracle_dir = d;+ k.mode = KvaConfig::Mode::Oracle;+ k.oracle_fp32 = kva_env_flag("STRATA_KVA_ORACLE_FP32", false);+ }+ if (const char* d = std::getenv("STRATA_KVA_PROJECTOR"); d != nullptr && *d != '\0') {+ k.projector = d;+ k.mode = KvaConfig::Mode::Linear;+ }+ if (const char* m = std::getenv("STRATA_KVA_MODE"); m != nullptr && *m != '\0') {+ if (std::string(m) == "oracle") k.mode = KvaConfig::Mode::Oracle;+ else if (std::string(m) == "linear") k.mode = KvaConfig::Mode::Linear;+ else k.mode = KvaConfig::Mode::Identity;+ }+ return k;+ }();+ return c;+}++} // namespace strata::prefilldiff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 6cb888e50..d69d03ae3 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -46,6 +46,7 @@#include "strata/prefill/moe_mmq.hpp"#include "strata/core/peer_experts.hpp"#include "strata/prefill/kernels.hpp"+#include "strata/prefill/kva.hpp" // STRATA_KVA: the optional approximate-prefill plan (research, off by default)#include <cuda_runtime.h>@@ -2582,6 +2583,123 @@ struct PeTimer {cudaSetDevice(prev);}};++// ---- KVA (STRATA_KVA, research): the capture and oracle taps of the late layers' block inputs ---------------+//+// A capture run writes, for every layer at or after the split and every chunk, the chunk's multi-stream residual+// R [T, hc*n_embd] as it ENTERS the layer - exactly the vector a projector would have to produce. One file per+// layer (`L<layer>.bin`), rows in position order, bf16 (fp32 with STRATA_KVA_CAPTURE_FP32, for parity runs): a+// reader seeks straight to a position. The oracle mode replays such a file in place of a projector, which is how+// the state-only path is checked against the full one before any weights are fitted.+struct KvaTap {+ std::string dir;+ bool fp32 = false;+ std::vector<std::FILE*> f; // one per layer, opened on first use+ std::vector<int64_t> next_pos; // where the next record must start (0: nothing written yet)+ std::vector<float> host; // one chunk's rows, fp32+ std::vector<uint16_t> bf; // their bf16 image+ std::string err;++ bool open(const std::string& d, bool is_fp32) {+ dir = d;+ fp32 = is_fp32;+ f.assign(128, nullptr);+ next_pos.assign(128, 0);+ return true;+ }+ void close() {+ for (std::FILE*& p : f) { if (p != nullptr) std::fclose(p); p = nullptr; }+ }+ /// A new cold read starts: a capture file holds one prompt's rows, so every layer starts over.+ void begin_prompt() {+ close();+ if (next_pos.empty()) next_pos.assign(f.size(), 0);+ for (int64_t& p : next_pos) p = 0;+ }+ std::string path(int64_t layer) const { return dir + "/L" + std::to_string((long long) layer) + ".bin"; }++ /// The chunk's rows of `layer` from the device residual `R` - a capture.+ bool write(int64_t layer, const float* R, int64_t T, int64_t p0, int64_t D, cudaStream_t cs) {+ if (layer < 0 || layer >= (int64_t) f.size()) { err = "kva capture: layer out of range"; return false; }+ if (next_pos[(size_t) layer] != p0) {+ err = "kva capture: layer " + std::to_string((long long) layer) + " expects position " ++ std::to_string((long long) next_pos[(size_t) layer]) + ", got " + std::to_string((long long) p0) ++ " (a capture needs one cold, in-order read)";+ return false;+ }+ host.resize((size_t) T * D);+ if (cudaMemcpyAsync(host.data(), R, (size_t) T * D * 4, cudaMemcpyDeviceToHost, cs) != cudaSuccess ||+ cudaStreamSynchronize(cs) != cudaSuccess) {+ err = "kva capture: the residual copy failed";+ return false;+ }+ std::FILE*& p = f[(size_t) layer];+ if (p == nullptr) {+ p = std::fopen(path(layer).c_str(), "wb");+ if (p == nullptr) { err = "kva capture: cannot write " + path(layer); return false; }+ }+ if (fp32) {+ std::fwrite(host.data(), 4, (size_t) T * D, p);+ } else {+ bf.resize((size_t) T * D);+ for (size_t i = 0; i < bf.size(); ++i) { // round to nearest even, as STRATA_PREFILL_DUMP_R_ALL does+ uint32_t u;+ std::memcpy(&u, &host[i], 4);+ u += 0x7fffu + ((u >> 16) & 1u);+ bf[i] = (uint16_t) (u >> 16);+ }+ std::fwrite(bf.data(), 2, bf.size(), p);+ }+ std::fflush(p);+ next_pos[(size_t) layer] = p0 + T;+ return true;+ }++ /// The chunk's rows of `layer` into the device residual `R` - an oracle replay.+ bool read(int64_t layer, float* R, int64_t T, int64_t p0, int64_t D, cudaStream_t cs) {+ if (layer < 0 || layer >= (int64_t) f.size()) { err = "kva oracle: layer out of range"; return false; }+ std::FILE*& p = f[(size_t) layer];+ if (p == nullptr) {+ p = std::fopen(path(layer).c_str(), "rb");+ if (p == nullptr) { err = "kva oracle: cannot read " + path(layer); return false; }+ }+ const int64_t n = T * D;+ const int64_t esz = fp32 ? 4 : 2;+ const long long off = (long long) ((int64_t) p0 * D * esz);+#if defined(_WIN32)+ const bool seek_ok = _fseeki64(p, off, SEEK_SET) == 0;+#else+ const bool seek_ok = fseeko(p, (off_t) off, SEEK_SET) == 0;+#endif+ if (!seek_ok) { err = "kva oracle: cannot seek " + path(layer); return false; }+ host.resize((size_t) n);+ if (fp32) {+ const size_t got = std::fread(host.data(), 4, (size_t) n, p);+ if (got != (size_t) n) {+ err = "kva oracle: short read of " + path(layer) + " at position " + std::to_string((long long) p0) ++ ": wanted " + std::to_string((long long) n) + " values, got " + std::to_string((long long) got);+ return false;+ }+ } else {+ bf.resize((size_t) n);+ const size_t got = std::fread(bf.data(), 2, (size_t) n, p);+ if (got != (size_t) n) {+ err = "kva oracle: short read of " + path(layer) + " at position " + std::to_string((long long) p0) ++ ": wanted " + std::to_string((long long) n) + " values, got " + std::to_string((long long) got);+ return false;+ }+ for (size_t i = 0; i < bf.size(); ++i) { // bf16 -> fp32: the top half, exactly+ const uint32_t u = (uint32_t) bf[i] << 16;+ std::memcpy(&host[i], &u, 4);+ }+ }+ if (cudaMemcpyAsync(R, host.data(), (size_t) n * 4, cudaMemcpyHostToDevice, cs) != cudaSuccess) {+ err = "kva oracle: the upload failed";+ return false;+ }+ return true;+ }+};} // namespacebool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err) {@@ -2595,6 +2713,19 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stricore::SessionState& ss = *m.ss;const auto t_start = Clock::now();const int64_t LB = stage_lb_, LE = stage_le_;+ // STRATA_KVA: the approximate-prefix plan of this run, if it qualifies (a cold read of one long prompt, one+ // stage, no capture). Positions [0, kva_frontier) are approximated in the layers at and after the split.+ const KvaConfig& kva = kva_config();+ const bool kva_run = kva.applies(pos0, n, LB, LE < 0 ? -1 : LE, (int64_t) g.n_layers);+ const int64_t kva_split = std::min<int64_t>(std::max<int64_t>(kva.split, 1), g.n_layers);+ const int64_t kva_frontier = std::max<int64_t>(0, n - kva.suffix);+ static KvaTap kva_capture; // process-wide: one cold read at a time, so the per-layer position is enough+ static KvaTap kva_oracle;+ int64_t kva_positions = 0, kva_chunks = 0; // what actually took the approximate path (diagnostics)+ if (kva.capture && kva_capture.f.empty()) kva_capture.open(kva.capture_dir, kva.capture_fp32);+ if (kva.capture && pos0 == 0 && hand_in_ == nullptr) kva_capture.begin_prompt(); // one prompt per capture+ if (kva_run && kva.mode == KvaConfig::Mode::Oracle && kva_oracle.f.empty())+ kva_oracle.open(kva.oracle_dir, kva.oracle_fp32);// The direct successor's future lives on the Prefill object. Intermediate// stages therefore do not drain the complete remaining GPU chain here.double host_sync_ms = 0, host_chunk_ms = 0, host_setup_ms = 0; // STRATA_PREFILL_TIMING: the host's share@@ -2719,6 +2850,13 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri}if (std::getenv("STRATA_TRACE")) { std::fprintf(stderr, "strata trace: prompt chunk %lld of %lld\n", (long long) c0, (long long) n); std::fflush(stderr); }const int64_t T = std::min(C, n - c0), p0 = pos0 + c0;+ // STRATA_KVA: this chunk is approximated in the layers at and after the split when its last position is+ // before the request's full-computation suffix; a chunk in the suffix runs the ordinary path everywhere.+ const bool kva_approx = kva_run && (p0 + T <= kva_frontier);+ if (kva_approx) {+ kva_positions += T;+ ++kva_chunks;+ }core::progress_at("reading the prompt (batched): preparing the chunk from token", p0); // #251#if defined(STRATA_HAVE_DFLASH2)// DFlash2 prompt taps: the rows of this chunk at positions >= tap_first_ (the drafter looks back one window)@@ -2959,7 +3097,9 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::strim.pp->p_issued = m.pp->pk = 0;}double ps_acc = 0.0;- for (int64_t l = LB; l < LE; ++l) {+ // STRATA_KVA: an approximated chunk queues no transfer for a layer whose MoE half will not run+ const int64_t LE_stream = kva_approx ? std::min<int64_t>(LE, kva_split) : LE;+ for (int64_t l = LB; l < LE_stream; ++l) {seq_start[(size_t) l] = seq.size();if (ps_on) m.pp->pseq_start[(size_t) l] = m.pp->pseq.size();for (int32_t e = 0; e < m.g->n_expert; ++e) {@@ -2990,7 +3130,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striseq.push_back({(int32_t) l, e, b, job});}}- for (int64_t l = LE; l <= g.n_layers; ++l) {+ for (int64_t l = LE_stream; l <= g.n_layers; ++l) {seq_start[(size_t) l] = seq.size();if (ps_on) m.pp->pseq_start[(size_t) l] = m.pp->pseq.size();}@@ -3112,6 +3252,11 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::striif (l > LB) pf_step("reading the prompt (batched, step sync): the experts and the rest of layer", l - 1);core::progress_at("reading the prompt (batched): layer", l, p0); // #251: a stall names layer and chunkconst core::LayerView v(*m.wt, l);+ // STRATA_KVA: `kva_layer` means this layer's output has no consumer left for this chunk, so only its+ // state advances; `kva_subst` means the block input itself is replayed from a capture (oracle mode).+ const bool kva_layer = kva_approx && l >= kva_split;+ const bool kva_subst = kva_layer && kva.mode == KvaConfig::Mode::Oracle;+ const bool kva_cap_layer = kva.capture && LB == 0 && (LE < 0 || LE == g.n_layers) && l >= kva_split;if (l == std::max<int64_t>(LB, 1)) { // the PLE rows, read from layer 1 on (a wait for the SSD read of the first chunk lands on "ple")pt.mark(kPfPle, cs);if (!ple_land()) return false;@@ -3171,6 +3316,13 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri}stats_.ms_ple += ms_since(tp);}+ // STRATA_KVA: the capture and the oracle hook the residual ENTERING the layer - the vector a projector+ // has to produce for it. A capture run is an ordinary full read; the oracle replaces the input.+ if (kva_cap_layer && !kva_capture.write(l, m.R, T, p0, D, cs)) { err = kva_capture.err; return false; }+ if (kva_subst) {+ if (!kva_oracle.read(l, m.R, T, p0, D, cs)) { err = kva_oracle.err; return false; }+ normed = false; // the previous half's norm of R is stale once the input is replaced+ }for (int half = 0; half < 2; ++half) {// ---- the hyper-connection read of this halfconst char* pre = half == 0 ? "hc_attn_" : "hc_ffn_";@@ -3277,8 +3429,10 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::strigdn_recurrence(state, m.hbuf, m.gate, m.beta, m.z, (const float*) wnm->data, EPS, m.y, m.y_h, T, m.cs,ld_y);pt.mark(kPfGdnOut, cs);- if (!native_proj(m.gemm, wo, m.y_h, m.bo, T, v.name("ssm_out.weight"), err, 0, ld_y)) return false;+ if (!kva_layer) // KVA: the out projection feeds only the residual write and the MoE half+ if (!native_proj(m.gemm, wo, m.y_h, m.bo, T, v.name("ssm_out.weight"), err, 0, ld_y)) return false;++gdn_index;+ if (kva_layer) break; // state advanced: no gated residual write, no MoE half of this layer} else if (half == 0) {// ======================= QSA =======================const core::QsaState& st = ss.qsa_states[qsa_index];@@ -3293,9 +3447,10 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stript.mark(kPfQsa, cs);if (!native_proj(m.gemm, wk, m.mixed_h, m.Kc, T, v.name("attn_k.weight"), err)) return false;if (!native_proj(m.gemm, wv, m.mixed_h, m.Vc, T, v.name("attn_v.weight"), err)) return false;- if (!native_proj(m.gemm, wq, m.mixed_h, m.Qf, T, v.name("attn_q.weight"), err)) return false;+ // KVA: the query projections have no consumer when only the KV/indexer state advances+ if (!kva_layer && !native_proj(m.gemm, wq, m.mixed_h, m.Qf, T, v.name("attn_q.weight"), err)) return false;if (!bf16_proj(m.gemm, wik, m.mixed_bf, m.idx_raw, T, v.name("indexer.k_proj.weight"), err, 0, m.mixed_bf_lo)) return false;- if (!bf16_proj(m.gemm, wiq, m.mixed_bf, m.q_idx, T, v.name("indexer.q_proj.weight"), err, 0, m.mixed_bf_lo)) return false;+ if (!kva_layer && !bf16_proj(m.gemm, wiq, m.mixed_bf, m.q_idx, T, v.name("indexer.q_proj.weight"), err, 0, m.mixed_bf_lo)) return false;rms_rows(m.Kc, (const float*) wkn->data, T * 2, 256, 256, EPS, m.cs);rope(m.Kc, T, 2, 256, 512, p0, strata::kernels::rope_scaling(), m.cs);// KV streaming: this layer's cells [0, p0) come in from the host copy to the staging pool, and the@@ -3384,6 +3539,14 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stristrata::kernels::rope_scaling(), m.cs);}} catch (const std::exception& e) { err = std::string("prefill indexer: ") + e.what(); return false; }+ // KVA: the KV pages and the indexer's keys/pool are complete for this chunk. Nothing that+ // reads them (query side, selection, attention, output projection, the MoE half) has a+ // consumer left, so the layer ends here.+ if (kva_layer) {+ ++qsa_index;+ if (!kv_prefetch_after(l + 1, qsa_index)) return false;+ break;+ }pt.mark(kPfQsaSel, cs);for (int64_t t0 = 0; t0 < T; t0 += m.sel_batch) {const int64_t nb = std::min(m.sel_batch, T - t0);@@ -5074,6 +5237,19 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri(long long) m.pp->ps_experts, m.pp->layers ? (double) m.pp->ps_experts / m.pp->layers : 0.0);}}+ if (kva_run || kva.capture) {+ const char* mode = kva.mode == KvaConfig::Mode::Oracle ? "oracle"+ : kva.mode == KvaConfig::Mode::Linear ? "linear" : "identity";+ std::fprintf(stderr, "strata prefill: KVA %s: split %lld, suffix %lld, approximated %lld of %lld positions "+ "in %lld of %lld chunks, mode %s%s\n",+ kva_run ? "on" : "off (capture)", (long long) kva_split, (long long) kva.suffix,+ (long long) kva_positions, (long long) n, (long long) kva_chunks,+ (long long) (n > 0 && m.T > 0 ? (n + m.T - 1) / m.T : 0), mode,+ kva.capture ? ", capturing the late layers' inputs" : "");+ if (kva_positions > 0)+ std::fprintf(stderr, "strata prefill: KVA: the session state holds approximated cells; it is not an exact "+ "prefix and must not be reused as one (no checkpoint, no follow-up reuse)\n");+ }if (std::getenv("STRATA_STATE_HASH_GDN") != nullptr) { // debug: the GDN states as the prompt path leaves themcudaStreamSynchronize(m.cs);std::vector<uint8_t> b((size_t) gdn_floats * 4);