Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

KVA: optional approximate late-layer prefill (split 24, chunk-aligned suffix, capture/oracle modes)

Niko1221f4012d63690c
diff --git a/include/strata/prefill/kva.hpp b/include/strata/prefill/kva.hpp
new file mode 100644
index 000000000..560effd45
--- /dev/null
+++ b/include/strata/prefill/kva.hpp
@@ -0,0 +1,97 @@
+// include/strata/prefill/kva.hpp - STRATA_KVA: the optional approximate-prefill plan (research, opt-in).
+//
+// After layer `split` the prompt's late layers need only their recurrent state (GDN recurrence + conv history)
+// and their KV/indexer caches; the attention output, the output projection, the MoE half and its expert
+// transfers have no remaining consumer for the approximate positions. The plan is:
+//
+// positions [0, n - suffix) approximated: late layers run state-only on a predicted block input
+// positions [n - suffix, n) fully computed (the suffix), request-global
+//
+// v1 keeps the suffix on a CHUNK boundary (a chunk is approximated only when its last position is before the
+// frontier), so no layer ever has to split one chunk into two regimes. The late layers' block input is the
+// hyper-connection residual R (10,240 floats = hc * n_embd) that the chunk's layer loop carries; it comes from
+// identity the residual entering layer `split` (no projector at all, the zero-cost baseline),
+// oracle a captured real input per late layer (STRATA_KVA_ORACLE=<dir>; the plumbing/parity mode),
+// linear a fitted low-rank projector (STRATA_KVA_PROJECTOR=<file>; not implemented in this step).
+//
+// Everything here is read from the environment once per process and is off unless STRATA_KVA is set, so a
+// default build behaves exactly as before.
+#pragma once
+
+#include <cstdint>
+#include <cstdlib>
+#include <string>
+
+namespace strata::prefill {
+
+struct KvaConfig {
+ enum class Mode { Identity, Oracle, Linear };
+
+ bool on = false; ///< STRATA_KVA=1
+ bool capture = false; ///< STRATA_KVA_CAPTURE=<dir>: dump every late layer's real input rows
+ bool capture_fp32 = false; ///< STRATA_KVA_CAPTURE_FP32=1: fp32 records (else bf16; fp32 is for parity runs)
+ bool oracle_fp32 = false; ///< STRATA_KVA_ORACLE_FP32=1: the oracle files are fp32 (else bf16)
+ bool parity = false; ///< STRATA_KVA_PARITY=1: report the state-only path against the full path
+ int64_t split = 24; ///< STRATA_KVA_SPLIT: first approximated layer (zero-based)
+ int64_t suffix = 2048; ///< STRATA_KVA_SUFFIX: fully computed tail positions (rounded up to a chunk)
+ int64_t min_prompt = 4096; ///< STRATA_KVA_MIN_PROMPT: shorter prompts stay on the full path
+ Mode mode = Mode::Identity; ///< STRATA_KVA_MODE=identity|oracle|linear
+ std::string oracle_dir; ///< STRATA_KVA_ORACLE
+ std::string capture_dir; ///< STRATA_KVA_CAPTURE
+ std::string projector; ///< STRATA_KVA_PROJECTOR
+
+ /// Does this run qualify at all? A cold read of one prompt, on one stage, long enough for a suffix to
+ /// leave approximated positions behind it.
+ bool applies(int64_t pos0, int64_t n, int64_t lb, int64_t le, int64_t n_layers) const {
+ if (!on || capture) return false; // capture runs are ordinary full reads
+ if (pos0 != 0 || lb != 0 || (le >= 0 && le != n_layers)) return false;
+ return n >= min_prompt && n > suffix + 64;
+ }
+};
+
+inline int64_t kva_env_i64(const char* name, int64_t dflt) {
+ const char* v = std::getenv(name);
+ if (v == nullptr || *v == '\0') return dflt;
+ return (int64_t) std::atoll(v);
+}
+
+inline bool kva_env_flag(const char* name, bool dflt) {
+ const char* v = std::getenv(name);
+ if (v == nullptr || *v == '\0') return dflt;
+ return v[0] != '0';
+}
+
+/// The process's KVA settings, parsed once (the engine's own options are parsed once too).
+inline const KvaConfig& kva_config() {
+ static const KvaConfig c = [] {
+ KvaConfig k;
+ k.on = kva_env_flag("STRATA_KVA", false);
+ k.split = kva_env_i64("STRATA_KVA_SPLIT", 24);
+ k.suffix = kva_env_i64("STRATA_KVA_SUFFIX", 2048);
+ k.min_prompt = kva_env_i64("STRATA_KVA_MIN_PROMPT", 4096);
+ k.parity = kva_env_flag("STRATA_KVA_PARITY", false);
+ if (const char* d = std::getenv("STRATA_KVA_CAPTURE"); d != nullptr && *d != '\0') {
+ k.capture = true;
+ k.capture_dir = d;
+ k.capture_fp32 = kva_env_flag("STRATA_KVA_CAPTURE_FP32", false);
+ }
+ if (const char* d = std::getenv("STRATA_KVA_ORACLE"); d != nullptr && *d != '\0') {
+ k.oracle_dir = d;
+ k.mode = KvaConfig::Mode::Oracle;
+ k.oracle_fp32 = kva_env_flag("STRATA_KVA_ORACLE_FP32", false);
+ }
+ if (const char* d = std::getenv("STRATA_KVA_PROJECTOR"); d != nullptr && *d != '\0') {
+ k.projector = d;
+ k.mode = KvaConfig::Mode::Linear;
+ }
+ if (const char* m = std::getenv("STRATA_KVA_MODE"); m != nullptr && *m != '\0') {
+ if (std::string(m) == "oracle") k.mode = KvaConfig::Mode::Oracle;
+ else if (std::string(m) == "linear") k.mode = KvaConfig::Mode::Linear;
+ else k.mode = KvaConfig::Mode::Identity;
+ }
+ return k;
+ }();
+ return c;
+}
+
+} // namespace strata::prefill
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index 6cb888e50..d69d03ae3 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -46,6 +46,7 @@
#include "strata/prefill/moe_mmq.hpp"
#include "strata/core/peer_experts.hpp"
#include "strata/prefill/kernels.hpp"
+#include "strata/prefill/kva.hpp" // STRATA_KVA: the optional approximate-prefill plan (research, off by default)
#include <cuda_runtime.h>
@@ -2582,6 +2583,123 @@ struct PeTimer {
cudaSetDevice(prev);
}
};
+
+// ---- KVA (STRATA_KVA, research): the capture and oracle taps of the late layers' block inputs ---------------
+//
+// A capture run writes, for every layer at or after the split and every chunk, the chunk's multi-stream residual
+// R [T, hc*n_embd] as it ENTERS the layer - exactly the vector a projector would have to produce. One file per
+// layer (`L<layer>.bin`), rows in position order, bf16 (fp32 with STRATA_KVA_CAPTURE_FP32, for parity runs): a
+// reader seeks straight to a position. The oracle mode replays such a file in place of a projector, which is how
+// the state-only path is checked against the full one before any weights are fitted.
+struct KvaTap {
+ std::string dir;
+ bool fp32 = false;
+ std::vector<std::FILE*> f; // one per layer, opened on first use
+ std::vector<int64_t> next_pos; // where the next record must start (0: nothing written yet)
+ std::vector<float> host; // one chunk's rows, fp32
+ std::vector<uint16_t> bf; // their bf16 image
+ std::string err;
+
+ bool open(const std::string& d, bool is_fp32) {
+ dir = d;
+ fp32 = is_fp32;
+ f.assign(128, nullptr);
+ next_pos.assign(128, 0);
+ return true;
+ }
+ void close() {
+ for (std::FILE*& p : f) { if (p != nullptr) std::fclose(p); p = nullptr; }
+ }
+ /// A new cold read starts: a capture file holds one prompt's rows, so every layer starts over.
+ void begin_prompt() {
+ close();
+ if (next_pos.empty()) next_pos.assign(f.size(), 0);
+ for (int64_t& p : next_pos) p = 0;
+ }
+ std::string path(int64_t layer) const { return dir + "/L" + std::to_string((long long) layer) + ".bin"; }
+
+ /// The chunk's rows of `layer` from the device residual `R` - a capture.
+ bool write(int64_t layer, const float* R, int64_t T, int64_t p0, int64_t D, cudaStream_t cs) {
+ if (layer < 0 || layer >= (int64_t) f.size()) { err = "kva capture: layer out of range"; return false; }
+ if (next_pos[(size_t) layer] != p0) {
+ err = "kva capture: layer " + std::to_string((long long) layer) + " expects position " +
+ std::to_string((long long) next_pos[(size_t) layer]) + ", got " + std::to_string((long long) p0) +
+ " (a capture needs one cold, in-order read)";
+ return false;
+ }
+ host.resize((size_t) T * D);
+ if (cudaMemcpyAsync(host.data(), R, (size_t) T * D * 4, cudaMemcpyDeviceToHost, cs) != cudaSuccess ||
+ cudaStreamSynchronize(cs) != cudaSuccess) {
+ err = "kva capture: the residual copy failed";
+ return false;
+ }
+ std::FILE*& p = f[(size_t) layer];
+ if (p == nullptr) {
+ p = std::fopen(path(layer).c_str(), "wb");
+ if (p == nullptr) { err = "kva capture: cannot write " + path(layer); return false; }
+ }
+ if (fp32) {
+ std::fwrite(host.data(), 4, (size_t) T * D, p);
+ } else {
+ bf.resize((size_t) T * D);
+ for (size_t i = 0; i < bf.size(); ++i) { // round to nearest even, as STRATA_PREFILL_DUMP_R_ALL does
+ uint32_t u;
+ std::memcpy(&u, &host[i], 4);
+ u += 0x7fffu + ((u >> 16) & 1u);
+ bf[i] = (uint16_t) (u >> 16);
+ }
+ std::fwrite(bf.data(), 2, bf.size(), p);
+ }
+ std::fflush(p);
+ next_pos[(size_t) layer] = p0 + T;
+ return true;
+ }
+
+ /// The chunk's rows of `layer` into the device residual `R` - an oracle replay.
+ bool read(int64_t layer, float* R, int64_t T, int64_t p0, int64_t D, cudaStream_t cs) {
+ if (layer < 0 || layer >= (int64_t) f.size()) { err = "kva oracle: layer out of range"; return false; }
+ std::FILE*& p = f[(size_t) layer];
+ if (p == nullptr) {
+ p = std::fopen(path(layer).c_str(), "rb");
+ if (p == nullptr) { err = "kva oracle: cannot read " + path(layer); return false; }
+ }
+ const int64_t n = T * D;
+ const int64_t esz = fp32 ? 4 : 2;
+ const long long off = (long long) ((int64_t) p0 * D * esz);
+#if defined(_WIN32)
+ const bool seek_ok = _fseeki64(p, off, SEEK_SET) == 0;
+#else
+ const bool seek_ok = fseeko(p, (off_t) off, SEEK_SET) == 0;
+#endif
+ if (!seek_ok) { err = "kva oracle: cannot seek " + path(layer); return false; }
+ host.resize((size_t) n);
+ if (fp32) {
+ const size_t got = std::fread(host.data(), 4, (size_t) n, p);
+ if (got != (size_t) n) {
+ err = "kva oracle: short read of " + path(layer) + " at position " + std::to_string((long long) p0) +
+ ": wanted " + std::to_string((long long) n) + " values, got " + std::to_string((long long) got);
+ return false;
+ }
+ } else {
+ bf.resize((size_t) n);
+ const size_t got = std::fread(bf.data(), 2, (size_t) n, p);
+ if (got != (size_t) n) {
+ err = "kva oracle: short read of " + path(layer) + " at position " + std::to_string((long long) p0) +
+ ": wanted " + std::to_string((long long) n) + " values, got " + std::to_string((long long) got);
+ return false;
+ }
+ for (size_t i = 0; i < bf.size(); ++i) { // bf16 -> fp32: the top half, exactly
+ const uint32_t u = (uint32_t) bf[i] << 16;
+ std::memcpy(&host[i], &u, 4);
+ }
+ }
+ if (cudaMemcpyAsync(R, host.data(), (size_t) n * 4, cudaMemcpyHostToDevice, cs) != cudaSuccess) {
+ err = "kva oracle: the upload failed";
+ return false;
+ }
+ return true;
+ }
+};
} // namespace
bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err) {
@@ -2595,6 +2713,19 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
core::SessionState& ss = *m.ss;
const auto t_start = Clock::now();
const int64_t LB = stage_lb_, LE = stage_le_;
+ // STRATA_KVA: the approximate-prefix plan of this run, if it qualifies (a cold read of one long prompt, one
+ // stage, no capture). Positions [0, kva_frontier) are approximated in the layers at and after the split.
+ const KvaConfig& kva = kva_config();
+ const bool kva_run = kva.applies(pos0, n, LB, LE < 0 ? -1 : LE, (int64_t) g.n_layers);
+ const int64_t kva_split = std::min<int64_t>(std::max<int64_t>(kva.split, 1), g.n_layers);
+ const int64_t kva_frontier = std::max<int64_t>(0, n - kva.suffix);
+ static KvaTap kva_capture; // process-wide: one cold read at a time, so the per-layer position is enough
+ static KvaTap kva_oracle;
+ int64_t kva_positions = 0, kva_chunks = 0; // what actually took the approximate path (diagnostics)
+ if (kva.capture && kva_capture.f.empty()) kva_capture.open(kva.capture_dir, kva.capture_fp32);
+ if (kva.capture && pos0 == 0 && hand_in_ == nullptr) kva_capture.begin_prompt(); // one prompt per capture
+ if (kva_run && kva.mode == KvaConfig::Mode::Oracle && kva_oracle.f.empty())
+ kva_oracle.open(kva.oracle_dir, kva.oracle_fp32);
// The direct successor's future lives on the Prefill object. Intermediate
// stages therefore do not drain the complete remaining GPU chain here.
double host_sync_ms = 0, host_chunk_ms = 0, host_setup_ms = 0; // STRATA_PREFILL_TIMING: the host's share
@@ -2719,6 +2850,13 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
}
if (std::getenv("STRATA_TRACE")) { std::fprintf(stderr, "strata trace: prompt chunk %lld of %lld\n", (long long) c0, (long long) n); std::fflush(stderr); }
const int64_t T = std::min(C, n - c0), p0 = pos0 + c0;
+ // STRATA_KVA: this chunk is approximated in the layers at and after the split when its last position is
+ // before the request's full-computation suffix; a chunk in the suffix runs the ordinary path everywhere.
+ const bool kva_approx = kva_run && (p0 + T <= kva_frontier);
+ if (kva_approx) {
+ kva_positions += T;
+ ++kva_chunks;
+ }
core::progress_at("reading the prompt (batched): preparing the chunk from token", p0); // #251
#if defined(STRATA_HAVE_DFLASH2)
// DFlash2 prompt taps: the rows of this chunk at positions >= tap_first_ (the drafter looks back one window)
@@ -2959,7 +3097,9 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
m.pp->p_issued = m.pp->pk = 0;
}
double ps_acc = 0.0;
- for (int64_t l = LB; l < LE; ++l) {
+ // STRATA_KVA: an approximated chunk queues no transfer for a layer whose MoE half will not run
+ const int64_t LE_stream = kva_approx ? std::min<int64_t>(LE, kva_split) : LE;
+ for (int64_t l = LB; l < LE_stream; ++l) {
seq_start[(size_t) l] = seq.size();
if (ps_on) m.pp->pseq_start[(size_t) l] = m.pp->pseq.size();
for (int32_t e = 0; e < m.g->n_expert; ++e) {
@@ -2990,7 +3130,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
seq.push_back({(int32_t) l, e, b, job});
}
}
- for (int64_t l = LE; l <= g.n_layers; ++l) {
+ for (int64_t l = LE_stream; l <= g.n_layers; ++l) {
seq_start[(size_t) l] = seq.size();
if (ps_on) m.pp->pseq_start[(size_t) l] = m.pp->pseq.size();
}
@@ -3112,6 +3252,11 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
if (l > LB) pf_step("reading the prompt (batched, step sync): the experts and the rest of layer", l - 1);
core::progress_at("reading the prompt (batched): layer", l, p0); // #251: a stall names layer and chunk
const core::LayerView v(*m.wt, l);
+ // STRATA_KVA: `kva_layer` means this layer's output has no consumer left for this chunk, so only its
+ // state advances; `kva_subst` means the block input itself is replayed from a capture (oracle mode).
+ const bool kva_layer = kva_approx && l >= kva_split;
+ const bool kva_subst = kva_layer && kva.mode == KvaConfig::Mode::Oracle;
+ const bool kva_cap_layer = kva.capture && LB == 0 && (LE < 0 || LE == g.n_layers) && l >= kva_split;
if (l == std::max<int64_t>(LB, 1)) { // the PLE rows, read from layer 1 on (a wait for the SSD read of the first chunk lands on "ple")
pt.mark(kPfPle, cs);
if (!ple_land()) return false;
@@ -3171,6 +3316,13 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
}
stats_.ms_ple += ms_since(tp);
}
+ // STRATA_KVA: the capture and the oracle hook the residual ENTERING the layer - the vector a projector
+ // has to produce for it. A capture run is an ordinary full read; the oracle replaces the input.
+ if (kva_cap_layer && !kva_capture.write(l, m.R, T, p0, D, cs)) { err = kva_capture.err; return false; }
+ if (kva_subst) {
+ if (!kva_oracle.read(l, m.R, T, p0, D, cs)) { err = kva_oracle.err; return false; }
+ normed = false; // the previous half's norm of R is stale once the input is replaced
+ }
for (int half = 0; half < 2; ++half) {
// ---- the hyper-connection read of this half
const char* pre = half == 0 ? "hc_attn_" : "hc_ffn_";
@@ -3277,8 +3429,10 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
gdn_recurrence(state, m.hbuf, m.gate, m.beta, m.z, (const float*) wnm->data, EPS, m.y, m.y_h, T, m.cs,
ld_y);
pt.mark(kPfGdnOut, cs);
- if (!native_proj(m.gemm, wo, m.y_h, m.bo, T, v.name("ssm_out.weight"), err, 0, ld_y)) return false;
+ if (!kva_layer) // KVA: the out projection feeds only the residual write and the MoE half
+ if (!native_proj(m.gemm, wo, m.y_h, m.bo, T, v.name("ssm_out.weight"), err, 0, ld_y)) return false;
++gdn_index;
+ if (kva_layer) break; // state advanced: no gated residual write, no MoE half of this layer
} else if (half == 0) {
// ======================= QSA =======================
const core::QsaState& st = ss.qsa_states[qsa_index];
@@ -3293,9 +3447,10 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
pt.mark(kPfQsa, cs);
if (!native_proj(m.gemm, wk, m.mixed_h, m.Kc, T, v.name("attn_k.weight"), err)) return false;
if (!native_proj(m.gemm, wv, m.mixed_h, m.Vc, T, v.name("attn_v.weight"), err)) return false;
- if (!native_proj(m.gemm, wq, m.mixed_h, m.Qf, T, v.name("attn_q.weight"), err)) return false;
+ // KVA: the query projections have no consumer when only the KV/indexer state advances
+ if (!kva_layer && !native_proj(m.gemm, wq, m.mixed_h, m.Qf, T, v.name("attn_q.weight"), err)) return false;
if (!bf16_proj(m.gemm, wik, m.mixed_bf, m.idx_raw, T, v.name("indexer.k_proj.weight"), err, 0, m.mixed_bf_lo)) return false;
- if (!bf16_proj(m.gemm, wiq, m.mixed_bf, m.q_idx, T, v.name("indexer.q_proj.weight"), err, 0, m.mixed_bf_lo)) return false;
+ if (!kva_layer && !bf16_proj(m.gemm, wiq, m.mixed_bf, m.q_idx, T, v.name("indexer.q_proj.weight"), err, 0, m.mixed_bf_lo)) return false;
rms_rows(m.Kc, (const float*) wkn->data, T * 2, 256, 256, EPS, m.cs);
rope(m.Kc, T, 2, 256, 512, p0, strata::kernels::rope_scaling(), m.cs);
// KV streaming: this layer's cells [0, p0) come in from the host copy to the staging pool, and the
@@ -3384,6 +3539,14 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
strata::kernels::rope_scaling(), m.cs);
}
} catch (const std::exception& e) { err = std::string("prefill indexer: ") + e.what(); return false; }
+ // KVA: the KV pages and the indexer's keys/pool are complete for this chunk. Nothing that
+ // reads them (query side, selection, attention, output projection, the MoE half) has a
+ // consumer left, so the layer ends here.
+ if (kva_layer) {
+ ++qsa_index;
+ if (!kv_prefetch_after(l + 1, qsa_index)) return false;
+ break;
+ }
pt.mark(kPfQsaSel, cs);
for (int64_t t0 = 0; t0 < T; t0 += m.sel_batch) {
const int64_t nb = std::min(m.sel_batch, T - t0);
@@ -5074,6 +5237,19 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
(long long) m.pp->ps_experts, m.pp->layers ? (double) m.pp->ps_experts / m.pp->layers : 0.0);
}
}
+ if (kva_run || kva.capture) {
+ const char* mode = kva.mode == KvaConfig::Mode::Oracle ? "oracle"
+ : kva.mode == KvaConfig::Mode::Linear ? "linear" : "identity";
+ std::fprintf(stderr, "strata prefill: KVA %s: split %lld, suffix %lld, approximated %lld of %lld positions "
+ "in %lld of %lld chunks, mode %s%s\n",
+ kva_run ? "on" : "off (capture)", (long long) kva_split, (long long) kva.suffix,
+ (long long) kva_positions, (long long) n, (long long) kva_chunks,
+ (long long) (n > 0 && m.T > 0 ? (n + m.T - 1) / m.T : 0), mode,
+ kva.capture ? ", capturing the late layers' inputs" : "");
+ if (kva_positions > 0)
+ std::fprintf(stderr, "strata prefill: KVA: the session state holds approximated cells; it is not an exact "
+ "prefix and must not be reused as one (no checkpoint, no follow-up reuse)\n");
+ }
if (std::getenv("STRATA_STATE_HASH_GDN") != nullptr) { // debug: the GDN states as the prompt path leaves them
cudaStreamSynchronize(m.cs);
std::vector<uint8_t> b((size_t) gdn_floats * 4);