Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
11.0 KB
Raw Blame
// include/strata/prefill/kva.hpp - STRATA_KVA: the approximate late-layer prompt read (opt-in since 0.2.7).
//
// After layer `split` the prompt's late layers need only their recurrent state (GDN recurrence + conv history)
// and their KV/indexer caches; the attention output, the output projection, the MoE half and its expert
// transfers have no remaining consumer for the approximate positions.  The plan is:
//
//   positions [0, n - suffix)          approximated: late layers run state-only on a predicted block input
//   positions [n - suffix, n)          fully computed (the suffix), request-global
//
// v1 keeps the suffix on a CHUNK boundary (a chunk is approximated only when its last position is before the
// frontier), so no layer ever has to split one chunk into two regimes.  The late layers' block input is the
// hyper-connection residual R (10,240 floats = hc * n_embd) that the chunk's layer loop carries; it comes from
//   identity  the residual entering layer `split` (no projector at all, the zero-cost baseline),
//   oracle    a captured real input per late layer (STRATA_KVA_ORACLE=<dir>; the plumbing/parity mode),
//   linear    a fitted low-rank projector (STRATA_KVA_PROJECTOR=<file>; not implemented in this step).
//
// 0.2.7: the read is OFF unless STRATA_KVA=1 is set (the quality runs found lost records and invented rows in long
// extraction and agent tasks, docs/KVA.md).  Unset, nothing here runs and the engine is 0.2.6's.  Switched on, it applies
// to a cold prompt of STRATA_KVA_MIN_PROMPT tokens or more on one CUDA/HIP device, approximating only the layers from
// 36 on, with a 2048-token exact tail (the configuration the 0.2.7 quality runs found cleanest; STRATA_KVA_SPLIT and
// STRATA_KVA_SUFFIX override).  Everything is read from the environment once per process.
//
// What the engine tells this header at start-up (kva_engine()): whether its configuration rules the read out for the
// whole process (batch slots, a layer split, a peer device).
//
// The draft layer (MTP) builds its K/V from the final rows of the prompt, which an approximated chunk does not have.
// Its window is NOT changed, and by default neither is anything else: the draft cells of an approximated chunk are
// built from the rows that chunk has and the draft layer attends to them (measured: draft acceptance after a KVA
// read stays within noise on the P100).  The research option STRATA_KVA_DRAFT_FLOOR=1 gives the draft layer a "cell
// floor" per session (MtpDrafter::set_cell_floor; kva_floor.hpp keeps it with the tokens of the session,
// window_start.hpp is the one rule the kernel and the tests share): the first cell built from a real row, below
// which no draft cell is built or read.  The target model verifies every token whatever the draft read.
#pragma once

#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <string>

namespace strata::prefill {

/// Does this build have the state-only late-layer path?  CUDA and HIP: yes.  The Metal engine compiles prefill.cpp
/// against its own kernel library, which has none.
#if defined(STRATA_GPU_METAL)
inline constexpr bool kKvaBackend = false;
#else
inline constexpr bool kKvaBackend = true;
#endif

/// The reason decline() gives for a model whose shape KVA was not measured on (prefill.cpp prints it with the numbers).
inline constexpr const char* kKvaShapeReason = "this model's shape is not one KVA was measured on";

/// The fully computed tail of every approximated read.
inline constexpr int64_t kKvaSuffixDefault = 2048;
/// The first approximated layer when switched on: 36 of 48 (the cleanest measured; 24 lost records and agent loops).
inline constexpr int64_t kKvaSplitDefault = 36;

/// What the running engine knows and the environment cannot say.
struct KvaEngine {
    std::string blocked;           ///< non-empty: this process never reads approximately, and why
};

inline KvaEngine& kva_engine() {
    static KvaEngine e;
    return e;
}

struct KvaConfig {
    enum class Mode { Identity, Oracle, Linear };
    enum class Only { All, Gdn, Qsa };
    /// What a late layer of an approximated chunk still does: StateOnly (the default) advances only its state (GDN
    /// recurrence, K/V and indexer pages); Moe runs the whole attention half (outputs into the residual) and skips only
    /// the MoE half and its expert transfers.
    enum class Skip { StateOnly, Moe };

    bool on = false;             ///< STRATA_KVA=1 (and this build has a path); unset or 0: off
    bool explicit_on = false;    ///< STRATA_KVA was set to something other than 0: every declined read says why
    bool capture = false;        ///< STRATA_KVA_CAPTURE=<dir>: dump every late layer's real input rows
    bool capture_fp32 = false;   ///< STRATA_KVA_CAPTURE_FP32=1: fp32 records (else bf16; fp32 is for parity runs)
    bool oracle_fp32 = false;    ///< STRATA_KVA_ORACLE_FP32=1: the oracle files are fp32 (else bf16)
    bool parity = false;         ///< STRATA_KVA_PARITY=1: report the state-only path against the full path
    bool any_model = false;      ///< STRATA_KVA_ANY_MODEL=1 (research): also run on a shape KVA was not measured on, with a warning
    bool draft_floor = false;    ///< STRATA_KVA_DRAFT_FLOOR=1: the draft layer skips the cells of approximated chunks (research; default: it reads them)
    int64_t split = kKvaSplitDefault;   ///< STRATA_KVA_SPLIT: first approximated layer (zero-based)
    int64_t suffix_env = 0;      ///< STRATA_KVA_SUFFIX: fully computed tail positions (0 = not set: kKvaSuffixDefault)
    int64_t min_prompt = 4096;   ///< STRATA_KVA_MIN_PROMPT: shorter prompts stay on the full path
    Mode mode = Mode::Identity;  ///< STRATA_KVA_MODE=identity|oracle|linear
    Only only = Only::All;       ///< STRATA_KVA_ONLY=all|gdn|qsa: which late layers take the approximate path
    Skip skip = Skip::StateOnly; ///< STRATA_KVA_SKIP=moe: skip only the MoE half of the late layers (opt-in)
    std::string oracle_dir;      ///< STRATA_KVA_ORACLE
    std::string capture_dir;     ///< STRATA_KVA_CAPTURE
    std::string projector;       ///< STRATA_KVA_PROJECTOR

    /// The fully computed tail of a read: STRATA_KVA_SUFFIX when set, otherwise 2048.
    int64_t effective_suffix() const { return suffix_env > 0 ? suffix_env : kKvaSuffixDefault; }

    /// Does the draft layer need the cell floor at all?  Only when a read can approximate something: KVA on, not ruled
    /// out for the process, and a draft layer loaded.  Otherwise the draft layer's attention is 0.2.6's own code.
    bool wants_cell_floor(const KvaEngine& e, bool has_drafter) const {
        return kKvaBackend && on && draft_floor && e.blocked.empty() && has_drafter;
    }

    /// Why this read cannot take the approximate path, or nullptr when it can: a cold read of one prompt (not a
    /// continuation, no layer split, no pictures, no DFlash2 taps), on an engine that allows it, long enough for a
    /// suffix to leave approximated positions behind it.  `routine` (optional) is set when the reason is an
    /// everyday one (a continuation, a short prompt) that is not worth a log line.
    const char* decline(const KvaEngine& e, int64_t pos0, int64_t n, int64_t lb, int64_t le, int64_t n_layers,
                        bool pictures = false, bool taps = false, bool* routine = nullptr,
                        bool unmeasured_shape = false) const {
        if (routine) *routine = false;
        if (!on) return "off (STRATA_KVA is not 1)";
        if (capture) return "capture mode";
        if (!e.blocked.empty()) return e.blocked.c_str();
        if (pos0 != 0) {
            if (routine) *routine = true;
            return "a continuation read (pos0 != 0)";
        }
        if (lb != 0 || (le >= 0 && le != n_layers)) return "a layer split (not the whole trunk)";
        if (pictures) return "a prompt with pictures";
        if (taps) return "the DFlash2 drafter reads the late layers' hidden states";
        if (n < min_prompt) {
            if (routine) *routine = true;
            return "the prompt is shorter than STRATA_KVA_MIN_PROMPT";
        }
        if (n <= effective_suffix() + 64) {
            if (routine) *routine = true;
            return "the suffix leaves too few positions to approximate";
        }
        // the split layer, the state-only layers and the tail were measured on one geometry (Qwen3.8-Flash-Next, 48
        // layers, 512 experts); another model reads exactly unless the research override asks for it
        if (unmeasured_shape && !any_model) return kKvaShapeReason;
        return nullptr;
    }
};

inline int64_t kva_env_i64(const char* name, int64_t dflt) {
    const char* v = std::getenv(name);
    if (v == nullptr || *v == '\0') return dflt;
    return (int64_t) std::atoll(v);
}

inline bool kva_env_flag(const char* name, bool dflt) {
    const char* v = std::getenv(name);
    if (v == nullptr || *v == '\0') return dflt;
    return v[0] != '0';
}

/// The settings as the environment says now (uncached: the tests call it after changing the environment).
inline KvaConfig kva_config_from_env() {
    KvaConfig k;
    const char* sw = std::getenv("STRATA_KVA");
    const bool set = sw != nullptr && *sw != '\0';
    k.on = set && sw[0] != '0';
    k.explicit_on = k.on;
    if (!kKvaBackend) k.on = false;
    k.split = kva_env_i64("STRATA_KVA_SPLIT", kKvaSplitDefault);
    k.suffix_env = std::max<int64_t>(0, kva_env_i64("STRATA_KVA_SUFFIX", 0));
    k.min_prompt = kva_env_i64("STRATA_KVA_MIN_PROMPT", 4096);
    k.parity = kva_env_flag("STRATA_KVA_PARITY", false);
    k.draft_floor = kva_env_flag("STRATA_KVA_DRAFT_FLOOR", false);
    k.any_model = kva_env_flag("STRATA_KVA_ANY_MODEL", false);
    if (const char* d = std::getenv("STRATA_KVA_CAPTURE"); d != nullptr && *d != '\0') {
        k.capture = true;
        k.capture_dir = d;
        k.capture_fp32 = kva_env_flag("STRATA_KVA_CAPTURE_FP32", false);
    }
    if (const char* d = std::getenv("STRATA_KVA_ORACLE"); d != nullptr && *d != '\0') {
        k.oracle_dir = d;
        k.mode = KvaConfig::Mode::Oracle;
        k.oracle_fp32 = kva_env_flag("STRATA_KVA_ORACLE_FP32", false);
    }
    if (const char* d = std::getenv("STRATA_KVA_PROJECTOR"); d != nullptr && *d != '\0') {
        k.projector = d;
        k.mode = KvaConfig::Mode::Linear;
    }
    if (const char* m = std::getenv("STRATA_KVA_MODE"); m != nullptr && *m != '\0') {
        if (std::string(m) == "oracle") k.mode = KvaConfig::Mode::Oracle;
        else if (std::string(m) == "linear") k.mode = KvaConfig::Mode::Linear;
        else k.mode = KvaConfig::Mode::Identity;
    }
    if (const char* sk = std::getenv("STRATA_KVA_SKIP"); sk != nullptr && std::string(sk) == "moe") k.skip = KvaConfig::Skip::Moe;
    if (const char* o = std::getenv("STRATA_KVA_ONLY"); o != nullptr && *o != '\0') {
        if (std::string(o) == "gdn") k.only = KvaConfig::Only::Gdn;
        else if (std::string(o) == "qsa") k.only = KvaConfig::Only::Qsa;
        else k.only = KvaConfig::Only::All;
    }
    return k;
}

/// The process's KVA settings, parsed once (the engine's own options are parsed once too).
inline const KvaConfig& kva_config() {
    static const KvaConfig c = kva_config_from_env();
    return c;
}

}  // namespace strata::prefill