Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
3.0 KB
Raw Blame
// include/strata/prefill/kva_floor.hpp - which cells of a session were built from approximated rows (KVA).
//
// A KVA read leaves the draft layer's K/V with cells [0, floor) built from rows the late layers never computed.  The
// draft layer must not attend to them (it would only cost draft acceptance, but that is the decode speed), and a
// session that KVA did not approximate must not lose any draft context at all.  The floor therefore follows the
// SESSION'S TOKENS, not the process: after a KVA read the registry remembers the prefix (its token ids) that was
// approximated; whatever state a later request continues from (the live session, a conversation checkpoint, a parked
// conversation), the floor of that state is how much of its tokens agree with a remembered prefix, capped by the
// prefix's floor.  A state no KVA read produced has no common prefix, so its floor is 0 and its draft layer attends
// to the whole window, as with STRATA_KVA=0.
//
// A mislabel can only cost draft cells (an exact state that shares a prefix with an approximated one reports the
// floor too high); it can never make the draft attend to approximated cells that were registered, and the target
// model verifies every token whatever the draft read.
#pragma once

#include <algorithm>
#include <cstdint>
#include <vector>

namespace strata::prefill {

class KvaFloorRegistry {
public:
    explicit KvaFloorRegistry(size_t cap = 8) : cap_(cap) {}

    /// A KVA read approximated the cells [0, floor) of the prompt `ids` (at least `floor` tokens).
    template <class Token>
    void note(const Token* ids, int64_t floor) {
        if (floor <= 0) return;
        ++clock_;
        for (Entry& e : v_)
            if ((int64_t) e.ids.size() == floor && std::equal(e.ids.begin(), e.ids.end(), ids)) { e.used = clock_; return; }
        Entry e;
        e.ids.assign(ids, ids + floor);
        e.used = clock_;
        if (v_.size() >= cap_) {
            size_t lru = 0;
            for (size_t i = 1; i < v_.size(); ++i) if (v_[i].used < v_[lru].used) lru = i;
            v_.erase(v_.begin() + (std::ptrdiff_t) lru);
        }
        v_.push_back(std::move(e));
    }

    /// The first cell of a state holding the tokens ids[0, n) that is NOT an approximated one: the longest common
    /// prefix with a remembered read, over the remembered reads.  0 when the tokens match none.
    template <class Token>
    int64_t floor_for(const Token* ids, int64_t n) const {
        int64_t best = 0;
        for (const Entry& e : v_) {
            const int64_t m = std::min<int64_t>(n, (int64_t) e.ids.size());
            int64_t i = 0;
            while (i < m && (int64_t) ids[i] == e.ids[(size_t) i]) ++i;
            best = std::max(best, i);
        }
        return best;
    }

    size_t size() const { return v_.size(); }
    void clear() { v_.clear(); }

private:
    struct Entry {
        std::vector<int64_t> ids;
        uint64_t used = 0;
    };
    std::vector<Entry> v_;
    size_t cap_;
    uint64_t clock_ = 0;
};

}  // namespace strata::prefill