Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
28.8 KB
Raw Blame
// Strata. src/dsv41/ds4_store.hpp - the RAM tier of the DeepSeek-V4.1 routed experts as a cache of its own: whole experts read
// from the model's files by unbuffered requests (the OS page cache neither used nor disturbed), kept in a fixed arena of slots
// that a replacement policy owns, filled on demand (the CPU lane's picks) and ahead of time (the lookahead's guesses).
//
// A slot holds an expert in the engine's blob layout (gate | up | down at the layer's offsets tr[r], the same as a VRAM slot), so
// the CPU lane's rows read it in place and - when the arena is registered with the GPU - the device kernels read it over PCIe
// (zero-copy, 24 GB/s on a 5070) through the same plan pointers as a VRAM expert.  An expert is three matrices at three file
// offsets (the GGUF's one tensor a role and layer): each is read aligned (4096) into the slot and moved into place (the read's
// lead of up to 4095 bytes), the blob layout leaving 8192 bytes between matrices (and around the blob) for the aligned spans.
//
// Memory: slots are allocated in chunks as they are first needed (private memory), never past the budget nor past the RAM this
// process can still get less `floor_bytes` (the owner's session and the system keep theirs).
//
// Policy (the slot to give up, among the unpinned, not GPU-visible ones): lru (oldest touch), lfu (lowest decayed hit count).
// A slot whose expert a window is computing is pinned from ensure() to release(); a slot the GPU may read is `visible` until
// sync() takes it back at a window boundary (the GPU idle), which also publishes the newly ready ones.
#pragma once

#include "ds4_os.hpp"
#include "ds4_pool.hpp"

#include <algorithm>
#include <atomic>
#include <chrono>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <functional>
#include <memory>
#include <mutex>
#include <string>
#include <vector>

namespace strata::ds4 {

class ExpertStore {
public:
    struct LayerGeo {
        int n_exp = 0;
        int file[3] = {-1, -1, -1};
        uint64_t off[3] = {0, 0, 0}, stride[3] = {0, 0, 0};   // matrix r of expert e: file[r], off[r] + e stride[r]
        uint64_t bytes[3] = {0, 0, 0};
        uint64_t tr[3] = {0, 0, 0};                            // matrix r's place in the blob
        uint64_t blob = 0;                                     // the blob's bytes (gaps of 8192 between matrices)
    };
    enum Policy { kLru = 0, kLfu = 1 };
    struct Stats {
        std::atomic<int64_t> hits{0}, misses{0}, prefetched{0}, prefetch_used{0}, prefetch_wasted{0}, evictions{0}, fails{0};
        std::atomic<int64_t> bytes_read{0}, wait_ns{0}, no_slot{0}, cancelled{0}, put{0};
    };
    struct Vis {                 // a slot the GPU may read from now on (dev != 0), or no longer (dev == 0)
        int layer = 0, e = 0;
        unsigned long long dev = 0;
    };

    struct Cfg {
        int threads = 8;
        size_t budget_bytes = 0;           // the arena at most (grown in chunks as slots are first needed)
        size_t floor_bytes = 8ull << 30;   // RAM left alone
        Policy policy = kLru;
        float decay = 0.97f;               // lfu: heat x decay every `decay_every` accesses of its layer
        int decay_every = 240;
        std::function<uint64_t()> ram_avail;   // the RAM this process can still get (~0: unknown)
        // a new chunk of the arena for the GPU to read in place: returns the device address of `host` (null: not visible)
        std::function<void*(void* host, size_t bytes)> map_chunk;
        std::function<void(void* host)> unmap_chunk;
        size_t reserve = 192;              // slots kept evictable (not visible) for the window's demand and guesses
        int vis_percent = 100;             // the share of experts the GPU may take (by a fixed hash): the rest stay the CPU lane's, so both lanes work
    };

    bool init(const Cfg& c, const std::vector<LayerGeo>& layers, const std::vector<os::fd_t>& dfds, const std::vector<os::fd_t>& pfds) {
        cfg_ = c;
        geo_ = layers;
        dfds_ = dfds;
        pfds_ = pfds;
        size_t blob = 0;
        for (const LayerGeo& g : geo_) blob = std::max<size_t>(blob, g.blob);
        slot_bytes_ = (blob + 2 * kSlack + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1);
        if (blob == 0 || c.budget_bytes < slot_bytes_) return false;
        nmax_ = c.budget_bytes / slot_bytes_;
        // one reserved range for the whole arena (chunks committed as they are needed), 64 KB aligned chunks
        arena_bytes_ = ((nmax_ + kChunk - 1) / kChunk) * kChunk * slot_bytes_;
        arena_ = (uint8_t*) os::range_reserve(arena_bytes_);
        if (arena_ == nullptr) return false;
        slots_.reset(new Slot[nmax_]);
        where_.resize(geo_.size());
        heat_.resize(geo_.size());
        hits_in_layer_.assign(geo_.size(), 0);
        for (size_t l = 0; l < geo_.size(); ++l) {
            where_[l].assign((size_t) geo_[l].n_exp, -1);
            heat_[l].assign((size_t) geo_[l].n_exp, 0.0f);
        }
        io_.reset(new IoQueue(std::max(1, c.threads)));
        grow();   // (the first chunk now: whether the GPU can map it is known from the start)
        // DS4_STORE_PREGROW=1 (default on Linux with the GPU mapping, =0 off): the whole arena now.  cuMemHostRegister (the GPU mapping of a chunk) taken while a kernel of
        // the decode is running waited on it for good on Linux (H100, driver 580): registering all chunks before the first
        // kernel avoids it
#if defined(__linux__)
        const bool pregrow = cfg_.map_chunk && !(std::getenv("DS4_STORE_PREGROW") != nullptr && std::atoi(std::getenv("DS4_STORE_PREGROW")) == 0);   // (Linux: on with the GPU mapping)
#else
        const bool pregrow = std::getenv("DS4_STORE_PREGROW") != nullptr && std::atoi(std::getenv("DS4_STORE_PREGROW")) != 0;
#endif
        if (pregrow)
            while (grow()) {
            }
        return true;
    }
    ~ExpertStore() {
        io_.reset();
        for (size_t i = 0; i < chunks_.size(); ++i)
            if (cfg_.unmap_chunk && chunk_dev_[i] != nullptr) cfg_.unmap_chunk(chunks_[i].first);
        os::range_release(arena_, arena_bytes_);
    }

    size_t slot_bytes() const { return slot_bytes_; }
    size_t slots_allocated() const { return nalloc_; }
    size_t slots_max() const { return nmax_; }
    const Stats& stats() const { return st_; }
    double busy_ms() const { return io_ ? io_->busy_ms() : 0.0; }
    static constexpr size_t kSlack = 8192;

    /// The slot holding (layer, e) ready to read, or -1.  (Any thread; a slot found stays valid while it is pinned or visible.)
    int find(int layer, int e) const {
        if (layer < 0 || layer >= (int) where_.size()) return -1;   // (a layer the store does not cover: the DSpark drafter's)
        const int s = where_[(size_t) layer][(size_t) e];
        if (s < 0) return -1;
        Slot& sp = slots_[(size_t) s];
        try_ready(sp);
        return sp.state.load(std::memory_order_acquire) == kReady && sp.layer == layer && sp.expert == e ? s : -1;
    }
    const uint8_t* mat(int slot, int r) const { return slots_[(size_t) slot].base + geo_[(size_t) slots_[(size_t) slot].layer].tr[r]; }
    const uint8_t* blob(int slot) const { return slots_[(size_t) slot].base; }
    /// The device address of the slot's blob (0: the arena is not mapped for the GPU)
    unsigned long long dev_blob(int slot) const {
        const size_t c = (size_t) slot / kChunk;
        if (c >= chunk_dev_.size() || chunk_dev_[c] == nullptr) return 0;
        return (unsigned long long) ((uint8_t*) chunk_dev_[c] + (slots_[(size_t) slot].base - (uint8_t*) chunks_[c].first));
    }

    /// Makes the experts es[0..n) of `layer` ready and pinned (reading what is not here), waiting for the reads; the
    /// caller releases them.  False if a read failed or no slot could be had for one (ok[i] says which are fine; the caller
    /// reads the others another way).
    bool ensure(int layer, const int* es, int n, std::vector<char>& ok) {
        std::vector<int> sl;
        begin(layer, es, n, ok, sl);
        return wait_slots(sl, ok, 0, n);
    }
    /// The first half of ensure(): the experts' slots found or their reads started (pinned), no waiting.  sl[i] is the slot or -1.
    void begin(int layer, const int* es, int n, std::vector<char>& ok, std::vector<int>& sl) {
        ok.assign((size_t) n, 0);
        sl.assign((size_t) n, -1);
        if (layer < 0 || layer >= (int) geo_.size()) return;
        std::lock_guard<std::mutex> lk(mu_);
        for (int i = 0; i < n; ++i) {
            const int e = es[i];
            touch_heat(layer, e);
            int s = where_[(size_t) layer][(size_t) e];
            if (s >= 0) {
                Slot& sp = slots_[(size_t) s];
                try_ready(sp);
                if (sp.state.load() == kReady && !sp.prefetch) st_.hits.fetch_add(1, std::memory_order_relaxed);
                if (sp.prefetch) st_.prefetch_used.fetch_add(1, std::memory_order_relaxed);
                sp.prefetch = false;
                sp.pin++;
                sp.tick = ++clock_;
                sl[(size_t) i] = s;
                continue;
            }
            st_.misses.fetch_add(1, std::memory_order_relaxed);
            s = start_fill(layer, e, true);
            sl[(size_t) i] = s;
        }
    }
    /// True when the reads of slot sl (from begin) are all done.
    bool loaded(int s) const { return s >= 0 && slots_[(size_t) s].left.load(std::memory_order_acquire) <= 0; }
    /// The second half: waits for the slots sl[a..b) and marks ok[i] for the good ones.  False if any failed or had no slot.
    bool wait_slots(std::vector<int>& sl, std::vector<char>& ok, int a, int b) {
        const auto t0 = std::chrono::steady_clock::now();
        bool all = true;
        for (int i = a; i < b; ++i) {
            const int s = sl[(size_t) i];
            if (s < 0) {
                all = false;
                continue;
            }
            Slot& sp = slots_[(size_t) s];
            while (sp.left.load(std::memory_order_acquire) > 0) io_->wait_any(2000);
            std::lock_guard<std::mutex> lk(mu_);
            finish(sp);
            if (sp.state.load() == kReady) ok[(size_t) i] = 1;
            else {
                sp.pin--;
                sl[(size_t) i] = -1;
                all = false;
            }
        }
        st_.wait_ns.fetch_add(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count(),
                              std::memory_order_relaxed);
        return all;
    }
    void release(int layer, const int* es, int n, const std::vector<char>& ok) {
        if (layer < 0 || layer >= (int) geo_.size()) return;
        std::lock_guard<std::mutex> lk(mu_);
        for (int i = 0; i < n; ++i)
            if (ok[(size_t) i]) {
                const int s = where_[(size_t) layer][(size_t) es[i]];
                if (s >= 0 && slots_[(size_t) s].pin > 0) slots_[(size_t) s].pin--;
            }
    }
    /// Starts reading (layer, e) if it is neither here nor on its way (a guess: unpinned, may be given up before it is used).
    void prefetch(int layer, int e) {
        if (layer < 0 || layer >= (int) geo_.size() || e < 0 || e >= geo_[(size_t) layer].n_exp) return;
        if (where_[(size_t) layer][(size_t) e] >= 0) return;
        std::lock_guard<std::mutex> lk(mu_);
        if (where_[(size_t) layer][(size_t) e] >= 0) return;
        if (start_fill(layer, e, false) >= 0) st_.prefetched.fetch_add(1, std::memory_order_relaxed);
    }
    /// Reads the experts of `list` (layer, expert; hottest first) into free slots and returns when they are in (a start that
    /// is already warm: the profile's hottest, so the first tokens find them).  The first `list.size()` fit the arena less
    /// `leave` slots; returns how many were read.
    size_t preload(const std::vector<std::pair<int, int>>& list, size_t leave) {
        size_t n = 0;
        {
            std::lock_guard<std::mutex> lk(mu_);
            const size_t room = nmax_ > leave ? nmax_ - leave : 0;
            const size_t take = std::min(list.size(), room);
            for (size_t k = take; k-- > 0;) {   // (coldest first: the hottest get the newest ticks)
                const int layer = list[k].first, e = list[k].second;
                if (layer < 0 || layer >= (int) geo_.size() || e < 0 || e >= geo_[(size_t) layer].n_exp) continue;
                if (where_[(size_t) layer][(size_t) e] >= 0) continue;
                const int s = start_fill(layer, e, false);
                if (s < 0) break;
                slots_[(size_t) s].prefetch = false;   // (warm, not a guess)
                ++n;
            }
        }
        for (size_t i = 0;; ++i) {   // until every read is done
            bool busy = false;
            {
                std::lock_guard<std::mutex> lk(mu_);
                for (size_t j = 0; j < nalloc_; ++j) {
                    Slot& sp = slots_[j];
                    if (sp.state.load(std::memory_order_acquire) != kLoading) continue;
                    if (sp.left.load(std::memory_order_acquire) > 0) busy = true;
                    else {
                        try_ready(sp);
                        if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
                    }
                }
            }
            if (!busy) break;
            io_->wait_any(5000);
        }
        return n;
    }
    /// A slot for (layer, e) that the caller fills with the expert's blob itself (an expert the VRAM cache gives back to the CPU lane: its bytes
    /// are in memory already, so no read from the SSD while the lane waits).  Returns the blob's address, or nullptr (it is here already, or
    /// no slot can be had).  The slot is pinned and not findable until put_done().
    uint8_t* put_begin(int layer, int e) {
        if (layer < 0 || layer >= (int) geo_.size() || e < 0 || e >= geo_[(size_t) layer].n_exp) return nullptr;
        std::lock_guard<std::mutex> lk(mu_);
        if (where_[(size_t) layer][(size_t) e] >= 0) return nullptr;
        const int s = take_slot(false);
        if (s < 0) return nullptr;
        Slot& sp = slots_[(size_t) s];
        if (sp.layer >= 0 && sp.expert >= 0) {
            where_[(size_t) sp.layer][(size_t) sp.expert] = -1;
            if (sp.prefetch) st_.prefetch_wasted.fetch_add(1, std::memory_order_relaxed);
            st_.evictions.fetch_add(1, std::memory_order_relaxed);
        }
        sp.layer = layer;
        sp.expert = e;
        sp.pin = 1;
        sp.prefetch = false;
        sp.visible = false;
        sp.tick = ++clock_;
        sp.bad.store(false);
        sp.cancel.store(false);
        sp.left.store(1, std::memory_order_release);
        sp.state.store(kLoading, std::memory_order_release);
        where_[(size_t) layer][(size_t) e] = s;
        st_.put.fetch_add(1, std::memory_order_relaxed);
        return sp.base;
    }
    void put_done(int layer, int e, bool ok) {
        std::lock_guard<std::mutex> lk(mu_);
        const int s = where_[(size_t) layer][(size_t) e];
        if (s < 0) return;
        Slot& sp = slots_[(size_t) s];
        if (sp.pin > 0) sp.pin--;
        if (ok) {
            sp.left.store(0, std::memory_order_release);
            sp.state.store(kReady, std::memory_order_release);
        } else {
            where_[(size_t) layer][(size_t) e] = -1;
            sp.layer = sp.expert = -1;
            sp.left.store(0, std::memory_order_release);
            sp.state.store(kFree, std::memory_order_release);
            free_hint_ = std::min<size_t>(free_hint_, (size_t) s);
        }
    }
    /// Guesses still waiting in the queue for a layer the token is past: dropped unread (their slots free once the queue has gone by)
    void cancel_stale(int layer_now) {
        std::lock_guard<std::mutex> lk(mu_);
        for (size_t i = 0; i < nalloc_; ++i) {
            Slot& sp = slots_[i];
            if (sp.prefetch && sp.pin == 0 && sp.layer >= 0 && sp.layer <= layer_now && sp.state.load(std::memory_order_acquire) == kLoading)
                sp.cancel.store(true, std::memory_order_release);
        }
    }
    /// An access the host saw for an expert the GPU computed (keeps it warm in the policy's eyes).
    void touch(int layer, int e) {
        if (layer < 0 || layer >= (int) where_.size()) return;
        std::lock_guard<std::mutex> lk(mu_);
        touch_heat(layer, e);
        const int s = where_[(size_t) layer][(size_t) e];
        if (s >= 0) slots_[(size_t) s].tick = ++clock_;
    }
    bool gpu_mapped() const { return !chunk_dev_.empty() && chunk_dev_[0] != nullptr; }
    /// The arena's address range (the device sees its host pointers as they are when the mapping is the identity)
    unsigned long long arena_lo() const { return (unsigned long long) (uintptr_t) arena_; }
    unsigned long long arena_hi() const { return (unsigned long long) (uintptr_t) (arena_ + arena_bytes_); }
    bool identity_mapped() const {
        for (size_t i = 0; i < chunks_.size(); ++i)
            if (chunk_dev_[i] != nullptr && chunk_dev_[i] != chunks_[i].first) return false;
        return true;
    }
    size_t max_blob() const { return slot_bytes_ - 2 * kSlack; }

    /// At a window boundary (nothing reading the slots; the caller applies `ev` to the device's table before the next window):
    /// every ready slot not visible yet becomes visible (`gpu_ok(layer, e)`: false for an expert VRAM holds), and visible
    /// slots of the lowest score are taken back until `cfg.reserve` slots can be given up.  Returns the number of events.
    size_t sync(std::vector<Vis>& ev, const std::function<bool(int, int)>& gpu_ok) {
        std::lock_guard<std::mutex> lk(mu_);
        size_t nvis = 0, nfree = 0;
        for (size_t i = 0; i < nalloc_; ++i) {
            Slot& sp = slots_[i];
            if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {
                try_ready(sp);
                if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
            }
            const int st = sp.state.load(std::memory_order_acquire);
            if (st == kFree) {
                ++nfree;
                continue;
            }
            if (st != kReady) continue;
            if (sp.visible) {
                ++nvis;
                continue;
            }
            if (sp.pin == 0) ++nfree;
            if (sp.pin == 0 && dev_blob((int) i) != 0 && gpu_ok(sp.layer, sp.expert) && !sp.prefetch && vis_hash(sp.layer, sp.expert) < cfg_.vis_percent) {
                sp.visible = true;
                ++nvis;
                if (sp.pin == 0) --nfree;
                ev.push_back({sp.layer, sp.expert, dev_blob((int) i)});
            }
        }
        // give slots back until the reserve is there: the visible ones with the lowest score
        if (nfree < cfg_.reserve && nvis > 0) {
            std::vector<std::pair<double, size_t>> cand;
            cand.reserve(nvis);
            for (size_t i = 0; i < nalloc_; ++i)
                if (slots_[i].visible && slots_[i].state.load(std::memory_order_acquire) == kReady) cand.push_back({score(slots_[i]), i});
            const size_t want = std::min(cand.size(), cfg_.reserve - nfree);
            std::partial_sort(cand.begin(), cand.begin() + (long) want, cand.end());
            for (size_t k = 0; k < want; ++k) {
                Slot& sp = slots_[cand[k].second];
                sp.visible = false;
                ev.push_back({sp.layer, sp.expert, 0ull});
            }
        }
        return ev.size();
    }
    /// RAM is short: gives back whole chunks from the end of the arena (their experts forgotten) - up to `want`, stopping at one
    /// with a slot in use (pinned, loading, or GPU-visible).  Returns the chunks given back.
    size_t shrink_chunks(size_t want) {
        std::lock_guard<std::mutex> lk(mu_);
        size_t done = 0;
        while (done < want && !chunks_.empty()) {
            const size_t c = chunks_.size() - 1, first = c * kChunk;
            bool busy = false;
            for (size_t i = first; i < nalloc_ && !busy; ++i) {
                Slot& sp = slots_[i];
                if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {   // (read by now)
                    try_ready(sp);
                    if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
                }
                busy = sp.pin > 0 || sp.visible || sp.state.load(std::memory_order_acquire) == kLoading;
            }
            if (busy) break;
            for (size_t i = first; i < nalloc_; ++i) {
                Slot& sp = slots_[i];
                if (sp.layer >= 0 && sp.expert >= 0) {
                    where_[(size_t) sp.layer][(size_t) sp.expert] = -1;
                    st_.evictions.fetch_add(1, std::memory_order_relaxed);
                }
                sp.layer = sp.expert = -1;
                sp.prefetch = false;
                sp.state.store(kFree, std::memory_order_release);
                sp.base = nullptr;
            }
            if (cfg_.unmap_chunk && chunk_dev_[c] != nullptr) cfg_.unmap_chunk(chunks_[c].first);
            os::range_decommit(chunks_[c].first, chunks_[c].second);
            chunks_.pop_back();
            chunk_dev_.pop_back();
            nalloc_ = first;
            free_hint_ = std::min(free_hint_, first);
            ++done;
            ++shrunk_;
        }
        return done;
    }
    size_t shrunk() const { return shrunk_; }
    /// An expert VRAM took over: no longer a visible slot
    void unmark(int layer, int e) {
        if (layer < 0 || layer >= (int) where_.size()) return;
        std::lock_guard<std::mutex> lk(mu_);
        const int s = where_[(size_t) layer][(size_t) e];
        if (s >= 0) slots_[(size_t) s].visible = false;
    }
    size_t visible_count() const {
        size_t n = 0;
        for (size_t i = 0; i < nalloc_; ++i) n += slots_[i].visible ? 1 : 0;
        return n;
    }

private:
    static constexpr int kFree = 0, kLoading = 1, kReady = 2, kFailed = 3;
    // slots a chunk of the arena (the unit the GPU mapping registers; DS4_STORE_CHUNK_SLOTS: a test knob for the registration segment size)
    static size_t chunk_slots() {
        const char* e = std::getenv("DS4_STORE_CHUNK_SLOTS");
        const long v = e ? std::atol(e) : 0;
        return v >= 8 && v <= 4096 ? (size_t) v : 64;
    }
    const size_t kChunk = chunk_slots();
    struct Slot {
        uint8_t* base = nullptr;   // the blob's first byte
        int layer = -1, expert = -1, pin = 0;
        bool prefetch = false, visible = false;
        std::atomic<int> state{kFree}, left{0};
        std::atomic<bool> bad{false}, cancel{false};
        uint64_t tick = 0;
    };

    static int vis_hash(int layer, int e) { return (int) (((unsigned) layer * 2654435761u + (unsigned) e * 40503u) >> 7) % 100; }
    double score(const Slot& sp) const {
        if (cfg_.policy == kLru) return (double) sp.tick;
        return (double) heat_[(size_t) sp.layer][(size_t) sp.expert] + 1e-9 * (double) sp.tick;
    }
    void touch_heat(int layer, int e) {
        heat_[(size_t) layer][(size_t) e] += 1.0f;
        if (cfg_.policy != kLru && ++hits_in_layer_[(size_t) layer] % (uint64_t) cfg_.decay_every == 0)
            for (float& h : heat_[(size_t) layer]) h *= cfg_.decay;   // (a linear pass over 384 floats every 240 accesses of that layer)
    }
    // a slot for (layer, e) and its reads started; -1 if there is none to be had.  mu_ held.
    int start_fill(int layer, int e, bool demand) {
        int s = take_slot(demand);
        if (s < 0) {
            st_.no_slot.fetch_add(1, std::memory_order_relaxed);
            return -1;
        }
        Slot& sp = slots_[(size_t) s];
        if (sp.layer >= 0 && sp.expert >= 0) {
            where_[(size_t) sp.layer][(size_t) sp.expert] = -1;
            if (sp.prefetch) st_.prefetch_wasted.fetch_add(1, std::memory_order_relaxed);
            st_.evictions.fetch_add(1, std::memory_order_relaxed);
        }
        sp.layer = layer;
        sp.expert = e;
        sp.pin = demand ? 1 : 0;
        sp.prefetch = !demand;
        sp.visible = false;
        sp.tick = ++clock_;
        sp.state.store(kLoading, std::memory_order_release);
        sp.bad.store(false);
        sp.cancel.store(false);
        sp.left.store(3, std::memory_order_release);
        where_[(size_t) layer][(size_t) e] = s;
        const LayerGeo& g = geo_[(size_t) layer];
        IoQueue::Job jobs[3];
        for (int r = 0; r < 3; ++r) {
            const uint64_t off = g.off[r] + (uint64_t) e * g.stride[r];
            const uint64_t lead = off % os::kDirectAlign;
            uint8_t* fin = sp.base + g.tr[r];   // where the matrix belongs
            // the aligned read lands the file's byte `off` at dst + lead: dst is fin - lead rounded down to 4096 (its span spills
            // under 4096 bytes either way into the blob's gaps), the matrix is then moved up to fin
            uint8_t* dst = (uint8_t*) ((uintptr_t) (fin - lead) & ~(uintptr_t) (os::kDirectAlign - 1));
            IoQueue::Job& j = jobs[r];
            j.fd = pfds_[(size_t) g.file[r]];
            j.dfd = dfds_[(size_t) g.file[r]];
            j.off = off;
            j.len = g.bytes[r];
            j.dst = dst;
            j.keep = fin;
            j.left = &sp.left;
            j.bad = &sp.bad;
            j.cancel = demand ? nullptr : &sp.cancel;
        }
        st_.bytes_read.fetch_add((int64_t) (g.bytes[0] + g.bytes[1] + g.bytes[2]), std::memory_order_relaxed);
        io_->push(jobs, 3, !demand);   // (a guess waits behind every demand read)
        return s;
    }
    // loading -> ready once its reads are done and whole (lock-free; a failed one is cleaned up by finish / take_slot)
    void try_ready(Slot& sp) const {
        int want = kLoading;
        if (sp.left.load(std::memory_order_acquire) == 0 && !sp.bad.load(std::memory_order_acquire))
            sp.state.compare_exchange_strong(want, kReady, std::memory_order_acq_rel);
    }
    void finish(Slot& sp) {   // mu_ held; reads done
        if (sp.state.load() != kLoading) return;
        if (sp.bad.load()) {
            where_[(size_t) sp.layer][(size_t) sp.expert] = -1;
            sp.layer = sp.expert = -1;
            sp.state.store(kFree, std::memory_order_release);
            sp.prefetch = false;
            if (sp.cancel.load()) st_.cancelled.fetch_add(1, std::memory_order_relaxed);
            else st_.fails.fetch_add(1, std::memory_order_relaxed);
            return;
        }
        sp.state.store(kReady, std::memory_order_release);
    }
    bool grow() {
        if (nalloc_ >= nmax_) return false;
        const size_t n = std::min(kChunk, nmax_ - nalloc_);
        const size_t bytes = n * slot_bytes_;
        const uint64_t avail = cfg_.ram_avail ? cfg_.ram_avail() : ~uint64_t{0};
        if (avail != ~uint64_t{0} && avail < bytes + cfg_.floor_bytes) return false;
        void* p = arena_ + nalloc_ * slot_bytes_;
        if (!os::range_commit(p, bytes)) return false;
        chunks_.push_back({p, bytes});
        void* dev = cfg_.map_chunk ? cfg_.map_chunk(p, bytes) : nullptr;
        chunk_dev_.push_back(dev);
        for (size_t i = 0; i < n; ++i) slots_[nalloc_ + i].base = (uint8_t*) p + i * slot_bytes_ + kSlack;
        nalloc_ += n;
        return true;
    }
    // the slot to give up (or a free one): the policy's lowest among the unpinned, not visible, ready ones.  mu_ held.
    int take_slot(bool demand) {
        for (size_t i = free_hint_; i < nalloc_; ++i)
            if (slots_[i].state.load(std::memory_order_acquire) == kFree) {
                free_hint_ = i + 1;
                return (int) i;
            }
        if (grow()) return take_slot(demand);
        int best = -1;
        double bv = 0;
        for (size_t i = 0; i < nalloc_; ++i) {
            Slot& sp = slots_[i];
            if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {
                try_ready(sp);
                if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp);
                if (sp.state.load(std::memory_order_acquire) == kFree) {
                    free_hint_ = std::min(free_hint_, i);
                    return (int) i;
                }
            }
            if (sp.pin > 0 || sp.visible || sp.state.load(std::memory_order_acquire) != kReady) continue;
            const double v = score(sp);
            if (best < 0 || v < bv) {
                best = (int) i;
                bv = v;
            }
        }
        if (best < 0) return -1;
        free_hint_ = std::min<size_t>(free_hint_, (size_t) best);
        return best;
    }

    Cfg cfg_;
    std::vector<LayerGeo> geo_;
    std::vector<os::fd_t> dfds_, pfds_;
    size_t slot_bytes_ = 0, nmax_ = 0, nalloc_ = 0, free_hint_ = 0, arena_bytes_ = 0;
    uint8_t* arena_ = nullptr;
    size_t shrunk_ = 0;
    std::unique_ptr<Slot[]> slots_;
    std::vector<std::pair<void*, size_t>> chunks_;
    std::vector<void*> chunk_dev_;
    std::vector<std::vector<int>> where_;
    std::vector<std::vector<float>> heat_;
    std::vector<uint64_t> hits_in_layer_;
    uint64_t clock_ = 0;
    std::mutex mu_;
    std::unique_ptr<IoQueue> io_;
    Stats st_;
};

}  // namespace strata::ds4