Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
43.8 KB
Raw Blame
// serve/prefetch.cc — advisory expert-prefetch router + self-managed L1 pool + accounting.
// Depends only on llama public/near-public APIs (declared inline, linked from llama.lib)
// and parses GGUF/safetensors headers by hand; llama internal headers are not touched.
// The real routing gate is never modified: predictions only warm the page cache. A wrong
// prediction costs latency, never quality.
// Three-way overlap accounting (printed as [pref-trace]):
//   head   = pred@t-1 ∩ teacher@t        teacher = host-side recompute of gate·m_in (gate is F32 in our GGUF)
//   sticky = teacher@t-1 ∩ teacher@t     (the free win the OS page-cache LRU already provides)
//   random = uniform sample of K ∩ teacher@t   (popularity floor)
#include "prefetch.h"

#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <psapi.h>

#include <atomic>
#include <chrono>
#include <cmath>
#include <condition_variable>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <map>
#include <mutex>
#include <string>
#include <thread>
#include <vector>

// Near-public staging API (src/llama-ext.h). NOTE: that header has **no extern "C"
// wrapper** — its symbols are C++-decorated, so you must include the real header rather
// than hand-write declarations. This unit is globbed into the llama target by the patch
// band (patches/llama.cpp/common #3), which puts src/ on the PRIVATE include path;
// llama.h is reachable via ../include PUBLIC.
#include "llama.h"
#include "llama-ext.h"


namespace {

// Pool final telemetry (defined in the pool section below) — fwd-declared for ~Pref.
static void e0_pool_trace_final();

// ── Shapes (pinned for the 35B GGUF: E=256, H=2048, feat=2560, K=4, owners 6..38 → consumers 7..39) ──
constexpr int   EK = 256, HH = 2048, FE = 2560, KDIM = 4, NH = 33;
constexpr int   OWNER0 = 6;                       // first owner
std::vector<int> consumers;                        // 7..39

// ── GGUF/safetensors header parsing ──
std::string rd_str(FILE * f) {
    uint64_t n = 0;
    if (fread(&n, 8, 1, f) != 1) return {};
    std::string s((size_t) n, '\0');
    if (n && fread(s.data(), 1, (size_t) n, f) != n) return {};
    return s;
}
struct GGUFType { uint32_t t; };
size_t align_up(size_t x, size_t a) { return (x + a - 1) / a * a; }
// GGUF KV value types: 0u8 1i8 2u16 3i16 4u32 5i32 6f32 7bool 8str 9arr 10u64 11i64 12f64
size_t scalar_size(uint32_t t) { static const size_t s[] = {1,1,2,2,4,4,4,1,0,0,8,8,8}; return t <= 12 ? s[t] : 0; }
void skip_val(FILE * f, uint32_t t) {
    if (t == 8) { rd_str(f); return; }
    if (t == 9) {
        uint32_t et; uint64_t n;
        if (fread(&et, 4, 1, f) != 1 || fread(&n, 8, 1, f) != 1) return;
        for (uint64_t i = 0; i < n; ++i) skip_val(f, et);
        return;
    }
    size_t s = scalar_size(t);
    if (s) fseek(f, (long) s, SEEK_CUR);
}
// Tensor dtype uses the GGML_TYPE enum: 0=F32 1=F16 2=BF16 3=Q4_1 (NOT the GGUF KV enum — don't mix).
bool   is_q4_1(uint32_t t) { return t == 3; }
size_t plain_esz(uint32_t t) { return t == 0 ? 4 : (t == 1 || t == 2) ? 2 : 0; }

struct TT {
    std::string name;
    uint32_t type = 0;
    uint64_t dim[4] = {1, 1, 1, 1};
    uint64_t off = 0;
};
size_t tt_bytes(const TT & x) {
    size_t n = 1;
    for (int i = 0; i < 4; ++i) n *= (size_t) x.dim[i];
    if (is_q4_1(x.type)) return (size_t) x.dim[1] * x.dim[2] * x.dim[3] * ((size_t) x.dim[0] / 32) * 20;
    return n * plain_esz(x.type);
}

// IEEE half → float (subnormals included)
float f16_to_f32(uint16_t h) {
    uint32_t sign = (uint32_t) (h & 0x8000) << 16;
    uint32_t e = (h >> 10) & 0x1F, m = h & 0x3FF;
    uint32_t bits;
    if (e == 0) {
        if (m == 0) bits = sign;
        else {                                        // subnormal: renormalize
            e = 127 - 15 + 1;
            while (!(m & 0x400)) { m <<= 1; e--; }
            m &= 0x3FF;
            bits = sign | (e << 23) | (m << 13);
        }
    } else if (e == 0x1F) {
        bits = sign | 0x7F800000u | (m << 13);
    } else {
        bits = sign | ((e + 127 - 15) << 23) | (m << 13);
    }
    float r;
    memcpy(&r, &bits, 4);
    return r;
}

// safetensors: { "<key>": {"dtype":"F16","shape":[..],"data_offsets":[a,b]} }
struct SEnt { long long a = -1, b = -1; std::vector<long long> shape; };
bool st_load(const char * path, std::map<std::string, SEnt> & out) {
    FILE * f = fopen(path, "rb");
    if (!f) return false;
    uint64_t hn = 0;
    if (fread(&hn, 8, 1, f) != 1) { fclose(f); return false; }
    std::string hdr((size_t) hn, '\0');
    bool ok = fread(hdr.data(), 1, (size_t) hn, f) == hn;
    fclose(f);
    if (!ok) return false;
    size_t i = 0;
    while ((i = hdr.find("\"layers.", i)) != std::string::npos) {
        size_t ke = hdr.find("\":", i + 1);
        size_t vo = hdr.find('{', ke);
        size_t vc = ke == std::string::npos || vo == std::string::npos ? std::string::npos : hdr.find('}', vo);
        if (vc == std::string::npos) break;
        std::string key = hdr.substr(i + 1, ke - i - 1);
        std::string o = hdr.substr(vo, vc - vo + 1);
        SEnt e;
        size_t doff = o.find("\"data_offsets\"");
        size_t lb = doff == std::string::npos ? std::string::npos : o.find('[', doff);
        size_t rb = lb == std::string::npos ? std::string::npos : o.find(']', lb);
        if (rb == std::string::npos) { i = vc; continue; }
        e.a = strtoll(o.c_str() + lb + 1, nullptr, 10);
        const char * p = o.c_str() + lb + 1;
        while (*p && *p != ',') p++;
        if (*p == ',') e.b = strtoll(p + 1, nullptr, 10);
        size_t sp = o.find("\"shape\""), slb = sp == std::string::npos ? std::string::npos : o.find('[', sp),
               srb = slb == std::string::npos ? std::string::npos : o.find(']', slb);
        if (srb != std::string::npos) {
            std::string nums = o.substr(slb + 1, srb - slb - 1);
            const char * q = nums.c_str();
            while (*q) {
                char * r; long long v = strtoll(q, &r, 10);
                if (r != q) e.shape.push_back(v);
                q = r; while (*q == ',' || *q == ' ') q++;
            }
        }
        out[key] = e;
        i = vc;
    }
    return !out.empty();
}

bool grab_f16(FILE * f, uint64_t data_base, const SEnt & e, std::vector<float> & dst) {
    size_t n = 1;
    for (auto d : e.shape) n *= (size_t) d;
    dst.resize(n);
    if (_fseeki64(f, (long long) (data_base + e.a), SEEK_SET) != 0) return false;
    std::vector<uint16_t> raw(n);
    if (fread(raw.data(), 2, n, f) != n) return false;
    for (size_t i = 0; i < n; ++i) dst[i] = f16_to_f32(raw[i]);
    return true;
}

// ── Runtime state ──
struct Head { std::vector<float> fc1, fc2, lin; };   // [512][2560] / [256][512] / [256][2560]
struct Hist { std::vector<float> cur, prev; };        // owner-layer two-step onehot (E)

struct Pref {
    bool active = false;
    std::map<int, Head> heads;                        // owner → head triple
    std::map<int, std::vector<float>> gate;           // consumer → [E][H]
    struct EXP { std::vector<TT> t; };               // consumer → up/gate/down exps metadata
    std::map<int, EXP> exps;
    std::map<int, Hist> hist;                         // layers 6..39 (39 keeps teacher onehot only)
    std::map<int, std::vector<int>> pred_prev;        // consumer → previous-step head top-K
    std::map<int, std::vector<int>> exec_prev;        // consumer → previous-step teacher (sticky)
    uint64_t data_base = 0;
    HANDLE fh = INVALID_HANDLE_VALUE, mh = NULL;
    void * view = nullptr;
    bool do_prefetch = false;
    int  trace_every = 32;
    int  stride = 1;          // full head/teacher recompute period (E0_PREF_STRIDE); skipped steps only re-issue prefetch
    bool async = false;       // E0_PREF_ASYNC=0 falls back to synchronous (tax-splitting A/B)
    // Predicted-range VirtualLock: pinned bytes are not trimmed while under budget;
    // lock failure once the cap is hit = natural ceiling. NOTE: VirtualLock normally
    // requires SeLockMemoryPrivilege, absent on consumer machines — treat this as best-effort.
    bool   do_pin = false;
    size_t pin_cap = 3ull << 30, pin_bytes = 0;
    std::map<int, std::vector<std::pair<size_t, size_t>>> pinned;   // consumer → currently pinned (file_off,len)
    // Self-managed L1 over the mmap view (E0_POOL): re-touch at step boundaries, in-table
    // age-based LRU; "evict" = drop from table (stop re-touching; reference-bit aging lets
    // the OS reclaim to standby — a free soft layer, no privileged APIs involved).
    int  pool = 0;                                    // 0=legacy advisory 1=sticky 2=sticky+observation
    struct Pin { size_t off, len; int age; uint8_t reg; };
    std::map<int, std::vector<Pin>> pins;             // consumer → pin table (age = steps since last hit)
    size_t reg_bytes[2] = {0, 0}, reg_cap[2] = {0, 0};   // 0=hot zone (teacher) 1=observation zone (head)
    long pool_hit = 0, pool_tot = 0, lock_fail = 0, evicts = 0, shrinks = 0;
    // Async: m_in row-snapshot mailbox ((NH+1)×H floats, contiguous); worker runs full steps.
    std::vector<float> snap;
    std::atomic<bool> busy{false}, quit{false};
    std::mutex wmtx;
    std::condition_variable cv;
    std::thread worker;
    long hits = 0, sticky = 0, rnd = 0, total = 0, steps = 0;
    long long pool_mb = 0;                        // private-page pool hot-zone MB (>0 enables; E0_POOL_MB)
    uint32_t lcg = 2463534242u;
    uint32_t next() { lcg ^= lcg << 13; lcg ^= lcg >> 17; lcg ^= lcg << 5; return lcg; }
    ~Pref() {
        quit = true; cv.notify_all();
        if (worker.joinable()) worker.join();
        if (!total) return;
        fprintf(stderr, "\n[pref-trace] head   overlap = %ld/%ld = %.1f%%\n", hits, total, 100.0 * hits / total);
        fprintf(stderr, "[pref-trace] sticky overlap = %ld/%ld = %.1f%%\n", sticky, total, 100.0 * sticky / total);
        fprintf(stderr, "[pref-trace] random overlap = %ld/%ld = %.1f%%\n", rnd, total, 100.0 * rnd / total);
        fprintf(stderr, "[pref-trace] head−sticky = %.1f pp | head−random = %.1f pp\n",
                100.0 * (hits - sticky) / total, 100.0 * (hits - rnd) / total);
        e0_pool_trace_final();
        if (pool)
            fprintf(stderr, "[pref-trace] POOL(%s) hit = %ld/%ld = %.1f%% | pins hot=%lluMB obs=%lluMB (cap %llu/%llu) evict=%ld lockfail=%ld shrink=%ld\n",
                    pool == 2 ? "sticky+obs" : "sticky", pool_hit, pool_tot, pool_tot ? 100.0 * pool_hit / pool_tot : 0.0,
                    (unsigned long long) (reg_bytes[0] >> 20), (unsigned long long) (reg_bytes[1] >> 20),
                    (unsigned long long) (reg_cap[0] >> 20), (unsigned long long) (reg_cap[1] >> 20),
                    evicts, lock_fail, shrinks);
        fflush(stderr);
    }
};
Pref g;

float gelu_erf(float x) { return 0.5f * x * (1.f + erff(x * 0.7071067811865475f)); }
float dotf(const float * __restrict a, const float * __restrict b, size_t n) {
    float s = 0.f;
    for (size_t i = 0; i < n; ++i) s += a[i] * b[i];   // /arch:AVX2 auto-FMA reduction; the bottleneck is weight streaming, not ALU
    return s;
}
void topk(const float * v, int n, int k, std::vector<int> & out) {
    out.clear();
    std::vector<char> used(n, 0);
    for (int j = 0; j < k; ++j) {
        int best = -1;
        for (int i = 0; i < n; ++i) if (!used[i] && (best < 0 || v[i] > v[best])) best = i;
        if (best < 0) break;
        used[best] = 1; out.push_back(best);
    }
}
int ovlp(const std::vector<int> & a, const std::vector<int> & b) {
    int n = 0;
    for (int x : a) for (int y : b) if (x == y) { ++n; break; }
    return n;
}

}  // namespace

// ══ Private-page expert pool (E0_POOL_MB): L1 with fill-once slots ══
// Design: decode reads expert bytes **directly from pool pages** via the expert-base
// resolver (patch surface #7), bypassing the view page-fault path entirely — a hit is a
// plain read with no kernel involvement. Slots fill once and never recycle (no eviction
// ⇒ no read-vs-overwrite race, correctness by construction); once the budget is
// exhausted, new experts pass through to the mmap view = graceful degradation to the
// un-pooled behavior. Three acquisition channels:
//   demand = synchronous whole-block ReadFile on the compute thread (first touch)
//   hot    = worker prefill from the teacher set
//   obs    = dedicated quota for head predictions
// Whole-block reads are also fragmentation-proof (1×1MB sequential vs a 256×4K fault
// storm — an order of magnitude apart on consumer SSDs).
static_assert(sizeof(LONG) == 4, "slot state atoms");
namespace {

struct PoolTT {                       // one 3D experts tensor (40 slots measured across all layers)
    uint8_t * arena = nullptr;        // MEM_RESERVE, MEM_COMMIT on demand
    volatile LONG * st = nullptr;     // slot state: 0=empty 1=filling 2=ready 3=not-poolable
    CRITICAL_SECTION * cvlk = nullptr;// per-slot pair: CS lock + CV (filler owns it, readers sleep until woken)
    CONDITION_VARIABLE * cv = nullptr;
    size_t per = 0;                   // bytes per expert
    uint64_t foff = 0;                // file offset of expert 0
};
static volatile LONG g_p_sc_ok = 0, g_p_sc_bad = 0;      // byte-level self-check (fill vs view memcmp sampling)
static bool g_pool_sc = false;                           // E0_POOL_SELFCHECK=1 enables
static void e0_pool_selfcheck(uint8_t * slot, uint64_t foff, size_t per) {
    if (!g_pool_sc || !g.view) return;
    const uint8_t * v = (const uint8_t *) g.view + foff;
    int bad = 0;
    for (size_t i = 0; i < per; i += 4096) if (memcmp(slot + i, v + i, 64) != 0) { bad = 1; break; }
    if (bad) InterlockedIncrement(&g_p_sc_bad); else InterlockedIncrement(&g_p_sc_ok);
}
// ⚠ The tensor *pointer* changes every forward (same tensor registered multiple times,
// measured) ⇒ key by tensor NAME (table built at init, read-only afterwards → lock-free).
std::map<std::string, PoolTT *> g_ptt_by_name;                  // GGUF name → PTT
std::map<int, std::vector<PoolTT *>> g_ptt_by_c;                // consumer → 3 PTTs (for prefill)
std::mutex g_pmtx;                                              // fill/accounting lock (fills are ms-scale, not a spin hotspot)
size_t g_pool_cap[2] = {0, 0};                                  // 0=hot 1=obs (E0_POOL_MB/E0_POOL_OBS_MB)
size_t g_pool_bytes[2] = {0, 0};
volatile LONG g_p_calls = 0;
bool g_pool_on = false;
volatile LONG g_p_hits = 0, g_p_miss_d = 0, g_p_bypass = 0, g_p_waits = 0, g_p_fills = 0;

// Expert-base resolver hook from the patch band (exported by ggml.dll, C decoration).
typedef const char * (*ggml_edge0_mmid_base_fn)(const struct ggml_tensor * src0, int64_t id, size_t nb02);
extern "C" __declspec(dllimport) void ggml_edge0_set_mmid_resolver(ggml_edge0_mmid_base_fn fn);

// Resolver callback (patch surface #7, runs on ggml compute threads): returns the base
// address of expert `id`'s data.
extern "C" static const char * e0_mmid_base_res(const ggml_tensor * s, int64_t id, size_t nb02) {
    LONG c = InterlockedIncrement(&g_p_calls);
    if (c <= 2) { fprintf(stderr, "[pref-trace] POOL2 call#%ld name=%s nb02=%zu\n", c, s->name ? s->name : "?", nb02); fflush(stderr); }
    const char * orig = (const char *) s->data + (size_t) id * nb02;
    auto it = g_ptt_by_name.find(s->name ? s->name : "");
    if (it == g_ptt_by_name.end() || !it->second->st || it->second->per != nb02) {
        LONG b = InterlockedIncrement(&g_p_bypass);
        if (b <= 6)
            fprintf(stderr, "[pref-trace] POOL2 miss#%ld name=%s nb02=%zu in_table=%d table=%zu\n",
                    b, s->name ? s->name : "?", nb02, (int) (it != g_ptt_by_name.end()), g_ptt_by_name.size());
        return orig;
    }
    PoolTT * pt = it->second;
    for (;;) {
        LONG st = pt->st[id];
        if (st == 1) {                                        // another thread is filling: sleep until woken (no spinning)
            InterlockedIncrement(&g_p_waits);
            CRITICAL_SECTION & lk = pt->cvlk[id];
            EnterCriticalSection(&lk);
            if (pt->st[id] == 1) SleepConditionVariableCS(&pt->cv[id], &lk, 2000);
            LeaveCriticalSection(&lk);
            continue;
        }
        if (st == 2) {
            LONG h = InterlockedIncrement(&g_p_hits);
            if (h <= 3 || (h & 0x3FFF) == 0)
                fprintf(stderr, "[pref-trace] POOL2 hit=%ld fillD=%ld bypass=%ld wait=%ld sc=%ld/%ld | hot=%lluMB obs=%lluMB\n",
                        h, g_p_miss_d, g_p_bypass, g_p_waits, (long) g_p_sc_ok, (long) g_p_sc_bad,
                        (unsigned long long) (g_pool_bytes[0] >> 20), (unsigned long long) (g_pool_bytes[1] >> 20));
            fflush(stderr);
            return (const char *) (pt->arena + (size_t) id * pt->per);
        }
        if (st == 3) { InterlockedIncrement(&g_p_bypass); return orig; }
        {                                                                   // claim the fill: CAS + exclusive CV section
            CRITICAL_SECTION & lk = pt->cvlk[id];
            if (InterlockedCompareExchange(&pt->st[id], 1, 0) != 0) continue;
            EnterCriticalSection(&lk);
        }
        LONG fs = InterlockedIncrement(&g_p_fills);
        if (fs <= 3 || (fs & 0x3FF) == 0) {
            fprintf(stderr, "[pref-trace] POOL2 fill#%ld name=%s id=%lld per=%zu hits=%ld\n",
                    fs, s->name ? s->name : "?", (long long) id, pt->per, g_p_hits);
            fflush(stderr);
        }
        // Demand fill on this thread (hot budget).
        bool ok = false;
        {
            std::lock_guard<std::mutex> lk(g_pmtx);
            if (g_pool_bytes[0] + pt->per <= g_pool_cap[0]) {
                uint8_t * dst = pt->arena + (size_t) id * pt->per;
                if (VirtualAlloc(dst, pt->per, MEM_COMMIT, PAGE_READWRITE)) {
                    uint64_t fo = pt->foff + (uint64_t) id * pt->per;
                    LARGE_INTEGER off; off.QuadPart = (LONGLONG) fo;
                    ok = SetFilePointerEx(g.fh, off, nullptr, FILE_BEGIN)
                         && ReadFile(g.fh, dst, (DWORD) pt->per, nullptr, nullptr);
                    if (ok) { g_pool_bytes[0] += pt->per; e0_pool_selfcheck(dst, fo, pt->per); }
                }
            }
            if (ok) InterlockedIncrement(&g_p_miss_d);
        }
        pt->st[id] = ok ? 2 : 3;
        WakeAllConditionVariable(&pt->cv[id]);
        LeaveCriticalSection(&pt->cvlk[id]);
        return ok ? (const char *) (pt->arena + (size_t) id * pt->per) : orig;
    }
}

// Worker prefill (teacher=hot / head=obs): called outside the resolver path; same per-slot CAS pattern.
static void e0_pool_prefill(int c, const std::vector<int> & ids, int reg) {
    auto it = g_ptt_by_c.find(c);
    if (it == g_ptt_by_c.end()) return;
    for (PoolTT * pt : it->second)
        for (int id : ids) {
            if (pt->st[id] != 0) continue;
            CRITICAL_SECTION & lk = pt->cvlk[id];
            if (InterlockedCompareExchange(&pt->st[id], 1, 0) != 0) continue;
            EnterCriticalSection(&lk);
            bool ok = false;
            {
                std::lock_guard<std::mutex> lk(g_pmtx);
                if (g_pool_bytes[reg] + pt->per <= g_pool_cap[reg]) {
                    uint8_t * dst = pt->arena + (size_t) id * pt->per;
                    if (VirtualAlloc(dst, pt->per, MEM_COMMIT, PAGE_READWRITE)) {
                        LARGE_INTEGER off = { (DWORD) (pt->foff + (uint64_t) id * pt->per),
                                              (LONG) ((pt->foff + (uint64_t) id * pt->per) >> 32) };
                        ok = SetFilePointerEx(g.fh, off, nullptr, FILE_BEGIN)
                             && ReadFile(g.fh, dst, (DWORD) pt->per, nullptr, nullptr);
                        if (ok) { g_pool_bytes[reg] += pt->per; e0_pool_selfcheck(dst, pt->foff + (uint64_t) id * pt->per, pt->per); }
                    }
                }
            }
            pt->st[id] = ok ? 2 : 3;
            WakeAllConditionVariable(&pt->cv[id]);
            LeaveCriticalSection(&lk);
        }
}

static void e0_pool_trace_final() {
    if (!g_pool_on) return;
    fprintf(stderr, "[pref-trace] POOL2 hit=%ld fillD=%ld bypass=%ld wait=%ld sc=%ld/%ld | hot=%lluMB obs=%lluMB\n",
            g_p_hits, g_p_miss_d, g_p_bypass, g_p_waits, (long) g_p_sc_ok, (long) g_p_sc_bad,
            (unsigned long long) (g_pool_bytes[0] >> 20), (unsigned long long) (g_pool_bytes[1] >> 20));
}

static void e0_pool_scan(const std::map<std::string, TT> & tinfo) {   // pool all *_exps tensors (independent of consumer shapes)
    for (auto & kv : tinfo) {
        const std::string & nm = kv.first;
        const TT & tt = kv.second;
        auto suff = [&](const char * sfx) -> size_t {           // exact suffix match (returns length on hit)
            size_t len = strlen(sfx), k = nm.rfind(sfx);
            return (k != std::string::npos && k + len == nm.size()) ? len : 0;
        };
        size_t l1 = suff(".ffn_up_exps.weight"), l2 = suff(".ffn_gate_exps.weight"), l3 = suff(".ffn_down_exps.weight");
        size_t sl = l1 ? l1 : (l2 ? l2 : l3);
        if (!sl || tt.dim[2] < 2 || tt.dim[2] > 4096) continue;
        const size_t per = tt_bytes(tt) / (size_t) tt.dim[2];
        if (!per) continue;
        PoolTT * e = new PoolTT();
        e->per = per;
        e->foff = g.data_base + tt.off;
        e->arena = (uint8_t *) VirtualAlloc(nullptr, per * (size_t) tt.dim[2], MEM_RESERVE, PAGE_READWRITE);
        e->st = (volatile LONG *) VirtualAlloc(nullptr, sizeof(LONG) * (size_t) tt.dim[2], MEM_COMMIT, PAGE_READWRITE);
        e->cvlk = new CRITICAL_SECTION[(size_t) tt.dim[2]];
        e->cv = new CONDITION_VARIABLE[(size_t) tt.dim[2]];
        for (size_t z = 0; z < (size_t) tt.dim[2]; ++z) { InitializeCriticalSection(&e->cvlk[z]); InitializeConditionVariable(&e->cv[z]); }
        if (!e->arena || !e->st) { delete[] e->cvlk; delete[] e->cv; e->cvlk = nullptr; e->cv = nullptr; delete e; continue; }
        g_ptt_by_name[nm] = e;
    }
    g_pool_on = !g_ptt_by_name.empty();
    if (g_pool_on) ggml_edge0_set_mmid_resolver(e0_mmid_base_res);   // attach to patch surface #7
    { int k = 0; for (auto & q : g_ptt_by_name) if (k++ < 3) fprintf(stderr, "[pref-trace] POOL2 key%d=%s per=%zu\n", k, q.first.c_str(), q.second->per); }
    fprintf(stderr, "[pref-trace] POOL2 init: tt=%zu caps hot=%lluMB obs=%lluMB resolver=%s\n",
            g_ptt_by_name.size(), (unsigned long long) (g_pool_cap[0] >> 20),
            (unsigned long long) (g_pool_cap[1] >> 20), g_pool_on ? "on" : "FAIL");
    fflush(stderr);
}

}  // namespace

static void enable_capture(llama_context * ctx) {
    for (int L = OWNER0; L <= OWNER0 + NH; ++L) llama_set_embeddings_layer_inp(ctx, (uint32_t) L, true);
}

// ── Pool re-touch ladder: deterministic accounting (zero prediction = sticky; + observation zone = sticky/obs) ──
// Mechanism: VirtualLock needs SeLockMemoryPrivilege, which consumer machines lack ⇒ pinning
// is NOT the product mechanism. What works unprivileged = demand touch + working-set hard cap
// + the OS clock's natural LRU:
//   "pin"   = worker volatile-reads the range in 4K steps (demands pages into the WS; any
//             trimmed page's fault tax is paid inside the worker, off the critical path);
//   "evict" = drop from the table only (stop re-touching → reference-bit ages → OS trims it
//             into standby = free soft layer);
//   "cap"   = at init, SetProcessWorkingSetSizeEx(MAX = baseline + hot + obs + headroom, HARDWS_MAX_ENABLE).
// Hot zone = re-touch of the previous step's teacher ids (same experts in a row = zero-fault
// hit); observation zone = head-guess quota (a bad guess only ages its own pages — the hot
// zone re-touches every step and never ages = contamination ceiling). Low-memory notification
// ⇒ shrink the tables (hot to 75%, observation to empty).
static volatile char g_touch_sink;                  // prevent DCE (every page is really read)
static void e0_touch(const void * view, size_t off, size_t len) {
    const volatile char * p = (const volatile char *) view + off;
    char s = 0;
    for (size_t i = 0; i < len; i += 4096) s += p[i];
    g_touch_sink = s;
}
static void e0_pool_evict(int R) {                  // evict the oldest entry in region R (age==0 entries are exempt)
    int wc = -1, wi = -1, ba = 0;
    for (auto & pc : g.pins)
        for (int k = 0; k < (int) pc.second.size(); ++k) {
            auto & p = pc.second[k];
            if (p.reg == R && p.age > ba) { ba = p.age; wc = pc.first; wi = k; }
        }
    if (wc < 0) return;
    g.reg_bytes[R] -= g.pins[wc][wi].len; ++g.evicts;
    g.pins[wc].erase(g.pins[wc].begin() + wi);      // dropping from the table = stop re-touching; physical reclaim is the OS clock's job
}
static void e0_issue_pool() {
    if (!g.view) return;
    const int nreg = g.pool == 2 ? 2 : 1;
    std::map<int, std::vector<std::pair<size_t, size_t>>> want[2];   // 0=hot(teacher) 1=observation(head)
    auto ranges = [&](const std::map<int, std::vector<int>> & src, int R) {
        for (auto & pc : src) {
            auto & v = want[R][pc.first];
            for (const TT & tt : g.exps[pc.first].t) {
                const size_t per = tt_bytes(tt) / EK;
                for (int e : pc.second)
                    v.push_back({ (size_t) g.data_base + tt.off + (size_t) e * per, per });
            }
        }
    };
    ranges(g.exec_prev, 0);
    if (nreg > 1) {
        std::map<int, std::vector<int>> preds;
        { std::lock_guard<std::mutex> lk(g.wmtx); preds = g.pred_prev; }
        ranges(preds, 1);
    }
    for (auto & kv : g.pins) for (auto & p : kv.second) ++p.age;     // uniform step-rate aging
    std::vector<std::pair<size_t, size_t>> touch;                     // this step's demand set (≤ table cap)
    for (int R = 0; R < nreg; ++R) {
        for (auto & wv : want[R]) {
            auto & cur = g.pins[wv.first];
            std::vector<char> hit(wv.second.size(), 0);
            for (size_t i = 0; i < wv.second.size(); ++i)
                for (auto & p : cur)
                    if (p.reg == R && p.off == wv.second[i].first) { p.age = 0; hit[i] = 1; if (!R) ++g.pool_hit; break; }
            if (!R) g.pool_tot += (long) wv.second.size();           // denominator = all hot-zone demand this step
            for (size_t i = 0; i < wv.second.size(); ++i) {
                if (hit[i]) continue;
                auto & w = wv.second[i];
                while (g.reg_bytes[R] + w.second > g.reg_cap[R]) {
                    size_t b = g.reg_bytes[R];
                    e0_pool_evict(R);
                    if (g.reg_bytes[R] == b) { ++g.lock_fail; break; }  // nothing evictable ⇒ hard ceiling (refused-pin counter)
                }
                if (g.reg_bytes[R] + w.second > g.reg_cap[R]) { ++g.lock_fail; continue; }
                cur.push_back({ w.first, w.second, 0, (uint8_t) R });
                g.reg_bytes[R] += w.second;
            }
        }
    }
    for (auto & pc : g.pins) for (auto & p : pc.second) e0_touch(g.view, p.off, p.len);  // re-touch the whole table
    typedef HANDLE(WINAPI * QMRN)(LPCWSTR, void *);                   // memory-pressure cooperation (table-side fast shrink)
    typedef BOOL(WINAPI * GMRN_T)(HANDLE, PBOOL);
    static QMRN qmrn = (QMRN) GetProcAddress(GetModuleHandleA("kernel32"), "QueryMemoryResourceNotification");
    static GMRN_T gmrn = (GMRN_T) GetProcAddress(GetModuleHandleA("kernel32"), "GetMemoryResourceNotification");
    static HANDLE hn = qmrn ? qmrn(L"MemoryLow", nullptr) : nullptr;
    if (hn && gmrn) {
        BOOL low = FALSE;
        if (gmrn(hn, &low) && low) {
            ++g.shrinks;
            for (int R = 0; R < nreg; ++R)
                while (g.reg_bytes[R] && (R || g.reg_bytes[R] * 4 > g.reg_cap[R] * 3)) {
                    size_t b = g.reg_bytes[R];
                    e0_pool_evict(R);
                    if (g.reg_bytes[R] == b) break;
                }
        }
    }
}

// ── Advisory issue: PVM soft warm-up + VirtualLock hard retention (lock failure once the
// budget is hit = natural ceiling). Callable from the worker thread or the main thread
// (synchronous / stride-skipped); pred_prev is copied under a short wmtx lock.
static void e0_issue() {
    if (g.pool) { e0_issue_pool(); return; }          // pool accounting takes over (legacy pin/prefetch path retired)
    if (!g.view || (!g.do_prefetch && !g.do_pin)) return;
    std::map<int, std::vector<int>> preds;
    { std::lock_guard<std::mutex> lk(g.wmtx); preds = g.pred_prev; }
    static WIN32_MEMORY_RANGE_ENTRY win[4 * 3 * NH];
    int nw = 0;
    for (auto & pc : preds) {
        const int c = pc.first;
        std::vector<std::pair<size_t, size_t>> want;
        for (const TT & tt : g.exps[c].t) {
            const size_t per = tt_bytes(tt) / EK;
            for (int e : pc.second) want.push_back({ (size_t) g.data_base + tt.off + (size_t) e * per, per });
        }
        if (g.do_pin) {
            auto & cur = g.pinned[c];
            for (auto & r : cur) {                                   // release stale pins
                bool keep = false;
                for (auto & w : want) if (w.first == r.first) { keep = true; break; }
                if (!keep) { VirtualUnlock((char *) g.view + r.first, r.second); g.pin_bytes -= r.second; }
            }
            for (auto & w : want) {                                  // pin new predictions (skipped once capped)
                bool have = false;
                for (auto & r : cur) if (r.first == w.first) { have = true; break; }
                if (have || g.pin_bytes + w.second > g.pin_cap) continue;
                if (VirtualLock((char *) g.view + w.first, w.second)) g.pin_bytes += w.second;
            }
            cur = want;   // remember current pin state (entries skipped by the cap unlock harmlessly next round)
        }
        if (g.do_prefetch)
            for (auto & w : want) {
                if (nw == 4 * 3 * NH) break;
                win[nw].VirtualAddress = (PVOID) ((char *) g.view + w.first);
                win[nw].NumberOfBytes  = w.second;
                ++nw;
            }
    }
    if (nw) {
        static decltype(&PrefetchVirtualMemory) pvm =
            (decltype(&PrefetchVirtualMemory)) GetProcAddress(GetModuleHandleA("kernel32"), "PrefetchVirtualMemory");
        if (pvm) pvm(GetCurrentProcess(), nw, win, 0);
    }
}

// ── Full-step computation: (1) teacher (2) three-way accounting (3) head → preds + roll history (4) issue ──
// rows = snapshot buffer, row order = layers OWNER0..OWNER0+NH, HH floats per row.
// This function has no ctx dependency ⇒ it can run on the worker thread (keeps the ~90ms
// head GEMV off the critical path).
static void e0_full_step(const float * rows) {
    std::map<int, std::vector<int>> teacher;
    for (int c : consumers) {
        const float * x = rows + (size_t) (c - OWNER0) * HH;
        std::vector<float> tl(EK);
        const float * gw = g.gate[c].data();
        for (int e = 0; e < EK; ++e) tl[e] = dotf(gw + (size_t) e * HH, x, HH);
        topk(tl.data(), EK, KDIM, teacher[c]);
    }
    // Three-way settlement: pred copy (short lock) vs teacher @ this snapshot.
    std::map<int, std::vector<int>> preds_copy;
    { std::lock_guard<std::mutex> lk(g.wmtx); preds_copy = g.pred_prev; }
    for (int c : consumers) {
        auto pp = preds_copy.find(c);
        if (pp != preds_copy.end()) {
            g.hits += ovlp(pp->second, teacher[c]); g.total += KDIM;
            auto sp = g.exec_prev.find(c);
            if (sp != g.exec_prev.end()) g.sticky += ovlp(sp->second, teacher[c]);
            std::vector<int> r;
            for (int tries = 0; tries < 64 && (int) r.size() < KDIM; ++tries) {
                int v = (int) (g.next() % (uint32_t) EK);
                bool dup = false; for (int y : r) if (y == v) { dup = true; break; }
                if (!dup) r.push_back(v);
            }
            g.rnd += ovlp(r, teacher[c]);
        }
        g.exec_prev[c] = teacher[c];
    }
    if (g_pool_on) for (auto & kv : g.exec_prev) e0_pool_prefill(kv.first, kv.second, 0);   // teacher → hot zone
    // head: owner o features = [m_in|cur_oh|prev_oh] → new pred (consumer = o+1)
    std::map<int, std::vector<int>> new_preds;
    std::vector<float> feat(FE), act(512), lg(EK);
    for (int o = OWNER0; o < OWNER0 + NH; ++o) {
        const int c = o + 1;
        auto & h = g.hist[o];
        memcpy(feat.data(),          rows + (size_t) (o - OWNER0) * HH, HH * 4);
        memcpy(feat.data() + HH,     h.cur.data(),  EK * 4);
        memcpy(feat.data() + HH + EK, h.prev.data(), EK * 4);
        const Head & hd = g.heads[o];
        for (int j = 0; j < 512; ++j) act[j] = gelu_erf(dotf(hd.fc1.data() + (size_t) j * FE, feat.data(), FE));
        for (int e = 0; e < EK; ++e)
            lg[e] = dotf(hd.fc2.data() + (size_t) e * 512, act.data(), 512)
                  + dotf(hd.lin.data() + (size_t) e * FE, feat.data(), FE);
        topk(lg.data(), EK, KDIM, new_preds[c]);
    }
    if (g_pool_on && g_pool_cap[1]) for (auto & kv : new_preds) e0_pool_prefill(kv.first, kv.second, 1);  // head → observation zone
    { std::lock_guard<std::mutex> lk(g.wmtx); g.pred_prev = std::move(new_preds); }
    // roll the onehot history (teacher→cur, cur→prev)
    for (auto & kv : g.hist) {
        auto t = teacher.find(kv.first);
        if (t == teacher.end()) continue;
        kv.second.prev = kv.second.cur;
        kv.second.cur.assign(EK, 0.f);
        for (int e : t->second) kv.second.cur[e] = 1.f;
    }
    e0_issue();
    if (g.trace_every && g.steps % g.trace_every == 0 && g.total) {
        fprintf(stderr, "[pref-trace] step %ld: head %.1f%% sticky %.1f%% rand %.1f%% pinned=%lluMB (n=%ld)\n",
                g.steps, 100.0 * g.hits / g.total, 100.0 * g.sticky / g.total,
                100.0 * g.rnd / g.total, (unsigned long long) (g.pin_bytes >> 20), g.total);
        if (g.pool)
            fprintf(stderr, "[pref-trace]  POOL hit=%.1f%% (n=%ld) pins %llu+%lluMB evict=%ld fail=%ld shrink=%ld\n",
                    100.0 * g.pool_hit / (g.pool_tot ? g.pool_tot : 1), g.pool_tot,
                    (unsigned long long) (g.reg_bytes[0] >> 20), (unsigned long long) (g.reg_bytes[1] >> 20),
                    g.evicts, g.lock_fail, g.shrinks);
        fflush(stderr);
    }
}

static void e0_worker() {
    for (;;) {
        {
            std::unique_lock<std::mutex> lk(g.wmtx);
            g.cv.wait(lk, [&] { return g.quit.load() || g.busy.load(); });
            if (g.quit) return;
        }
        if (g.busy.load()) {
            e0_full_step(g.snap.data());
            g.busy = false;
        }
    }
}

bool edge0_pref_init(llama_model * model, llama_context * ctx, int pool_mb, const char * gguf_path) {
    (void) model;
    if (g.active) { if (!g.gate.empty()) enable_capture(ctx); return true; }   // hot reload (only head mode needs capture)
    const char * side = getenv("E0_PREROUTER");
    const char * gg   = (gguf_path && *gguf_path) ? gguf_path : getenv("E0_GGUF");   // primary CLI channel = model path
    if (pool_mb > 0) g.pool_mb = pool_mb;                                        // --pool-mb formal CLI
    else if (const char * pmb = getenv("E0_POOL_MB")) g.pool_mb = atoll(pmb);    // env fallback (harness/debug); runnable without the sidecar
    const bool headson = side && *side;
    if ((!headson && !g.pool_mb) || !gg || !*gg) return false;

    if (consumers.empty()) for (int c = OWNER0 + 1; c <= OWNER0 + NH; ++c) consumers.push_back(c);  // 7..39

    // ── GGUF header: skip-scan KVs (for general.alignment) + build the tensor table ──
    FILE * f = fopen(gg, "rb");
    if (!f) return false;
    struct { uint32_t magic, ver; uint64_t nt, nkv; } h;
    if (fread(&h, sizeof h, 1, f) != 1 || memcmp(&h.magic, "GGUF", 4)) { fclose(f); return false; }
    size_t alignment = 32;
    for (uint64_t i = 0; i < h.nkv; ++i) {
        std::string k = rd_str(f);
        uint32_t t; if (fread(&t, 4, 1, f) != 1) { fclose(f); return false; }
        if (k == "general.alignment" && t == 4) fread(&alignment, 4, 1, f);
        else skip_val(f, t);
    }
    std::map<std::string, TT> tinfo;
    for (uint64_t i = 0; i < h.nt; ++i) {
        TT x; x.name = rd_str(f);
        uint32_t nd; fread(&nd, 4, 1, f);
        for (uint32_t d = 0; d < nd && d < 4; ++d) fread(&x.dim[d], 8, 1, f);
        fread(&x.type, 4, 1, f); fread(&x.off, 8, 1, f);
        tinfo[x.name] = x;
    }
    g.data_base = align_up((uint64_t) ftell(f), alignment);

    // Register exps metadata (needed by both pool and head) + copy the F32 gate weights host-side (head mode only).
    for (int c : consumers) {
        for (const char * suf : {"ffn_up_exps", "ffn_gate_exps", "ffn_down_exps"}) {
            auto et = tinfo.find("blk." + std::to_string(c) + "." + suf + ".weight");
            if (et != tinfo.end() && et->second.dim[2] == (uint64_t) EK) g.exps[c].t.push_back(et->second);
        }
        if (!headson) continue;
        auto it = tinfo.find("blk." + std::to_string(c) + ".ffn_gate_inp.weight");
        if (it == tinfo.end() || it->second.type != 0) {
            fclose(f); fprintf(stderr, "[pref-trace] gate missing/bad type c=%d => dormant\n", c); return false;
        }
        const TT & gt = it->second;
        size_t nb = tt_bytes(gt);
        std::vector<float> buf(nb / 4);
        _fseeki64(f, (long long) (g.data_base + gt.off), SEEK_SET);
        if (fread(buf.data(), 1, nb, f) != nb) { fclose(f); return false; }
        g.gate[c] = std::move(buf);
    }
    fclose(f);

    // sidecar head weights (head mode only)
    if (headson) {
    std::map<std::string, SEnt> st;
    if (!st_load(side, st)) { fprintf(stderr, "[pref-trace] sidecar parse failed => dormant\n"); return false; }
    FILE * sf = fopen(side, "rb");
    if (!sf) return false;
    uint64_t shn = 0; fread(&shn, 8, 1, sf);
    uint64_t sdata = 8 + shn;
    for (int o = OWNER0; o < OWNER0 + NH; ++o) {
        Head hd;
        auto key = [&](const char * w) { return "layers." + std::to_string(o) + "." + w + ".weight"; };
        bool ok = grab_f16(sf, sdata, st[key("fc1")], hd.fc1)
               && grab_f16(sf, sdata, st[key("fc2")], hd.fc2)
               && grab_f16(sf, sdata, st[key("linear_init")], hd.lin);
        if (!ok) { fclose(sf); fprintf(stderr, "[pref-trace] sidecar missing layer o=%d => dormant\n", o); return false; }
        g.heads[o] = std::move(hd);
    }
    fclose(sf);
    }   // headson

    // Dedicated read handle (pool fills via g.fh) + a self-owned read-only mapping (PVM anchor; head mode only).
    if (headson || g.pool_mb) {
        g.fh = CreateFileA(gg, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
                           NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
        if (g.fh != INVALID_HANDLE_VALUE && headson) {
            g.mh = CreateFileMappingA(g.fh, NULL, PAGE_READONLY, 0, 0, NULL);
            if (g.mh) g.view = MapViewOfFile(g.mh, FILE_MAP_READ, 0, 0, 0);
        }
    }
    g.do_prefetch = getenv("E0_PREFETCH") != nullptr;
    if (const char * tn = getenv("E0_PREF_TRACE_N")) g.trace_every = atoi(tn);
    if (const char * sn = getenv("E0_PREF_STRIDE")) g.stride = atoi(sn) > 0 ? atoi(sn) : 1;
    const char * an = getenv("E0_PREF_ASYNC");
    g.async = !(an && *an == '0');                     // async is the default; =0 falls back to synchronous (tax isolation)
    g.do_pin = getenv("E0_PREF_PIN") != nullptr;
    if (const char * pm = getenv("E0_PIN_MAX_MB")) g.pin_cap = (size_t) atoll(pm) << 20;
    if (const char * pl = getenv("E0_POOL")) {                      // sticky=zone1  obs=zone1+2
        if (!strcmp(pl, "obs")) g.pool = 2;
        else if (*pl && strcmp(pl, "0")) g.pool = 1;
    }
    if (g.pool) {
        g.reg_cap[0] = g.pin_cap;                                   // hot zone = E0_PIN_MAX_MB (default 3G)
        const char * om = getenv("E0_POOL_OBS_MB");
        g.reg_cap[1] = (size_t) (om ? atoll(om) : 1024) << 20;      // observation zone default 1G
        g.do_pin = false;                                           // the advisory pin path does not run in parallel
        // Working-set hard cap = startup baseline WS + pool quotas + headroom (the residency contract without SeLockMemory).
        PROCESS_MEMORY_COUNTERS_EX pmc = {};
        pmc.cb = sizeof(pmc);
        if (K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS *) &pmc, sizeof(pmc))) {
            const char * hm = getenv("E0_POOL_WSHEAD_MB");
            size_t head = (size_t) (hm ? atoll(hm) : 1024) << 20;
            SIZE_T wmax = (SIZE_T) pmc.WorkingSetSize + g.reg_cap[0] + g.reg_cap[1] + head;
            SIZE_T qmin = 0, qmax = 0; DWORD qf = 0;      // the Ex variant rejects min=-1 (error 87) ⇒ read current quotas and keep them
            GetProcessWorkingSetSizeEx(GetCurrentProcess(), &qmin, &qmax, &qf);
            BOOL r = SetProcessWorkingSetSizeEx(GetCurrentProcess(), qmin, wmax,
                                                QUOTA_LIMITS_HARDWS_MIN_DISABLE | QUOTA_LIMITS_HARDWS_MAX_ENABLE);
            fprintf(stderr, "[pref-trace] POOL ws-cap: base=%lluMB max=%lluMB set=%d err=%lu\n",
                    (unsigned long long) (pmc.WorkingSetSize >> 20), (unsigned long long) (wmax >> 20),
                    (int) r, r ? 0UL : GetLastError());
        }
    }
    g.snap.assign((size_t) (NH + 1) * HH, 0.f);

    // Open the layer-input capture channel (same staging API as speculative decode; head mode only).
    if (headson) {
        enable_capture(ctx);
        for (int L = OWNER0; L <= OWNER0 + NH; ++L) g.hist[L] = Hist{ std::vector<float>(EK, 0.f), std::vector<float>(EK, 0.f) };
    }
    if (g.pool_mb) {
        g_pool_cap[0] = (size_t) g.pool_mb << 20;
        if (const char * pom = getenv("E0_POOL_OBS_MB")) g_pool_cap[1] = (size_t) atoll(pom) << 20;
        g_pool_sc = getenv("E0_POOL_SELFCHECK") != nullptr;
        if (g_pool_sc && !g.view) {      // self-check needs the view: map it even in pool-only mode
            g.mh = CreateFileMappingA(g.fh, NULL, PAGE_READONLY, 0, 0, NULL);
            if (g.mh) g.view = MapViewOfFile(g.mh, FILE_MAP_READ, 0, 0, 0);
        }
        e0_pool_scan(tinfo);
        for (int c : consumers)
            for (const TT & tt : g.exps[c].t) {
                auto it = g_ptt_by_name.find(tt.name);
                if (it != g_ptt_by_name.end()) g_ptt_by_c[c].push_back(it->second);
            }
    }
    g.active = headson || g_pool_on;
    if (g.active && !headson) { g.async = false; }        // pool-only mode: no worker (demand is self-sufficient)
    if (g.active && headson && g.async) g.worker = std::thread(e0_worker);
    fprintf(stderr, "[pref-trace] INIT ok: heads=%zu gates=%zu exps=%zu map=%s prefetch=%s pin=%s(async=%d) stride=%d K=%d E=%d pool=%d(hot=%lluMB obs=%lluMB)\n",
            g.heads.size(), g.gate.size(), g.exps.size(), g.view ? "ok" : "FAIL",
            g.do_prefetch ? "on" : "off", g.do_pin ? "on" : "off", (int) g.async, g.stride, KDIM, EK,
            (int) g_pool_on, (unsigned long long) (g_pool_cap[0] >> 20), (unsigned long long) (g_pool_cap[1] >> 20));
    return g.active;
}

void edge0_pref_on_step(llama_context * ctx, int64_t last_row) {
    if (!g.active) return;
    if (g.gate.empty()) return;                 // pool-only mode: no capture channel, nothing to account — demand is autonomous
    llama_synchronize(ctx);
    ++g.steps;

    if (g.stride > 1 && (g.steps % g.stride) != 0) {   // downsampled: reuse existing preds, just re-issue (µs scale)
        if (!g.async && !g.pool) e0_issue();            // async=worker already issued on cadence; pool=skipped steps neither age nor evict
        return;
    }
    if (g.async) {
        if (g.busy.load()) return;                     // worker busy: drop this snapshot (preds already valid, lossless)
        float * dst = g.snap.data();
        for (int L = OWNER0; L <= OWNER0 + NH; ++L, dst += HH) {
            float * buf = llama_get_embeddings_layer_inp(ctx, (uint32_t) L);
            if (!buf) return;                          // channel not ready
            memcpy(dst, buf + last_row * HH, (size_t) HH * 4);
        }
        { std::lock_guard<std::mutex> lk(g.wmtx); g.busy = true; }
        g.cv.notify_one();
        return;
    }
    // Synchronous path: snapshot + full step inline (reference/fallback).
    float * dst = g.snap.data();
    for (int L = OWNER0; L <= OWNER0 + NH; ++L, dst += HH) {
        float * buf = llama_get_embeddings_layer_inp(ctx, (uint32_t) L);
        if (!buf) return;
        memcpy(dst, buf + last_row * HH, (size_t) HH * 4);
    }
    e0_full_step(g.snap.data());
}