Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
170 KB
Raw Blame
/* Motore di inferenza OLMoE in C puro, con EXPERT-STREAMING dal disco.
 * Porting del motore Python (engine.py). Obiettivo Stadio A: produrre gli STESSI
 * token id del riferimento (ref.json) -> valida il core prima di scalare a GLM-5.2.
 *
 * Densa (embed, attn, router, norme, lm_head) residente in RAM (float32).
 * Expert letti dal disco on-demand via pread, cache LRU per-layer; le pagine
 * restano nel page cache cosi' i miss LRU non tornano su disco (EXPERT_DROP=1
 * ripristina fadvise(DONTNEED) per macchine con poca RAM).
 * Matmul multi-thread con OpenMP (niente BLAS).
 *
 * ENV VARS:
 *   PILOT=0/1/2/3 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer, 3=3-layer lookahead
 *   HOT=N         : pin top-N hot experts per layer permanently (never evict)
 *   WARMUP=N      : tokens before hot pinning activates (default 5)
 *   WIDE=N        : prefetch top-K*N candidates (default 1, try 2 or 3)
 *   SMOOTH=F      : EMA coefficient for routing momentum (default 0.3, range 0.0-0.95)
 *   CONF_LIMIT=F  : cumulative gate probability threshold for prefetch cutoff (default 0.92)
 *   PILOT_EVICT_GUARD=0/1 : 1=enable LFRU prefetch eviction guard (default), 0=disable
 *   EXPERT_DROP=0/1: 1=fadvise(DONTNEED) after each expert read (old behaviour,
 *                    for RAM-tight boxes); 0=keep pages cached (default)
 *   ROUTE_TRACE=<path>: log every routing decision (one line per moe call,
 *                    position and layer: "<call> <row> <layer> <id>:<gate> ...")
 *                    for offline analysis — tools/route_pairs.py,
 *                    tools/route_coupling_report.py, tools/residency_sim.py.
 *                    Measurement only: it cannot change which experts run.
 *   (expert queue is sorted by eid for SSD read locality)
 */
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <time.h>
#include <pthread.h>
#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
#include <sys/resource.h>
#include <unistd.h>
#endif
#if defined(__GLIBC__)
#include <malloc.h>   /* malloc_trim: kv_room_fit gives the slots it frees back to the system */
#endif
#include "cli_args.h"
#include "st.h"
#ifdef _OPENMP
#include <omp.h>   /* omp_set_num_threads/omp_get_max_threads per omp_tune.h */
#endif
#include "omp_tune.h"
#include "route_trace.h"                    /* shared routing telemetry (#700) */
#include "kv_prefix.h"
#include "pin_pool.h"                       /* piu scatti annidati */   /* riuso del prefisso tra turni (shared) */
#include "serve_codec.h"
#include "serve_budget.h"
#ifdef COLI_SEGMENT_ADAPTER
#include "segment_runtime.h"
#include "segment_adapters.h"
#include "segment_adapter_internal.h"
#endif
#ifdef COLI_EDGE_ADAPTER
#include "edge_runtime.h"
#include "edge_adapters.h"
#include "tok.h"
#include "edge_tok_internal.h"
#endif
#ifdef COLI_VULKAN
/* Vulkan (opt-in, VK=1 build + COLI_VULKAN=1): the resident f32 matrices --
 * attention q/k/v/o, the router and lm_head -- through the shader's fmt 10, the
 * same f32 weights times f32 activations the CPU computes, uploaded on first use;
 * the routed experts through the shared expert tier (vk_tier.h, moe_vk_run), their
 * int8 rows as the slots hold them (fmt 1). The embedding lookup stays on the CPU. */
#include "backend_vulkan.h"
#include "vk_tier.h"
static int g_vk_ready = 0;
static int g_vk_chain = 0;   /* COLI_VK_CHAIN decided on, and the chain's pipelines are up (olmoe_chain.h) */
static int g_olc_fit_on;     /* the partial chain's fit ran: the chain set itself up in model_init (olmoe_chain.h) */
#endif
#ifndef COLI_VULKAN   /* exclusive RAM/VRAM (vk_tier.h) is the Vulkan build's: no device holds an expert */
static inline int  vkt_ram_first(int layer, int eid) { (void)layer; (void)eid; return 0; }
static inline void vkt_ram_gave(void) {}
#endif

#ifdef _WIN32
#include <windows.h>
#define sleep_ms(ms) Sleep(ms)
#else
#define sleep_ms(ms) usleep((ms) * 1000)
#endif



/* ---------- config ---------- */
typedef struct {
    int hidden, n_layers, n_heads, n_kv_heads, head_dim;
    int n_experts, topk, inter, vocab;
    float theta, eps; int norm_topk;
    int stop_ids[8], n_stop;   /* unused (no model-specific hardcoded stops) — chat mode's
                                 * stop set comes entirely from the tokenizer's own special-
                                 * token flags via sample.h's stops_arm_tok */
} Cfg;

/* ---------- pesi densi per-layer ---------- */
typedef struct {
    float *in_ln, *post_ln, *q, *k, *v, *o, *qn, *kn, *gate;
#ifdef COLI_VULKAN
    void *vk_q, *vk_k, *vk_v, *vk_o, *vk_gate;   /* device copies, on first use */
#endif
} Layer;

/* ---------- cache LRU degli expert (pesi QUANTIZZATI) ----------
 * Ogni weight [out,in] tenuto come int8 (per-riga) + scala float per riga.
 * Cosi' la RAM-cache scende da 4 byte/param (f32) a 1 byte/param: e' il
 * meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */
/* pinned=1 means this slot is strongly preferred to keep (hot expert); it will
 * not be evicted during normal LRU eviction, but may be displaced under extreme
 * cache pressure when all slots are pinned or in-flight.
 * busy counts the computations reading the slot, from expert_get to expert_put
 * (under g_pilot_mx): neither eviction takes a busy slot. Without it the PILOT
 * worker could pick the slot the forward pass was multiplying with as its LRU
 * victim and read another expert into it mid-matmul (at cap 1 the only slot). */
typedef struct { int eid; int pinned; int busy; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot;
typedef struct {
    Slot *slots;
    int *slot_by_expert;                  /* expert id -> resident slot, -1 if absent */
    int n, cap;
} LCache;

/* One conversation's KV for a multiplexed serve (KV_SLOTS>1, serve_mux below): its
 * key and value rows and the record of the tokens they hold. The Model holds the
 * conversation a prefill runs on (olm_seq_swap trades it for a parked one); a
 * multiplexed decode step parks them all and reads each row's from its OlmRow. */
typedef struct { float **K, **V; int kv_len, kv_hi; kv_prefix kvp; } OlmSeq;   /* kv_hi: the positions its KV pages reach */
typedef struct { OlmSeq *seq; int pos; } OlmRow;
static int g_olm_mux_slots = 1;   /* KV_SLOTS: the conversations a serve decodes at once */
static OlmSeq *g_olm_mux_seq;      /* [slots]: the conversations the Model does not hold */
static int g_olm_mux_cur;          /* the slot the Model holds, -1 when every one is parked */

typedef struct {
    Cfg c;
    shards S;
    int quant_bits;
    float *embed, *lm_head, *final_norm;
#ifdef COLI_VULKAN
    void *vk_lm_head;
    void *vkchain;          /* the dense chain's device state (olmoe_chain.h), NULL until it runs */
    void *vkchain2;         /* its layers on COLI_VK_DEV2's device, after the primary's (olmoe_chain.h) */
#endif
    Layer *L;
    LCache *cache;          /* [n_layers] */
    uint64_t clock, hits, miss;
    /* Nanoseconds spent inside expert reads, summed across threads. The reads
     * run unlocked and in parallel, so this is an atomic counter rather than a
     * plain double: a per-turn delta of it is what the PROF line reports. */
    uint64_t disk_ns;
    float **K, **V; int kv_len, max_t;
    /* The bytes the expert cache and the KV share when the cache was sized from
     * the automatic budget (0: an explicit cap, nothing shared), and the
     * positions the KV's pages already reach (see kv_room_fit). */
    int64_t room_bytes; int kv_room_t;
    /* What the cached keys and values were built from, so a serve turn that
     * resends the transcript prefills only the new tail. Recorded where the
     * tokens are fed (see kv_prefix.h), never derived from a counter. */
    kv_prefix kvp;
    double dense_load_s;
    /* IMPROVEMENT 2: expert frequency heatmap */
    uint32_t **freq;                   /* per-layer expert counts, owned by route_trace.h */
    uint8_t **ehit;                    /* experts routed this turn, for HITS (dashboard Brain) */
    int freq_token_count, hot_pinned, hot_n, warmup_tokens;
    int token_count;
    /* PREDICTION IMPROVEMENT A: per-layer EMA of gate logits across tokens.
     * momentum_logits[l*E .. (l+1)*E-1] = EMA of gate outputs for layer l.
     * Used exclusively by the PILOT prefetcher to stabilise routing predictions
     * across tokens; does NOT affect actual MoE routing (pr is unchanged). */
    float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */
    float pilot_smooth;     /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */
    uint8_t *is_pinned;     /* [n_layers * n_experts], 1 if expert is globally pinned */
    uint8_t *is_queued;     /* [n_layers * n_experts], 1 if expert is currently in the prefetch queue */
    float pilot_conf_limit; /* CONF_LIMIT env: cumulative gate probability threshold (e.g. 0.92) */
    uint64_t *last_access;  /* [n_layers * n_experts], clock time when expert was last accessed */
    /* A multiplexed decode step (olm_step_rows): row s is the token at
     * mux_rows[s].pos of mux_rows[s].seq; NULL in every other forward. */
    const OlmRow *mux_rows;
} Model;

static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t g_pilot_cv = PTHREAD_COND_INITIALIZER; /* broadcast on every publish */
static struct { int l, e; } pilot_q[4096];
static volatile unsigned pilot_r = 0, pilot_w = 0;
static Model *pilot_m = NULL;
static int g_pilot = 0;
static int g_wide  = 1;  /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */
static int g_pilot_evict_guard = 1; /* PILOT_EVICT_GUARD=0 to disable LFRU prefetch eviction guard */
static int g_expert_drop = 0;       /* EXPERT_DROP=1 restores fadvise(DONTNEED) after expert reads */
#if defined(__AVX2__) || !defined(OLMOE_NO_MAIN)
static int g_fused3 = 0;            /* FUSED3=1: AVX2 activation quant + gate/up pair matmul
                                     * (fused_simd.h: quant_x_q8_avx2, matmul_q_idot_v3,
                                     * matmul_q_idot_pair_v3). Exact integer arithmetic only —
                                     * bit-identical to the stock matmul_q path; OFF by default. */
#endif

static uint64_t lfru_score(uint32_t heat, uint64_t last, uint64_t clock) {
    uint64_t age = (clock > last) ? (clock - last) : 0;
    uint64_t recent = (age < 255) ? (255 - age) : 0;
    return ((uint64_t)heat << 8) | recent;
}

static void pilot_prefetch(Model *m, int lnext, const float *x, int S);
static void *pilot_worker(void *arg);
static void ensure_pilot_worker_started(Model *m);
static void slot_ensure_allocated(Model *m, Slot *s);

#ifdef COLI_CACHE_INDEX_TEST
static uint64_t g_slot_index_probes;
#endif

/* Runtime callers hold g_pilot_mx.  The defensive eid check is intentional:
 * an index bug must degrade to a miss, never serve another expert's weights. */
static Slot *slot_indexed(Model *m, int layer, int eid) {
    if (layer < 0 || layer >= m->c.n_layers || eid < 0 ||
        eid >= m->c.n_experts) return NULL;
    LCache *lc = &m->cache[layer];
    if (!lc->slot_by_expert) return NULL;
#ifdef COLI_CACHE_INDEX_TEST
    g_slot_index_probes++;
#endif
    int i = lc->slot_by_expert[eid];
    if (i < 0 || i >= lc->n || lc->slots[i].eid != eid) return NULL;
    return &lc->slots[i];
}

static void cache_unindex(Model *m, int layer, Slot *s) {
    LCache *lc = &m->cache[layer];
    int eid = s->eid, i = (int)(s - lc->slots);
    if (lc->slot_by_expert && eid >= 0 && eid < m->c.n_experts &&
        lc->slot_by_expert[eid] == i)
        lc->slot_by_expert[eid] = -1;
}

static void cache_hide(Model *m, int layer, Slot *s) {
    cache_unindex(m, layer, s);
    s->eid = -1;
}

static void cache_publish(Model *m, int layer, Slot *s, int eid) {
    LCache *lc = &m->cache[layer];
    cache_unindex(m, layer, s);
    s->eid = eid;
    if (lc->slot_by_expert && eid >= 0 && eid < m->c.n_experts)
        lc->slot_by_expert[eid] = (int)(s - lc->slots);
}

/* A slot being read keeps the index entry of the expert it is loading, marked
 * -(eid+2) as in colibri.c's ecache_reserve: lookups still miss and eviction
 * still skips it (eid < 0), but a second loader of the same expert can see the
 * read in flight instead of starting another one into another slot. */
static void cache_reserve(Model *m, int layer, Slot *s, int eid) {
    LCache *lc = &m->cache[layer];
    cache_hide(m, layer, s);
    s->eid = -(eid + 2);
    if (lc->slot_by_expert && eid >= 0 && eid < m->c.n_experts)
        lc->slot_by_expert[eid] = (int)(s - lc->slots);
}

/* Caller holds g_pilot_mx. */
static int slot_in_flight(Model *m, int layer, int eid) {
    if (layer < 0 || layer >= m->c.n_layers || eid < 0 ||
        eid >= m->c.n_experts) return 0;
    LCache *lc = &m->cache[layer];
    if (!lc->slot_by_expert) return 0;
    int i = lc->slot_by_expert[eid];
    return i >= 0 && i < lc->n && lc->slots[i].eid == -(eid + 2);
}

static void ensure_pilot_worker_started(Model *m) {
    if (!pilot_m) {
        pilot_m = m;
        pthread_t t;
        if (pthread_create(&t, NULL, pilot_worker, NULL) != 0) {
            fprintf(stderr, "Error: Failed to create pilot prefetch worker thread\n");
            exit(1);
        }
        pthread_detach(t);
    }
}

/* ---------- utility ---------- */
static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; }
#if defined(__APPLE__)
static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0*1024.0); }  /* macOS: byte */
#else
static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0); }        /* Linux: KB */
#endif
/* Quanta RAM il sistema offre ancora, in GB. Serve a dimensionare la cache
 * degli esperti quando nessuno ha scelto un numero: senza questa, il default
 * e' una costante che non sa nulla ne' del modello ne' della macchina.
 *
 * Fuori da Linux e dai sistemi con _SC_AVPHYS_PAGES ritorna 0, il che rende il
 * budget automatico pari a cio' che il processo gia' tiene: la cache risulta
 * minima invece che sbagliata, e --ram (o --cap) resta la via esplicita. */
static double mem_available_gb(void) {
    /* The one cross-platform "RAM available now" probe, shared with every other
     * engine (colibri.c, glm53.c). This engine used to read /proc/meminfo
     * directly and had no macOS or Windows branch, so on a Mac or a Windows
     * box it returned 0.0 and the auto budget collapsed to the dense resident
     * footprint -- a one-slot-per-layer cache with no warning (#1601).
     * Dev already had a fallback to half physical (7ed6084), but it missed the
     * warm-macOS case where free+inactive+purgeable is 0.x GB on a 128 GB box:
     * not zero, but implausible. The 2% floor treats that as unmeasured too. */
    double total = 0, avail = 0;
    compat_meminfo_gb(&total, &avail);
    static int warned = 0;
    if (avail <= 0.0
#ifdef __APPLE__
        || (total > 0.0 && avail < total * 0.02)
#endif
    ) {
        if (!warned) {
            warned = 1;
            if (total > 0.0) {
                fprintf(stderr,
                    "[ram] auto-detect read %.2f GB available of %.2f GB physical -- "
                    "implausibly low, treating as unmeasured and sizing from half "
                    "the physical total; pass --ram <GB> to override\n",
                    avail, total);
                avail = total * 0.5;
            } else {
                fprintf(stderr,
                    "[olmoe] could not measure available RAM on this platform; assuming 8.0 GB "
                    "(a fixed default). Pass --ram <GB> to set the budget explicitly.\n");
                avail = 8.0;
            }
        } else {
            avail = total > 0.0 ? total * 0.5 : 8.0;
        }
    }
    return avail;
}
static float *falloc(int64_t n) { float *p = malloc(n*sizeof(float)); if(!p){fprintf(stderr,"OOM %ld\n",(long)n);exit(1);} return p; }

/* chat mode only (main()'s CHAT=1 path): sampling temperature/top-p and the
 * tokenizer + stop-set machinery. g_temp<=0 -> greedy (sample.h's pick_tok).
 * Declared before #include "sample.h" — it references these by name, and
 * Cfg/falloc above, without its own extern declarations. */
static float g_temp = 0.7f;   /* TEMP env overrides */
static float g_nuc  = 0.95f;  /* NUCLEUS env overrides */
#include "sample.h"

#include "matmul_f32.h"   /* y[S,O] = x[S,I] @ W^T, W [O,I] f32 row-major */

/* matmul for a RESIDENT f32 matrix W [O,I], with *vk caching its device copy:
 * on Vulkan when the device is up, the CPU's matmul otherwise. The backend has one
 * command buffer, so only from the main thread and never inside a parallel region;
 * an upload the device refused leaves the matrix on the CPU for good. Without
 * COLI_VULKAN it is the plain matmul call it replaces. */
#ifdef COLI_VULKAN
static char g_vk_refused;                  /* *vk == &g_vk_refused: stays on the CPU */
static float *olm_dho_reload(void **vk);   /* below: a dropped host copy read back from disk */
static void matmul_res(float *y, const float *x, const float *W, void **vk, int S, int I, int O) {
    int serial = 1;
#ifdef _OPENMP
    serial = !omp_in_parallel();
#endif
    /* with the dense weights on the device only the device is the matrix's one home */
    if (g_vk_ready && serial && *vk != (void *)&g_vk_refused && (coli_vk_dense() || coli_vk_dense_device_only()) &&
        !(*vk && coli_vk_tensor_dev((ColiVkTensor *)*vk))) {   /* a matrix of the second device's layers: only its chain */
        if (coli_vk_matmul((ColiVkTensor **)vk, y, x, W, NULL, 10, S, I, O, 0)) return;
        if (!*vk) *vk = &g_vk_refused;
    }
    if (!W) W = olm_dho_reload(vk);   /* the CPU needs a matrix the device holds alone (a lost device) */
    matmul(y, x, W, S, I, O);
}
static void vk_res_free(void **vk) {
    if (*vk && *vk != (void *)&g_vk_refused) coli_vk_tensor_free((ColiVkTensor *)*vk);
    *vk = NULL;
}
static void olmoe_vk_report(void) {
    if (!g_vk_ready) return;
    size_t bytes = 0, tensors = 0;   /* the dense matrices on the device (a partial chain: its layers' only) */
    coli_vk_mem_info(&bytes, &tensors);
    fprintf(stderr, "[VK] olmoe: %llu matmuls on the GPU (%zu matrices resident, %.1f MiB)\n", coli_vk_matmul_calls(),
            tensors, bytes / 1048576.0);
}
#define MATMUL_RES(y, x, W, vk, S, I, O) matmul_res(y, x, W, &(vk), S, I, O)
#else
#define MATMUL_RES(y, x, W, vk, S, I, O) matmul(y, x, W, S, I, O)
#endif

/* y[1,O] = x[1,I] @ W^T con W quantizzato: q[O,I] int8 + scala per riga.
 * W[o,i] ~= q[o,i]*scale[o]  ->  y[o] = scale[o] * sum_i x[i]*q[o,i].
 * Su ARM: attivazione quantizzata Q8_0 (scala per blocco di 16) + dot int8
 * NEON (sdot dove c'e' dotprod) — stessa famiglia IDOT di glm.c, IDOT=0 per
 * la via scalare byte-esatta. Misurato 2.7x end-to-end su M5.
 *
 * NB: la quantizzazione delle ATTIVAZIONI rende questo percorso non
 * equivalente alla via scalare (issue #1044). Vale per NEON come per AVX2:
 * IDOT e' opt-in (IDOT=1), non piu' attivo di default. */
#if defined(__ARM_NEON)
#include <arm_neon.h>
static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {
    int32x4_t acc = vdupq_n_s32(0);
    int8x16_t va = vld1q_s8(a), vb = vld1q_s8(b);
#if defined(__ARM_FEATURE_DOTPROD)
    acc = vdotq_s32(acc, va, vb);
#else
    acc = vpadalq_s16(acc, vmull_s8(vget_low_s8(va),  vget_low_s8(vb)));
    acc = vpadalq_s16(acc, vmull_s8(vget_high_s8(va), vget_high_s8(vb)));
#endif
    return vaddvq_s32(acc);
}
#define HAVE_FAST_DOT_I8 1
#elif defined(__AVX2__)
#include <immintrin.h>
/* x86 counterpart of the NEON path above (was scalar-only here before —
 * the only fast path was ARM, so x86 boxes silently used the scalar
 * fallback even when AVX2 was available).
 * Sign-extend both int8 vectors to int16 (exact, no precision loss) then
 * madd+horizontal-sum in int32: pure integer arithmetic, so THIS DOT is
 * bit-for-bit identical to a scalar int8 dot, just vectorized.
 *
 * That exactness does NOT extend to the branch that calls it. matmul_q's IDOT
 * path quantizes the ACTIVATIONS to Q8_0 per 16-block before calling this,
 * which the scalar fallback does not do -- so the two paths differ. This
 * comment previously read as if it covered the whole path, which is how
 * issue #1044 stayed invisible. See the note at the idot default below. */
static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {
    __m256i va16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)a));
    __m256i vb16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)b));
    __m256i prod = _mm256_madd_epi16(va16, vb16);           /* 8 x int32, adjacent pairs summed */
    __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(prod), _mm256_extracti128_si256(prod, 1));
    __m128i hi64   = _mm_unpackhi_epi64(sum128, sum128);
    __m128i sum64  = _mm_add_epi32(sum128, hi64);
    __m128i hi32   = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
    __m128i sum32  = _mm_add_epi32(sum64, hi32);
    return _mm_cvtsi128_si32(sum32);
}
#define HAVE_FAST_DOT_I8 1
#elif defined(__SSE4_1__)
#include <immintrin.h>
#include "sse41_kernels.h"
/* Sandy Bridge-EP path: AVX 1.0 only, no FMA, no AVX-2.
 * 16 int8 dot via two 8-wide SSE2 sign-extend + SSE4.1 madd pairs.
 * Bit-for-bit identical to the AVX2 version above (just 2x 128-bit ops
 * instead of 1x 256-bit op). NO FMA here -- this branch targets Sandy Bridge
 * which has no FMA -- so use explicit mul+add for the inner accumulation. */
static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {
    __m128i va_lo = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)a));        /* lower 8 int8 -> 8 int16 */
    __m128i vb_lo = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)b));
    __m128i va_hi = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(a + 8)));   /* upper 8 int8 -> 8 int16 */
    __m128i vb_hi = _mm_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(b + 8)));
    __m128i p_lo = _mm_madd_epi16(va_lo, vb_lo);   /* 4 x int32 from 8 int16 pairs */
    __m128i p_hi = _mm_madd_epi16(va_hi, vb_hi);   /* 4 x int32 from 8 int16 pairs */
    __m128i sum = _mm_add_epi32(p_lo, p_hi);
    /* horizontal reduce 4 x int32 -> 1 x int32 */
    __m128i hi64   = _mm_unpackhi_epi64(sum, sum);
    __m128i sum64  = _mm_add_epi32(sum, hi64);
    __m128i hi32   = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
    return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32));
}
#define HAVE_FAST_DOT_I8 1
#endif
/* Test-only hook, compiled out of the shipping binary.
 *
 * matmul_q reads IDOT once into a static, so a test cannot exercise both paths
 * in one process without it. Guarded by OLMOE_TESTING so production builds have
 * neither the global nor the branch: tests/test_olmoe_matmul_q.c defines it. */
#ifdef OLMOE_TESTING
int matmul_q_idot_force = -1;
static inline void matmul_q_reset_for_test(void) { matmul_q_idot_force = -1; }
#endif

#if defined(__AVX2__)
#include "fused_simd.h"   /* FUSED3=1: quant_x_q8_avx2 + matmul_q_idot{,_pair}_v3 (bit-exact) */
#endif

#if defined(HAVE_FAST_DOT_I8)
/* IDOT, read once (see matmul_q below for why it is opt-in) */
static int matmul_q_idot(void) {
    static int idot = -1;
    if (idot < 0) { const char *e = getenv("IDOT"); idot = (e && *e == '1'); }
#ifdef OLMOE_TESTING
    if (matmul_q_idot_force >= 0) idot = matmul_q_idot_force;
#endif
    return idot;
}
#endif

static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int I, int O) {
#if defined(HAVE_FAST_DOT_I8)
    /* IDOT is OPT-IN. It quantizes the ACTIVATIONS to Q8_0 per 16-block (below),
     * which the scalar fallback never does, so the two paths are not numerically
     * equivalent: measured 5/12 vs 12/12 matching tokens against a reference
     * (issue #1044). Before 2c4e9de x86 had no fast dot at all, so IDOT=1 fell
     * through to the exact scalar path and x86 was token-exact by accident; that
     * commit silently made every AVX2 box lossy by default. Defaulting to off
     * restores the behaviour users actually had.
     *
     * Cost of turning it on: ~+5.8e-4 relative error per dot from the activation
     * quantization (colibri.c:910-915 measures the same mechanism at ~+0.117
     * nats/token on GLM, which is why GLM keeps q/k/v off IDOT). On AVX2-only
     * hardware it is also not faster: 11.01 vs 10.67 tok/s measured on an
     * i5-9600K, n=4 interleaved. Set IDOT=1 to opt in knowingly. */
    if (matmul_q_idot() && I % 16 == 0 && I <= 4096) {
        int nb = I / 16; int8_t xi[4096]; float xs[256];
        for (int b = 0; b < nb; b++) {
            const float *xb = x + b*16;
            float am = 0.f; for (int i = 0; i < 16; i++) { float a = fabsf(xb[i]); if (a > am) am = a; }
            float s = am/127.f; if (s < 1e-12f) s = 1e-12f;
            xs[b] = s; float inv = 1.f/s;
            for (int i = 0; i < 16; i++) xi[b*16+i] = (int8_t)lrintf(xb[i]*inv);
        }
        #pragma omp parallel for schedule(static)
        for (int o = 0; o < O; o++) {
            const int8_t *w = q + (int64_t)o * I;
            float acc = 0.f;
            for (int b = 0; b < nb; b++) acc += xs[b]*(float)dot_i8_16(xi+b*16, w+b*16);
            y[o] = acc * scale[o];
        }
        return;
    }
#endif
    #pragma omp parallel for schedule(static)
    for (int o = 0; o < O; o++) {
        const int8_t *w = q + (int64_t)o * I;
        float acc = 0.f;
        #pragma omp simd reduction(+:acc)
        for (int i = 0; i < I; i++) acc += x[i] * (float)w[i];
        y[o] = acc * scale[o];
    }
}

/* matmul_q on n rows against one matrix: y[t] = matmul_q(x[t]) for t < n, the
 * same bits (each element is the loop above, unchanged), but every weight row is
 * read once for all n rows and the rows share one parallel region. A prompt's
 * MoE (moe_by_expert) hands it all the rows routed to one expert. The IDOT branch
 * quantizes each row on its own, so it keeps the one-row call. */
static void matmul_q_rows(float *const *y, const float *const *x, int n,
                          const int8_t *q, const float *scale, int I, int O) {
#if defined(HAVE_FAST_DOT_I8)
    if (matmul_q_idot() && I % 16 == 0 && I <= 4096) {
        for (int t = 0; t < n; t++) matmul_q(y[t], x[t], q, scale, I, O);
        return;
    }
#endif
    #pragma omp parallel for schedule(static)
    for (int o = 0; o < O; o++) {
        const int8_t *w = q + (int64_t)o * I;
        for (int t = 0; t < n; t++) {
            const float *xt = x[t];
            float acc = 0.f;
            #pragma omp simd reduction(+:acc)
            for (int i = 0; i < I; i++) acc += xt[i] * (float)w[i];
            y[t][o] = acc * scale[o];
        }
    }
}


/* rmsnorm su una riga di lunghezza D, in-place su out (out puo' essere == x) */
static void rmsnorm_row(float *out, const float *x, const float *w, int D, float eps) {
    double ms = 0; for (int i = 0; i < D; i++) ms += (double)x[i]*x[i];
    float r = 1.f / sqrtf((float)(ms / D) + eps);
    for (int i = 0; i < D; i++) out[i] = x[i] * r * w[i];
}

static void softmax_row(float *x, int n) {
    float m = -1e30f; for (int i = 0; i < n; i++) if (x[i] > m) m = x[i];
    float s = 0; for (int i = 0; i < n; i++) { x[i] = expf(x[i]-m); s += x[i]; }
    for (int i = 0; i < n; i++) x[i] /= s;
}

/* ---------- caricamento ---------- */
/* config.json arrives from an untrusted mirror: a missing key was a NULL-deref
 * (json_get(...)->num), so require each dimension present + numeric. */
static double req_num(jval *r, const char *k){
    jval *v=json_get(r,k);
    if(!v||v->t!=J_NUM){ fprintf(stderr,"config.json: missing or non-numeric \"%s\"\n",k); exit(1); }
    return v->num;
}
static void load_cfg(Cfg *c, const char *snap) {
    char path[2048]; snprintf(path, sizeof(path), "%s/config.json", snap);
    FILE *f = fopen(path, "rb"); if(!f){perror(path);exit(1);}
    fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
    if(n<0 || n>(256L<<20)){ fprintf(stderr,"%s: config.json missing or larger than 256 MB\n",path); exit(1); }  /* SEC-9 */
    char *buf = malloc((size_t)n+1); if(!buf){ fprintf(stderr,"OOM reading %s\n",path); exit(1); }
    if(fread(buf,1,(size_t)n,f)!=(size_t)n){ fprintf(stderr,"%s: short read\n",path); exit(1); } buf[n]=0; fclose(f);
    char *arena=NULL; jval *r = json_parse(buf, &arena);
    c->hidden    = (int)req_num(r,"hidden_size");
    c->n_layers  = (int)req_num(r,"num_hidden_layers");
    c->n_heads   = (int)req_num(r,"num_attention_heads");
    c->n_kv_heads= (int)req_num(r,"num_key_value_heads");
    c->n_experts = (int)req_num(r,"num_experts");
    c->topk      = (int)req_num(r,"num_experts_per_tok");
    c->inter     = (int)req_num(r,"intermediate_size");
    c->vocab     = (int)req_num(r,"vocab_size");
    /* range-check so bad dims can't drive a later malloc(inter*hidden) / div-by-zero */
    if(c->hidden<1||c->hidden>(1<<20) || c->n_heads<1||c->n_heads>(1<<16) ||
       c->inter<1||c->inter>(1<<24) || c->vocab<1||c->vocab>(1<<24) ||
       c->n_layers<1||c->n_layers>4096 || c->n_experts<1||c->n_experts>(1<<20) ||
       c->n_kv_heads<1 || c->topk<1||c->topk>c->n_experts){
        fprintf(stderr,"config.json: dimension out of range\n"); exit(1); }
    c->head_dim  = c->hidden / c->n_heads;
    jval *th = json_get(r,"rope_theta");  c->theta = th ? (float)th->num : 10000.f;
    jval *ep = json_get(r,"rms_norm_eps"); c->eps   = ep ? (float)ep->num : 1e-5f;
    jval *nt = json_get(r,"norm_topk_prob"); c->norm_topk = (nt && nt->t==J_BOOL) ? nt->boolean : 0;
    free(buf); free(arena);
}

/* The parameters of an automatic cache size (cap <= 0), for olm_dho_grow_cap: with the
 * dense weights on the device only, the RAM they held goes to the experts. */
typedef struct { int on; double ram_arg, resident, slot_gb; int layers; } OlmAutoCap;
static OlmAutoCap g_olm_auto_cap;

/* rows x columns is the shape the forward uses the tensor with, counted from
 * the config: q/k/v/o are [hidden, hidden], the router [num_experts, hidden],
 * embed_tokens and lm_head [vocab_size, hidden], a norm [hidden] (columns 0).
 * Allocated from the header's element count instead, a tensor shorter than
 * that was read past its end, and one of another shape was used as if it had
 * this one. Exact, like the experts in load_expert_merged(). */
static float *load_t(Model *m, const char *name, int64_t rows, int64_t columns) {
    st_tensor *t = st_find(&m->S, name);
    if (!t) { fprintf(stderr, "missing %s\n", name); exit(1); }
    if (t->rank != (columns ? 2 : 1) || t->shape[0] != rows || (columns && t->shape[1] != columns)) {
        char got[192] = "[";
        for (int k = 0; k < t->rank; k++)
            snprintf(got + strlen(got), sizeof(got) - strlen(got), "%s%lld", k ? ", " : "",
                     (long long)t->shape[k]);
        strncat(got, "]", sizeof(got) - strlen(got) - 1);
        char want[64];
        if (columns) snprintf(want, sizeof(want), "[%lld, %lld]", (long long)rows, (long long)columns);
        else         snprintf(want, sizeof(want), "[%lld]", (long long)rows);
        fprintf(stderr, "%s: shape %s, expected %s from config.json, refusing (untrusted container)\n",
                name, got, want);
        exit(1);
    }
    float *p = falloc(t->numel);
    st_read_f32(&m->S, name, p, 0);   /* densa: niente DONTNEED, resta residente */
    return p;
}

/* One slot holds one expert: three int8 matrices plus their row scales, the
 * same arithmetic the Segment adapter uses to turn a memory limit into a cap. */
static int64_t slot_bytes(const Cfg *c) {
    return (int64_t)c->hidden * c->inter * 3 +
           (int64_t)(c->inter * 2 + c->hidden) * (int64_t)sizeof(float);
}

static void model_init_range(Model *m, const char *snap, int cap, int bits,
                             int layer_begin, int layer_end,
                             int load_boundaries, int init_telemetry) {
    memset(m, 0, sizeof(*m));
    m->quant_bits = bits;
    load_cfg(&m->c, snap);
    st_init(&m->S, snap);
    Cfg *c = &m->c;
    if (layer_end == 0) layer_end = c->n_layers;
    if (layer_begin < 0 || layer_end > c->n_layers ||
        layer_begin >= layer_end) {
        fprintf(stderr, "invalid OLMoE layer range [%d,%d) for %d layers\n",
                layer_begin, layer_end, c->n_layers);
        exit(1);
    }
    double t0 = now_s();
    if (load_boundaries) {
        m->embed      = load_t(m, "model.embed_tokens.weight", c->vocab, c->hidden);
        m->lm_head    = load_t(m, "lm_head.weight", c->vocab, c->hidden);
        m->final_norm = load_t(m, "model.norm.weight", c->hidden, 0);
    }
    m->L = calloc(c->n_layers, sizeof(Layer));
    char nm[256];
    const int64_t D = c->hidden;
    for (int i = layer_begin; i < layer_end; i++) {
        Layer *l = &m->L[i];
        #define LD(field, suffix, rows, columns) \
            snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_t(m,nm,rows,columns)
        LD(in_ln,  "input_layernorm.weight", D, 0);
        LD(post_ln,"post_attention_layernorm.weight", D, 0);
        LD(q, "self_attn.q_proj.weight", D, D); LD(k, "self_attn.k_proj.weight", D, D);
        LD(v, "self_attn.v_proj.weight", D, D); LD(o, "self_attn.o_proj.weight", D, D);
        LD(qn,"self_attn.q_norm.weight", D, 0); LD(kn,"self_attn.k_norm.weight", D, 0);
        LD(gate, "mlp.gate.weight", c->n_experts, D);
        #undef LD
    }
    /* cap <= 0 is "you decide", the sentinel the launcher sends when nobody
     * asked for a number (glm53 already reads it that way; #1443 asked for the
     * same here). Sized HERE rather than in main because the dense weights are
     * resident by this line: rss_gb() is a measurement, not a projection, which
     * is what made the same budget in kimi_k3 (#855) the simpler of the two.
     *
     * Until now "no explicit choice" arrived as a constant 8 slots per layer,
     * which knows nothing about the model or the machine. On a 16 GB box whose
     * entire expert set is 6.5 GB that constant costs a factor of five:
     * measured on a 1204-token prefill, cap 8 gives 22.8% expert hit rate and
     * 0.045 tok/s, cap 64 gives 99.4% and 0.215 tok/s. */
    if (cap <= 0) {
        double resident = rss_gb();
        double avail = mem_available_gb();
        const char *ram_env = getenv("RAM_GB");
        double ram_arg = ram_env ? atof(ram_env) : 0.0;
        /* An explicit --ram is a ceiling on the WHOLE process. Without it take
         * 88% of what the OS still offers and add what we already hold, the
         * same fraction and the same reason as the sibling engines: overshoot
         * means an OOM kill mid-generation, which is worse than a small cache. */
        double budget = ram_arg > 0.0 ? ram_arg : resident + avail * 0.88;
        /* The KV is not set aside here. Its pages are faulted in as positions
         * are written, so the cache and the KV share this room: kv_room_fit()
         * takes a layer's slots back as the KV reaches their bytes. Setting
         * aside the KV of the whole context (CTX, 1.07 GB at 4096 positions on
         * OLMoE-1B-7B) cost the cache that much even for a request that writes
         * a few hundred positions. */
        double slot_gb = (double)slot_bytes(c) / 1e9;
        int layers = layer_end - layer_begin;
        if (layers < 1) layers = 1;
        double room = budget - resident - 0.5;   /* 0.5 GB: activations */
        m->room_bytes = room > 0.0 ? (int64_t)(room * 1e9) : 0;
        int derived = room > 0.0 && slot_gb > 0.0
                    ? (int)(room / slot_gb / (double)layers) : 0;
        if (derived < 1) {
            fprintf(stderr,
                "[cache] no room for even one expert slot/layer (room %.1f GB vs %.0f MB/expert) "
                "-- the cache will hold 1 slot/layer and decode will be slow; pass --ram <GB> "
                "or --cap <slots> to size it\n",
                room, slot_gb * 1000.0);
            derived = 1;
        }
        if (derived > c->n_experts) derived = c->n_experts;
        if (load_boundaries) {   /* the standalone engine: kept for olm_dho_grow_cap */
            g_olm_auto_cap = (OlmAutoCap){1, ram_arg, resident, slot_gb, layers};
        }
        fprintf(stderr, "[cache] %d slots/layer of %d experts: %.1f GB budget "
                        "(%s), %.1f GB dense resident, %.0f MB per expert; "
                        "the KV takes slots back as the context grows\n",
                derived, c->n_experts, budget,
                ram_arg > 0.0 ? "RAM_GB" : "88% of what the OS still offers",
                resident, slot_gb * 1000.0);
        cap = derived;
    }
    m->cache = calloc(c->n_layers, sizeof(LCache));
    for (int i = layer_begin; i < layer_end; i++) {
        m->cache[i].cap = cap;
        m->cache[i].slots = calloc(cap, sizeof(Slot));
        m->cache[i].slot_by_expert = malloc((size_t)c->n_experts * sizeof(int));
        if (!m->cache[i].slot_by_expert) { fprintf(stderr,"OOM expert cache index\n"); exit(1); }
        for (int e = 0; e < c->n_experts; e++) m->cache[i].slot_by_expert[e] = -1;
    }
    /* IMPROVEMENT 2: frequency heatmap for hot expert pinning */
    if (init_telemetry) {
        rt_init("olmoe", c->n_layers, c->n_experts);
        rt_drop_row(c->n_layers);                 /* every layer routes; no MTP row */
        m->freq = rt_counts_all();                /* read sites keep their shape */
        const char *up = getenv("COLI_USAGE");    /* optional history seed */
        if (up && *up) {
            int64_t h = rt_load(up);
            if (h > 0)
                fprintf(stderr,
                        "[USAGE] expert history: %lld selections (%s)\n",
                        (long long)h, up);
        }
    } else {
        /* A process can host several ranges. Keep their optional counters
         * detached from route_trace.h's process-global singleton. */
        m->freq = calloc((size_t)c->n_layers, sizeof(*m->freq));
    }
    m->hot_pinned = 0; m->freq_token_count = 0;
    m->hot_n         = getenv("HOT")    ? atoi(getenv("HOT"))    : 0;
    m->warmup_tokens = getenv("WARMUP") ? atoi(getenv("WARMUP")) : 5;
    m->token_count = 0;
    /* PREDICTION A: routing momentum — EMA of gate logits across tokens.
     * Initialized to zero; first token sets EMA = fresh logits. */
    m->momentum_logits = calloc((size_t)c->n_layers * c->n_experts, sizeof(float));
    float sv = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;
    if (sv < 0.f) sv = 0.f; if (sv > 0.95f) sv = 0.95f;
    m->pilot_smooth = sv;
    m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t));
    m->is_queued = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t));
    m->last_access = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint64_t));
    float cl = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;
    if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f;
    m->pilot_conf_limit = cl;
    m->dense_load_s = now_s() - t0;

    /* Persistent hot pinning belongs to the standalone runtime. A Segment
     * range must never enqueue expert reads for layers it does not own. */
    char pinpath[512];
    snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap);
    FILE *pinf = init_telemetry ? fopen(pinpath, "rb") : NULL;
    if (pinf) {
        size_t expected_size = (size_t)c->n_layers * c->n_experts;
        if (fread(m->is_pinned, 1, expected_size, pinf) == expected_size) {
            m->hot_pinned = 1;
            printf("[HOT] Loaded persistent pinning from %s\n", pinpath);
            
            if (g_pilot) {
                ensure_pilot_worker_started(m);
                for (int l = 0; l < c->n_layers; l++) {
                    for (int e = 0; e < c->n_experts; e++) {
                        if (m->is_pinned[l * c->n_experts + e]) {
                            unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED);
                            unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
                            if (w - r < 4096) {
                                pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = e;
                                pthread_mutex_lock(&g_pilot_mx);
                                m->is_queued[l * c->n_experts + e] = 1;
                                pthread_mutex_unlock(&g_pilot_mx);
                                __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE);
                            }
                        }
                    }
                }
                printf("[HOT] Pre-loading pinned experts into cache...\n");
                double t_wait = now_s();
                while (1) {
                    unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
                    unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE);
                    if (r == w) break;
                    sleep_ms(2);
                }
                printf("[HOT] Pre-loaded in %.1fs!\n", now_s() - t_wait);
            }
        }
        fclose(pinf);
    }
}

#ifdef COLI_VULKAN
static void olmoe_vk_tier_start(Model *m);
static void olc_start(Model *m);
static void olc_fit_start(Model *m);
static void olc_place(Model *m);
static void olm_dho_start(Model *m);
static void olm_dho_finish(Model *m);
#endif
static void model_init(Model *m, const char *snap, int cap, int bits) {
    model_init_range(m, snap, cap, bits, 0, 0, 1, 1);
#ifdef COLI_VULKAN
    /* After the weights, for the standalone engine only (Segment ranges stay on
     * the CPU). The host copies stay: they are the fallback, so on a GPU that
     * shares RAM with the CPU the dense set is held twice. The device opens saying
     * the expert tier will be tried (vk_tier.h step 0): on a device sharing the
     * CPU's RAM the dense matrices then stay on the CPU unless COLI_VK_DENSE=1. */
    if (!g_vk_ready) g_vk_ready = coli_vk_init_env_tier("olmoe", vkt_wanted() && m->c.n_experts > 0);
    if (g_vk_ready) olc_fit_start(m);   /* COLI_VK_CHAIN: how many layers the chain places (vkc_fit), before any upload */
    if (g_vk_ready) olm_dho_start(m);   /* COLI_VK_DENSE_HOST: the dense matrices on the device only, before the tier sizes its budget */
    if (g_vk_ready) { olc_place(m); olm_dho_finish(m); }   /* the chain's layers on the device now, so the tier sizes after them */
    if (g_vk_ready) olmoe_vk_tier_start(m);   /* after the history: COLI_USAGE, read above */
    if (g_vk_ready && !vkt_ready() && !coli_vk_dense()) coli_vk_dense_decide("olmoe", 0, 1);   /* no tier after all */
    if (g_vk_ready && coli_vk_dense())
        fprintf(stderr, "[VK] olmoe: %d resident f32 matrices (attention q/k/v/o, router, lm_head) "
                "go to the GPU on first use; %s\n", 5 * m->c.n_layers + 1,
                vkt_ready() ? "routed experts go to the expert tier, the embedding lookup stays on the CPU"
                            : "routed experts and the embedding lookup stay on the CPU");
    olc_start(m);   /* COLI_VK_CHAIN: every layer's dense part as one frame on the device */
#endif
}

static void slot_ensure_allocated(Model *m, Slot *s) {
    if (s->g) return;
    Cfg *c = &m->c;
    int64_t ng = (int64_t)c->inter * c->hidden;
    int64_t nd = (int64_t)c->hidden * c->inter;
    int8_t *w_block = malloc(ng + ng + nd);
    if (!w_block) {
        fprintf(stderr, "Error: Out of memory allocating slot weights block\n");
        exit(1);
    }
    s->g = w_block;
    s->u = w_block + ng;
    s->d = w_block + ng + ng;
    float *s_block = falloc(c->inter + c->inter + c->hidden);
    s->gs = s_block;
    s->us = s_block + c->inter;
    s->ds = s_block + c->inter + c->inter;
    s->pinned = 0;
}

#ifdef COLI_CACHE_INDEX_TEST
/* Model-free tests stand in for the disk read, so they can hold a load open
 * and count how many times each expert is read. */
static void (*g_test_expert_load)(Model *m, int layer, int eid, Slot *s);
#endif

static void load_expert_merged(Model *m, int layer, int eid, Slot *s) {
#ifdef COLI_CACHE_INDEX_TEST
    if (g_test_expert_load) { g_test_expert_load(m, layer, eid, s); return; }
#endif
    char nm[256], qsnm[256];
    snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid);
    snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid);
    /* SEC: st_init skips its numel*esz==nbytes cross-check for dtype-3 (U8/I8)
     * tensors, so a crafted header from an untrusted mirror can declare nbytes
     * far larger than the config-sized destination and st_read_raw would write
     * past w_block (heap overflow); an oversized .qs likewise overruns s->gs.
     * slot_ensure_allocated sizes those buffers exactly ng+ng+nd bytes and
     * inter+inter+hidden floats, so require an exact match. Reject, never
     * repair — same contract as qt_resolve_fmt in the GLM engine. */
    Cfg *cc = &m->c;
    int64_t ng = (int64_t)cc->inter * cc->hidden, nd = (int64_t)cc->hidden * cc->inter;
    int64_t want_w = ng + ng + nd;
    int64_t want_s = (int64_t)cc->inter + cc->inter + cc->hidden;
    st_tensor *tw = st_find(&m->S, nm), *ts = st_find(&m->S, qsnm);
    if (!tw || tw->nbytes != want_w) {
        fprintf(stderr, "%s: expert weight is %lld bytes — expected %lld for [inter=%d,hidden=%d], "
                "refusing (untrusted container)\n", nm, (long long)(tw ? tw->nbytes : -1),
                (long long)want_w, cc->inter, cc->hidden); exit(1); }
    if (!ts || ts->numel != want_s) {
        fprintf(stderr, "%s: scale array is %lld elems — expected %lld, refusing (untrusted container)\n",
                qsnm, (long long)(ts ? ts->numel : -1), (long long)want_s); exit(1); }
    double started = now_s();
    st_read_raw(&m->S, nm, s->g, g_expert_drop);
    st_read_f32(&m->S, qsnm, s->gs, 0);  /* scales are F32; use typed reader for dtype safety */
    __atomic_fetch_add(&m->disk_ns, (uint64_t)((now_s() - started) * 1e9), __ATOMIC_RELAXED);
}

#ifdef COLI_VULKAN
#define OLMOE_VK_ROWS 64   /* rows per expert-tier step (moe_vk_run) */
/* One expert as a slot holds it: int8 gate, up, down, one f32 scale per row. */
static VktExpertSrc olmoe_vk_src(const Slot *e) {
    return (VktExpertSrc){e->g, e->u, e->d, e->gs, e->us, e->ds};
}
/* Is the expert in this layer's RAM cache now (the tier's balance asks)? */
static int olmoe_vk_in_ram(void *ctx, int layer, int e) {
    pthread_mutex_lock(&g_pilot_mx);
    int r = slot_indexed((Model *)ctx, layer, e) != NULL;
    pthread_mutex_unlock(&g_pilot_mx);
    return r;
}
/* COLI_VULKAN=1: describe the experts to the tier (int8 rows with an f32 scale per
 * row, gate, up and down as the merged container holds them: fmt 1), after the
 * weights and the history (COLI_USAGE, read by model_init_range), and warm it from
 * that history: the hottest experts first, read from disk in parallel. */
/* The tier's streaming (a big prompt chunk's cold experts on the device): an expert's
 * bytes through the layer cache as the CPU path gets them, held until copied. */
static void expert_get(Model *m, int layer, int eid, Slot **out);
static void expert_put(Slot *s);
static int olmoe_vk_load(void *ctx, int layer, int e, VktExpertSrc *src, void **h) {
    Slot *s = NULL; expert_get((Model *)ctx, layer, e, &s);
    if (!s) return 0;
    *src = olmoe_vk_src(s); *h = s;
    return 1;
}
static void olmoe_vk_release(void *ctx, void *h) { (void)ctx; expert_put((Slot *)h); }
static void olmoe_vk_tier_start(Model *m) {
    Cfg *c = &m->c;
    if (!vkt_wanted() || c->n_experts < 1) return;
    int64_t D = c->hidden, I = c->inter;
    size_t slotb = (size_t)(3 * I * D + (2 * I + D) * 4);
    /* device only, or the chain set up already with its fit (olc_place: the CPU's layers
     * refused): placed already, or never going up */
    size_t dense = coli_vk_dense() && !coli_vk_dense_device_only() && !g_olc_fit_on
                 ? (size_t)c->n_layers * (size_t)(4 * D * D + (int64_t)c->n_experts * D) * 4 +
                   (size_t)c->vocab * (size_t)D * 4 : 0;
    VktConfig vc = {.engine = "olmoe", .layers = c->n_layers, .experts = c->n_experts,
                    .hidden = c->hidden, .inter = c->inter, .topk = c->topk,
                    .gate_up = {VKT_SRC_I8_ROW, 0}, .down = {VKT_SRC_I8_ROW, 0},
                    .act = VKT_ACT_SWIGLU, .max_rows = OLMOE_VK_ROWS * c->topk,
                    .ram_reserve = slotb * (size_t)m->cache[0].cap * (size_t)c->n_layers,
                    .dense_bytes = dense, .in_ram = olmoe_vk_in_ram, .ram_ctx = m,
                    .load = olmoe_vk_load, .release = olmoe_vk_release, .load_ctx = m};
    atexit(coli_vk_shutdown);   /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
    if (!vkt_init(&vc, m->freq)) return;
    atexit(vkt_shutdown);
    int all = c->n_layers * c->n_experts;
    int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));
    const char *warm = getenv("COLI_VK_TIER_WARM");   /* 0: no warm start, the tier fills as experts pass by */
    int n = pl && pe && !(warm && *warm == '0') ? vkt_plan(pl, pe, all) : 0;
    if (n > 0) {
        double t0 = now_s();
        #pragma omp parallel for schedule(dynamic, 4)
        for (int i = 0; i < n; i++) {
            Slot tmp; memset(&tmp, 0, sizeof tmp);
            slot_ensure_allocated(m, &tmp);
            load_expert_merged(m, pl[i], pe[i], &tmp);
            VktExpertSrc src = olmoe_vk_src(&tmp);
            vkt_put(pl[i], pe[i], &src);
            free(tmp.g); free(tmp.gs);   /* the two blocks slot_ensure_allocated made */
        }
        vkt_put_done();
        fprintf(stderr, "[VK] tier olmoe: warm start, %d experts from the history in %.1fs\n", n, now_s() - t0);
    }
    free(pl); free(pe);
}
#endif

/* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */
/* One byte per expert: routed in this turn or not. The dashboard's Brain tab
 * reads it as the HITS bitmap after every turn (serve_hits), and it is cleared
 * there. Outside OLMOE_NO_MAIN: expert_get is in the segment adapter object. */
static void ehit_mark(Model *m, int layer, int eid) {
    Cfg *c = &m->c;
    if (!m->ehit) {
        m->ehit = calloc((size_t)c->n_layers, sizeof(uint8_t *));
        for (int i = 0; i < c->n_layers; i++) m->ehit[i] = calloc((size_t)c->n_experts, 1);
    }
    if (layer >= 0 && layer < c->n_layers && eid >= 0 && eid < c->n_experts) m->ehit[layer][eid] = 1;
}

/* The unpinned slot a full layer cache gives up (caller holds g_pilot_mx), never one
 * being loaded or read: the least recently used, or before it the least recently used
 * of those whose expert the Vulkan tier holds. -1 when there is none. */
static int olmoe_victim(const LCache *lc, int layer) {
    int lru = -1, dev = -1;
    for (int i = 0; i < lc->n; i++) {
        if (lc->slots[i].pinned || lc->slots[i].eid < 0 || lc->slots[i].busy) continue;
        if (vkt_ram_first(layer, lc->slots[i].eid)) { if (dev < 0 || lc->slots[i].used < lc->slots[dev].used) dev = i; continue; }
        if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
    }
    return dev >= 0 ? dev : lru;
}
static void expert_get(Model *m, int layer, int eid, Slot **out) {
    LCache *lc = &m->cache[layer];
    pthread_mutex_lock(&g_pilot_mx);
    ehit_mark(m, layer, eid);          /* under the lock: the routing loop is parallel */
    Slot *hit = slot_indexed(m, layer, eid);
    /* The prefetcher usually reads the next layer's experts while this one
     * computes, so a routed expert is often already on its way: wait for that
     * read to publish rather than read the same bytes again into another slot. */
    while (!hit && slot_in_flight(m, layer, eid)) {
        pthread_cond_wait(&g_pilot_cv, &g_pilot_mx);
        hit = slot_indexed(m, layer, eid);
    }
    if (hit) {
        m->hits++; hit->used = ++m->clock; hit->busy++; *out = hit;
        if (m->last_access) m->last_access[layer * m->c.n_experts + eid] = m->clock;
        pthread_mutex_unlock(&g_pilot_mx);
        return;
    }
    m->miss++;
    Cfg *c = &m->c;
    Slot *s;
    if (lc->n < lc->cap) {
        s = &lc->slots[lc->n++];
        slot_ensure_allocated(m, s);
    } else {
        /* LRU eviction: skip pinned, in-flight (eid==-1) and busy slots; an expert the
         * Vulkan tier holds goes first (vkt_ram_first: no second copy when RAM is short) */
        int lru = olmoe_victim(lc, layer);
        if (lru < 0) {
            /* All slots are pinned, in-flight or busy; find oldest non-in-flight slot
             * (may be pinned, but never select one currently being loaded or read). */
            for (int i = 0; i < lc->n; i++) {
                if (lc->slots[i].eid < 0 || lc->slots[i].busy) continue; /* never evict in-flight */
                if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
            }
        }
        while (lru < 0) {
            /* EVERY slot is in flight: each buffer is owned by an unlocked pread
             * in the pilot worker (or a demand load) that will publish into it.
             * The old last resort (lru=0) stole such a slot mid-load — two writers
             * racing the same slab, then whichever published last decided the
             * expert id the resident bytes answered to. Wait for a publish instead
             * and rescan; in-flight always drains because a load either finishes
             * or the process is already dead in the water. */
            pthread_mutex_unlock(&g_pilot_mx);
            sleep_ms(1);
            pthread_mutex_lock(&g_pilot_mx);
            for (int i = 0; i < lc->n; i++) {
                if (lc->slots[i].eid < 0 || lc->slots[i].busy) continue;
                if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
            }
        }
        s = &lc->slots[lru];
        s->pinned = 0;
        if (vkt_ram_first(layer, s->eid)) vkt_ram_gave();
    }
    cache_reserve(m, layer, s, eid);
    s->used = ++m->clock;
    pthread_mutex_unlock(&g_pilot_mx);

    load_expert_merged(m, layer, eid, s);

    pthread_mutex_lock(&g_pilot_mx);
    cache_publish(m, layer, s, eid);
    s->pinned = m->is_pinned[layer * c->n_experts + eid];
    s->used = ++m->clock;
    s->busy++;
    if (m->last_access) m->last_access[layer * c->n_experts + eid] = m->clock;
    *out = s;
    pthread_cond_broadcast(&g_pilot_cv);
    pthread_mutex_unlock(&g_pilot_mx);
}
/* The computation that expert_get handed the slot to is done reading it. */
static void expert_put(Slot *s) {
    pthread_mutex_lock(&g_pilot_mx);
    s->busy--;
    pthread_mutex_unlock(&g_pilot_mx);
}

/* The bytes the KV's pages reach once `positions` are written: K and V of
 * every layer hold n_heads rows of max_t * head_dim floats, each written from
 * its start, and a row's first B bytes touch at most B / 4096 + 2 pages. */
static int64_t kv_room_bytes(const Model *m, int positions) {
    const Cfg *c = &m->c;
    int64_t rows = 2 * (int64_t)c->n_layers * c->n_heads;
    return rows * ((int64_t)positions * c->head_dim * (int64_t)sizeof(float) + 2 * 4096);
}

/* The expert cache and the KV share one room (room_bytes, set by the automatic
 * budget in model_init_range). Before a forward writes positions the KV has
 * not reached yet, every layer's cap comes down to what the room still holds,
 * never below one slot, and each layer frees its least recently used slots
 * down to it (a pinned one only when nothing else is left). Which experts sit
 * in RAM changes, the logits do not. Called from step(), between forwards: no
 * slot is busy, and a PILOT read in flight is waited for as in expert_get. */
static void kv_room_fit(Model *m, int positions) {
    if (m->room_bytes <= 0 || positions <= m->kv_room_t) return;
    m->kv_room_t = positions;
    Cfg *c = &m->c;
    int64_t left = m->room_bytes - kv_room_bytes(m, positions);
    int64_t fit = left > 0 ? left / slot_bytes(c) / c->n_layers : 0;
    int cap = fit < 1 ? 1 : fit > c->n_experts ? c->n_experts : (int)fit;
    int shrunk = 0;
    pthread_mutex_lock(&g_pilot_mx);
    for (int l = 0; l < c->n_layers; l++) {
        LCache *lc = &m->cache[l];
        if (!lc->slots || cap >= lc->cap) continue;
        lc->cap = cap; shrunk = 1;
        while (lc->n > cap) {
            int v = -1;
            for (int i = 0; i < lc->n; i++) {
                Slot *s = &lc->slots[i];
                if (s->eid < 0 || s->busy) continue;
                if (v < 0 || s->pinned < lc->slots[v].pinned ||
                    (s->pinned == lc->slots[v].pinned && s->used < lc->slots[v].used)) v = i;
            }
            Slot *last = &lc->slots[lc->n - 1];
            if (v < 0 || (v != lc->n - 1 && (last->eid < 0 || last->busy))) {
                /* the slot to free or the one to move into its place is being
                 * read into: wait for that read to publish */
                pthread_mutex_unlock(&g_pilot_mx);
                sleep_ms(1);
                pthread_mutex_lock(&g_pilot_mx);
                continue;
            }
            Slot *s = &lc->slots[v];
            cache_unindex(m, l, s);
            free(s->g); free(s->gs);   /* the two blocks slot_ensure_allocated made */
            if (s != last) {
                *s = *last;
                if (lc->slot_by_expert && s->eid >= 0 && s->eid < c->n_experts)
                    lc->slot_by_expert[s->eid] = v;
            }
            memset(last, 0, sizeof *last);
            lc->n--;
        }
    }
    pthread_mutex_unlock(&g_pilot_mx);
    if (!shrunk) return;
#if defined(__GLIBC__)
    malloc_trim(0);   /* glibc keeps freed blocks in its heap unless asked */
#endif
    fprintf(stderr, "[cache] the KV reaches %d positions: %d slots/layer\n", positions, cap);
}

/* ---------- IMPROVEMENT 2: pin top-N hot experts per layer ---------- */
static void pin_hot_experts(Model *m) {
    Cfg *c = &m->c;
    if (m->hot_n <= 0 || m->hot_pinned) return;
    m->hot_pinned = 1;
    
    int is_dynamic = (m->hot_n >= 100);
    double thresh = is_dynamic ? (double)m->hot_n / 1000.0 : 0.0;
    
    int pinned_total = 0;
    for (int l = 0; l < c->n_layers; l++) {
        uint32_t *freq_l = m->freq[l];
        if (!freq_l) continue;                    /* a layer with no row cannot be ranked */

        uint64_t layer_total = 0;
        for (int e = 0; e < c->n_experts; e++) layer_total += freq_l[e];
        if (layer_total == 0) continue;

        int max_pin = m->cache[l].cap - 8;
        if (max_pin < 4) max_pin = 4;
        
        int hn = is_dynamic ? max_pin : (m->hot_n < c->n_experts ? m->hot_n : c->n_experts);
        if (hn > 256) hn = 256;
        int hot_eids[256];
        int actual_hn = 0;
        
        for (int k = 0; k < hn; k++) {
            int best = -1; uint32_t bv = 0;
            for (int e = 0; e < c->n_experts; e++) {
                int already = 0;
                for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already = 1; break; }
                if (!already && freq_l[e] > bv) { bv = freq_l[e]; best = e; }
            }
            if (best < 0 || bv == 0) break;
            if (is_dynamic && bv < thresh * layer_total) break;
            hot_eids[k] = best;
            actual_hn++;
        }
        
        for (int k = 0; k < actual_hn; k++) {
            int eid = hot_eids[k];
            m->is_pinned[l * c->n_experts + eid] = 1;

            int found = 0;
            pthread_mutex_lock(&g_pilot_mx);
            Slot *resident = slot_indexed(m, l, eid);
            if (resident) { resident->pinned = 1; found = 1; }
            pthread_mutex_unlock(&g_pilot_mx);
            if (!found && g_pilot > 0) {
                /* Only enqueue when the prefetch worker is active (PILOT>0). */
                ensure_pilot_worker_started(m);
                unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED);
                unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
                int gidx = l * c->n_experts + eid;
                pthread_mutex_lock(&g_pilot_mx);
                int already = m->is_queued[gidx];
                if (!already && w - r < 4096) {
                    pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = eid;
                    m->is_queued[gidx] = 1;
                    __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE);
                }
                pthread_mutex_unlock(&g_pilot_mx);
            }
            pinned_total++;
        }
    }
    if (is_dynamic) {
        printf("[HOT] Dynamic Pinned %d experts total (thresh=%.1f%%) after %d warmup tokens\n",
               pinned_total, thresh * 100.0, m->freq_token_count);
    } else {
        printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n",
               pinned_total, m->hot_n, m->freq_token_count);
    }
}


/* ---------- RoPE su un vettore di una testa (head_dim) a posizione assoluta pos ---------- */
static void rope_head(float *x, int pos, const Cfg *c) {
    int h = c->head_dim / 2;
    for (int j = 0; j < h; j++) {
        float inv = powf(c->theta, -2.0f * j / c->head_dim);
        float ang = pos * inv, cs = cosf(ang), sn = sinf(ang);
        float a = x[j], b = x[j+h];
        x[j]   = a*cs - b*sn;
        x[j+h] = b*cs + a*sn;
    }
}

/* attenzione sui token nuovi x[S,hidden]; pos_base = posizione assoluta del primo token nuovo */
static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_base, float *out) {
    Cfg *c = &m->c; int H = c->n_heads, hd = c->head_dim, D = c->hidden;
    float *q = falloc((int64_t)S*D), *k = falloc((int64_t)S*D), *vv = falloc((int64_t)S*D);
    MATMUL_RES(q, x, l->q, l->vk_q, S, D, D);
    MATMUL_RES(k, x, l->k, l->vk_k, S, D, D);
    MATMUL_RES(vv, x, l->v, l->vk_v, S, D, D);
    /* qk-norm sull'intero vettore hidden, poi RoPE per testa */
    for (int s = 0; s < S; s++) {
        rmsnorm_row(q + (int64_t)s*D, q + (int64_t)s*D, l->qn, D, c->eps);
        rmsnorm_row(k + (int64_t)s*D, k + (int64_t)s*D, l->kn, D, c->eps);
        int pos = m->mux_rows ? m->mux_rows[s].pos : pos_base + s;
        for (int hh = 0; hh < H; hh++) { rope_head(q + (int64_t)s*D + hh*hd, pos, c); rope_head(k + (int64_t)s*D + hh*hd, pos, c); }
    }
    /* scrive k,v nella kv-cache alle posizioni pos_base..pos_base+S-1 (in un passo
     * multiplexato: ogni riga alla sua, nella cache della sua conversazione) */
    const OlmRow *mr = m->mux_rows;
    for (int s = 0; s < S; s++) for (int hh = 0; hh < H; hh++) {
        int t = mr ? mr[s].pos : pos_base + s;
        float *Kl = mr ? mr[s].seq->K[layer] : m->K[layer], *Vl = mr ? mr[s].seq->V[layer] : m->V[layer];
        memcpy(Kl + ((int64_t)hh*m->max_t + t)*hd, k + (int64_t)s*D + hh*hd, hd*sizeof(float));
        memcpy(Vl + ((int64_t)hh*m->max_t + t)*hd, vv + (int64_t)s*D + hh*hd, hd*sizeof(float));
    }
    int Tk = pos_base + S;             /* numero di key totali disponibili */
    float scale = 1.f / sqrtf((float)hd);
    float *ctx = falloc((int64_t)S*D);
    #pragma omp parallel for collapse(2) schedule(static)
    for (int hh = 0; hh < H; hh++) {
        for (int s = 0; s < S; s++) {
            int qpos = mr ? mr[s].pos : pos_base + s;
            const float *Kl = mr ? mr[s].seq->K[layer] : m->K[layer], *Vl = mr ? mr[s].seq->V[layer] : m->V[layer];
            const float *qv = q + (int64_t)s*D + hh*hd;
            float sc[4096];
            for (int t = 0; t <= qpos; t++) {          /* causale: t <= qpos */
                const float *kv = Kl + ((int64_t)hh*m->max_t + t)*hd;
                float acc = dot_f32_lanes(qv, kv, hd);
                sc[t] = acc * scale;
            }
            softmax_row(sc, qpos+1);
            float *cx = ctx + (int64_t)s*D + hh*hd;
            for (int dd = 0; dd < hd; dd++) cx[dd] = 0;
            for (int t = 0; t <= qpos; t++) {
                const float *vrow = Vl + ((int64_t)hh*m->max_t + t)*hd;
                float a = sc[t];
                for (int dd = 0; dd < hd; dd++) cx[dd] += a * vrow[dd];
            }
        }
    }
    (void)Tk;
    MATMUL_RES(out, ctx, l->o, l->vk_o, S, D, D);
    free(q); free(k); free(vv); free(ctx);
}

/* One row's routing: the momentum the PILOT prefetcher reads, softmax, the top-K
 * ids and gates (idx[K], val[K]), the optional renormalisation, the counts and
 * the trace. pr is the row's router logits, turned into probabilities in place. */
static void moe_route_row(Model *m, int layer, int s, float *pr, int *idx, float *val) {
    Cfg *c = &m->c; int E = c->n_experts, K = c->topk;
    if (m->momentum_logits && m->pilot_smooth > 0.f) {
        float *ema = m->momentum_logits + (int64_t)layer * E;
        int is_zero = 1;
        for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } }
        if (is_zero) {
            for (int e = 0; e < E; e++) ema[e] = pr[e];
        } else {
            for (int e = 0; e < E; e++) {
                ema[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e];
            }
        }
    }

    softmax_row(pr, E);
    /* top-K indici (selezione parziale) */
    for (int kk = 0; kk < K; kk++) {
        int best = -1; float bv = -1e30f;
        for (int e = 0; e < E; e++) {
            int taken = 0; for (int j = 0; j < kk; j++) if (idx[j]==e){taken=1;break;}
            if (!taken && pr[e] > bv) { bv = pr[e]; best = e; }
        }
        /* SEC: all-NaN probabilities leave best at -1, which reaches
         * expert_get() and then last_access[layer*E - 1] -- a heap write at
         * a negative index. See rt_router_pick in route_trace.h. */
        best = rt_router_pick(best, kk, E, layer);
        idx[kk] = best; val[kk] = pr[best];
    }
    if (c->norm_topk) { float sm=0; for(int kk=0;kk<K;kk++) sm+=val[kk]; for(int kk=0;kk<K;kk++) val[kk]/=sm; }
    /* IMPROVEMENT 2 activation heatmap AND the ROUTE_TRACE stream, in one
     * call. The counters were the only thing this engine recorded, and it
     * recorded them HERE, before pinning activates — rt_count keeps that
     * placement exactly. The trace is the half olmoe never had: it emits a
     * line per (moe call, position, layer), so tools/route_pairs.py,
     * route_coupling_report.py and residency_sim.py can read this engine's
     * routing the same way they read GLM's. Until now olmoe announced
     * ROUTE_TRACE at startup and then wrote a zero-byte file, because
     * rt_init() opens the stream but nothing here ever called rt_trace():
     * every consumer silently saw "no data" instead of an error.
     *
     * Only rt_route() is unconditional: it is a no-op for the counts when
     * this engine has no counter row (the !hot_pinned guard below is
     * unchanged) and a no-op for the trace when ROUTE_TRACE is unset, so a
     * run without the variable behaves exactly as before. Measurement only,
     * never the computation: idx[] and val[] are the ids and the
     * post-normalisation gates the layer is about to apply. */
    if (!m->hot_pinned) rt_route(layer, s, idx, val, K);
}

#if defined(__AVX2__)
/* FUSED3: same contract as matmul_q's IDOT fast branch (IDOT env,
 * dims %16==0, <=4096) — outside it the stock calls run unchanged.
 * Exact integer arithmetic only: bit-identical output (verified by memcmp
 * in tests/bench_fused3.c). OFF by default. */
static int moe_fused3(int D, int I) {
    static int idot_moe = -1;
    if (idot_moe < 0) { const char *ie = getenv("IDOT"); idot_moe = !(ie && *ie == '0'); }
    return g_fused3 && idot_moe && D % 16 == 0 && D <= 4096 && I % 16 == 0 && I <= 4096;
}
#endif
/* One routed expert on one row: hh[D] = down(silu(gate(xs)) * up(xs)); g, u are
 * scratch of I floats. */
static void moe_expert_row(const Model *m, const Slot *e, const float *xs, float *g, float *u, float *hh) {
    const Cfg *c = &m->c; int D = c->hidden, I = c->inter;
#if defined(__AVX2__)
    if (moe_fused3(D, I)) {
        matmul_q_idot_pair_v3(g, u, xs, e->g, e->gs, e->u, e->us, D, I);   /* gate+up share one quant of xs */
        for (int i = 0; i < I; i++) { float gv = g[i]; g[i] = (gv / (1.f + expf(-gv))) * u[i]; }
        matmul_q_idot_v3(hh, g, e->d, e->ds, I, D);                        /* down_proj [D,I] */
    } else
#endif
    {
    matmul_q(g, xs, e->g, e->gs, D, I);     /* gate_proj [I,D] */
    matmul_q(u, xs, e->u, e->us, D, I);     /* up_proj   [I,D] */
    for (int i = 0; i < I; i++) { float gv = g[i]; g[i] = (gv / (1.f + expf(-gv))) * u[i]; }
    matmul_q(hh, g, e->d, e->ds, I, D);     /* down_proj [D,I] */
    }
}
/* moe_expert_row on n rows of one expert: hh[t] = expert(xs[t]), the same bits;
 * g[t], u[t] are scratch of I floats each. */
static void moe_expert_rows(const Model *m, const Slot *e, const float *const *xs, int n,
                            float *const *g, float *const *u, float *const *hh) {
    const Cfg *c = &m->c; int D = c->hidden, I = c->inter;
#if defined(__AVX2__)
    if (moe_fused3(D, I)) {
        for (int t = 0; t < n; t++) moe_expert_row(m, e, xs[t], g[t], u[t], hh[t]);
        return;
    }
#endif
    matmul_q_rows(g, xs, n, e->g, e->gs, D, I);     /* gate_proj [I,D] */
    matmul_q_rows(u, xs, n, e->u, e->us, D, I);     /* up_proj   [I,D] */
    for (int t = 0; t < n; t++) {
        float *gt = g[t]; const float *ut = u[t];
        for (int i = 0; i < I; i++) { float gv = gt[i]; gt[i] = (gv / (1.f + expf(-gv))) * ut[i]; }
    }
    matmul_q_rows(hh, (const float *const *)g, n, e->d, e->ds, I, D);   /* down_proj [D,I] */
}

/* A prompt's MoE, expert by expert (S > 1). Per block of OLMOE_PREFILL_ROWS rows:
 * the rows are routed in order, as the one-row loop routes them; their (row, rank)
 * pairs are grouped by expert with a counting sort; each expert is fetched once
 * and multiplies all of its rows (moe_expert_rows) into a row of its own per pair;
 * then every rank joins its row in rank order, as the one-row loop adds them. So
 * each expert's weights are read once a block instead of once a row, and the sum's
 * order, hence the bits, are the one-row loop's. The block's other pairs on an
 * expert count as hits: they are served by the slot the fetch returned. */
#define OLMOE_PREFILL_ROWS 128
static void moe_by_expert(Model *m, int layer, const float *x, int S, float *logits, float *out) {
    Cfg *c = &m->c; int D = c->hidden, E = c->n_experts, K = c->topk, I = c->inter;
    int B = S < OLMOE_PREFILL_ROWS ? S : OLMOE_PREFILL_ROWS, P = B * K;
    int *idx = malloc((size_t)P * sizeof(int)), *pair = malloc((size_t)P * sizeof(int));
    int *at = malloc(((size_t)E + 1) * sizeof(int));
    float *val = malloc((size_t)P * sizeof(float));
    const float **xs = malloc((size_t)B * sizeof(*xs));
    float **gp = malloc((size_t)B * sizeof(*gp)), **up = malloc((size_t)B * sizeof(*up));
    float **hp = malloc((size_t)B * sizeof(*hp));
    float *g = falloc((int64_t)B * I), *u = falloc((int64_t)B * I), *ctb = falloc((int64_t)P * D);
    if (!idx || !pair || !at || !val || !xs || !gp || !up || !hp) { fprintf(stderr, "OOM moe_by_expert\n"); exit(1); }
    for (int t = 0; t < B; t++) { gp[t] = g + (int64_t)t * I; up[t] = u + (int64_t)t * I; }
    for (int s0 = 0; s0 < S; s0 += B) {
        int rows = S - s0 < B ? S - s0 : B, n = rows * K;
        for (int s = 0; s < rows; s++)
            moe_route_row(m, layer, s0 + s, logits + (int64_t)(s0 + s) * E, idx + (int64_t)s * K, val + (int64_t)s * K);
        memset(at, 0, ((size_t)E + 1) * sizeof(int));
        for (int i = 0; i < n; i++) at[idx[i] + 1]++;
        for (int e = 0; e < E; e++) at[e + 1] += at[e];
        for (int i = 0; i < n; i++) pair[at[idx[i]]++] = i;   /* at[e] ends at expert e's end */
        for (int e = 0, a = 0; e < E; a = at[e++]) {
            int cnt = at[e] - a;                               /* a row routes an expert once: cnt <= rows */
            if (!cnt) continue;
            for (int t = 0; t < cnt; t++) {
                int i = pair[a + t];
                xs[t] = x + (int64_t)(s0 + i / K) * D;
                hp[t] = ctb + (int64_t)i * D;
            }
            Slot *sl; expert_get(m, layer, e, &sl);
            moe_expert_rows(m, sl, xs, cnt, gp, up, hp);
            expert_put(sl);
            if (cnt > 1) { pthread_mutex_lock(&g_pilot_mx); m->hits += cnt - 1; pthread_mutex_unlock(&g_pilot_mx); }
        }
        for (int s = 0; s < rows; s++) {
            float *os = out + (int64_t)(s0 + s) * D;
            for (int kk = 0; kk < K; kk++) {
                const float *hh = ctb + ((int64_t)s * K + kk) * D;
                float w = val[s * K + kk];
                for (int d = 0; d < D; d++) os[d] += w * hh[d];
            }
        }
    }
    free(idx); free(pair); free(at); free(val); free(xs); free(gp); free(up); free(hp);
    free(g); free(u); free(ctb);
}

#ifdef COLI_VULKAN
/* ---- the Vulkan routed-expert tier (vk_tier.c) ---------------------------------
 * With the tier on, a MoE layer routes every row first, then runs per block of
 * rows: the block's resident experts go to the device as one batch (vkt_issue),
 * the CPU computes the other (row, rank) pairs into rows of their own with the
 * kernel moe() uses, then every rank of every row joins `out` in rank order, the
 * device's and the CPU's alike, as moe() adds them; so the order of the sum never
 * depends on which experts were resident. Every expert the CPU computed passes its
 * RAM bytes to the tier (vkt_note), which may promote it. */
/* The CPU's pairs of a block (want[i] set) into ctb[i]. The PILOT worker reloads
 * slots off the lock, so a slot is handed to the tier only under the lock and only
 * while it still holds that expert: a worker's read into it starts with a reserve
 * under the same lock, which changes its id. */
static void moe_vk_cpu(Model *m, int layer, const float *x, int n, const int *ib,
                       const uint8_t *want, float *ctb, float *g, float *u) {
    int D = m->c.hidden, K = m->c.topk;
    for (int i = 0; i < n; i++) {
        if (!want[i]) continue;
        Slot *e; expert_get(m, layer, ib[i], &e);
        moe_expert_row(m, e, x + (int64_t)(i / K) * D, g, u, ctb + (int64_t)i * D);
        pthread_mutex_lock(&g_pilot_mx);
        if (e->eid == ib[i]) { VktExpertSrc vs = olmoe_vk_src(e); vkt_note(layer, ib[i], &vs); }
        e->busy--;   /* expert_put, under the lock already held */
        pthread_mutex_unlock(&g_pilot_mx);
    }
}
static void moe_vk_run(Model *m, int layer, const float *x, int S, float *logits, float *out) {
    Cfg *c = &m->c; int D = c->hidden, E = c->n_experts, K = c->topk, I = c->inter;
    int *idx = malloc((size_t)S * K * sizeof(int));
    float *val = malloc((size_t)S * K * sizeof(float));
    int B = vkt_step_rows(S, OLMOE_VK_ROWS);   /* a whole prompt chunk when the tier streams */
    float *ctb = falloc((int64_t)B * K * D), *g = falloc(I), *u = falloc(I);
    uint8_t *taken = malloc((size_t)B * K), *want = malloc((size_t)B * K);
    const float **dev = malloc((size_t)B * K * sizeof(*dev));
    if (!idx || !val || !taken || !want || !dev) { fprintf(stderr, "OOM moe_vk_run\n"); exit(1); }
    for (int s = 0; s < S; s++) moe_route_row(m, layer, s, logits + (int64_t)s * E, idx + (int64_t)s * K, val + (int64_t)s * K);
    for (int s0 = 0; s0 < S; s0 += B) {
        int rows = S - s0 < B ? S - s0 : B, n = rows * K;
        const float *xb = x + (int64_t)s0 * D;
        const int *ib = idx + (int64_t)s0 * K;
        const float *vb = val + (int64_t)s0 * K;
        int ndev = vkt_issue(layer, xb, rows, K, ib, taken);
        pthread_mutex_lock(&g_pilot_mx);   /* HITS for the device's pairs, as expert_get marks the CPU's */
        for (int i = 0; i < n; i++) { want[i] = !taken[i]; if (taken[i]) ehit_mark(m, layer, ib[i]); }
        pthread_mutex_unlock(&g_pilot_mx);
        moe_vk_cpu(m, layer, xb, n, ib, want, ctb, g, u);
        if (ndev && !vkt_join(dev)) {   /* the batch failed (the tier stops): those pairs here */
            moe_vk_cpu(m, layer, xb, n, ib, taken, ctb, g, u);
            memset(taken, 0, (size_t)n);
        }
        for (int s = 0; s < rows; s++) {
            float *os = out + (int64_t)(s0 + s) * D;
            for (int kk = 0; kk < K; kk++) {
                int i = s * K + kk;
                const float *hh = taken[i] ? dev[i] : ctb + (int64_t)i * D;
                float w = vb[i];
                for (int d = 0; d < D; d++) os[d] += w * hh[d];
            }
        }
    }
    free(idx); free(val); free(ctb); free(g); free(u); free(taken); free(want); free(dev);
}
#endif

/* MoE sui token x[S,hidden] -> out[S,hidden], from the router's logits[S,E] (turned
 * into probabilities in place). moe() computes the logits first; the dense chain
 * (olmoe_chain.h) hands over the ones the device computed. */
static void moe_routed(Model *m, int layer, float *x, int S, float *logits, float *out) {
    Cfg *c = &m->c; int D = c->hidden, E = c->n_experts, K = c->topk, I = c->inter;
    memset(out, 0, (int64_t)S*D*sizeof(float));
#ifdef COLI_VULKAN
    if (vkt_ready()) {   /* the Vulkan expert tier (moe_vk_run) */
        moe_vk_run(m, layer, x, S, logits, out);
        rt_trace_end();
        return;
    }
#endif
    if (S > 1) {   /* a prompt: expert by expert, the same bits as the loop below */
        moe_by_expert(m, layer, x, S, logits, out);
        rt_trace_end();
        return;
    }
    float *g = falloc(I), *u = falloc(I), *hh = falloc(D);
    for (int s = 0; s < S; s++) {
        int idx[64]; float val[64];
        moe_route_row(m, layer, s, logits + (int64_t)s*E, idx, val);
        const float *xs = x + (int64_t)s*D;
        for (int kk = 0; kk < K; kk++) {
            Slot *e; expert_get(m, layer, idx[kk], &e);
            moe_expert_row(m, e, xs, g, u, hh);
            expert_put(e);
            float w = val[kk];
            float *os = out + (int64_t)s*D;
            for (int d = 0; d < D; d++) os[d] += w * hh[d];
        }
    }
    free(g); free(u); free(hh);
    /* Advance the trace call counter: once per moe() invocation, after all of
     * its rows are traced. rt_trace_end() is a no-op when no stream is open.
     *
     * Outside the row loop on purpose. A batch of S == 0 traces no rows and
     * must still consume a call id, or the ids stop being consecutive and
     * residency_sim.py rejects the trace outright ("trace lacks advancing GLM
     * call ids") rather than merging two forwards into one position space. GLM
     * and glm53 advance theirs the same way. */
    rt_trace_end();
}
static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
    Cfg *c = &m->c; int D = c->hidden, E = c->n_experts;
    float *logits = falloc((int64_t)S*E);
    MATMUL_RES(logits, x, l->gate, l->vk_gate, S, D, E);
    moe_routed(m, layer, x, S, logits, out);
    free(logits);
}

/* PROF phases (#1449): wall time in attention, in the MoE blocks (expert
 * loads included) and in the head, plus positions forwarded. The serve loop
 * reports per-turn deltas; expert_matmul_s is the MoE time minus the disk
 * seconds measured inside it, clamped at zero; forwards counts step() calls,
 * so tokens per forward reads 1.0 for this engine. Timing only. */
static double g_prof_attn_s = 0.0, g_prof_moe_s = 0.0, g_prof_head_s = 0.0;
static long long g_prof_forwards = 0;

static void layers_forward_range(Model *m, float *x, int S, int pos_base,
                                 int layer_begin, int layer_end,
                                 int allow_prefetch) {
    Cfg *c = &m->c;
    int D = c->hidden;
    float *nrm = falloc((int64_t)S*D), *tmp = falloc((int64_t)S*D);
    for (int i = layer_begin; i < layer_end; i++) {
        Layer *l = &m->L[i];
        for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->in_ln, D, c->eps);
        double t_attn = now_s();
        attention(m, l, i, nrm, S, pos_base, tmp);
        g_prof_attn_s += now_s() - t_attn;
        for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j];
        /* IMPROVEMENT 1: PILOT=1 -> 1-layer lookahead */
        if (allow_prefetch && g_pilot >= 1 && S <= 8 && i + 1 < c->n_layers)
            pilot_prefetch(m, i + 1, x, S);
        for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->post_ln, D, c->eps);
        double t_moe = now_s();
        moe(m, l, i, nrm, S, tmp);
        g_prof_moe_s += now_s() - t_moe;
        for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j];

        /* PREDICTION IMPROVEMENT C (Residual gate trick):
         * PILOT=2 -> prefetch layer i+2 using completed state x (containing MoE residual). */
        if (allow_prefetch && g_pilot >= 2 && S <= 8 && i + 2 < c->n_layers)
            pilot_prefetch(m, i + 2, x, S);
        if (allow_prefetch && g_pilot >= 3 && S <= 8 && i + 3 < c->n_layers)
            pilot_prefetch(m, i + 3, x, S);
        
    }
    free(nrm); free(tmp);
}

/* Canale logprobs: coda numerica per token, lettura del prefill, fotografia
 * dello stato. Questo motore e ad attenzione pura, quindi riavvolgere vuol
 * dire solo dichiarare che il prefisso tenuto e quello fotografato: le righe
 * KV di quelle posizioni non le ha toccate nessuno. Si salvano gli id e il
 * vettore di logit finale, che e il predittore del primo token fresco. */
static int    g_echo_k = 0;
static const char *g_echo_id = NULL;
static ColiPinPool g_pins;             /* piu scatti annidati, vedi pin_pool.h */
static const float *g_pin_logit = NULL; /* logit dello scatto rimesso, se c'e */
static int    g_pin_use_logit = 0;
static Tok   *g_echo_tok = NULL;

static void olmoe_echo(const char *id, int pos, int token, const float *lo, int V, int k){
    char tail[1024]; coli_logprob_tail(tail, sizeof tail, lo, V, token, k);
    char piece[512]; int n = g_echo_tok ? tok_decode(g_echo_tok, &token, 1, piece, (int)sizeof piece) : 0;
    if (n < 0) n = 0;
    printf("ECHO %s %d %d%s\n", id, n, pos, tail);
    if (n > 0) fwrite(piece, 1, (size_t)n, stdout);
    fputc('\n', stdout); fflush(stdout);
}

#ifdef COLI_VULKAN
#include "olmoe_chain.h"   /* COLI_VK_CHAIN: every layer's dense chain on the device */

/* ---- dense weights on the device only (COLI_VK_DENSE_HOST) ------------------------
 * With the dense part on the device (the chain, or COLI_VK_DENSE), the attention's
 * q/k/v/o, the router and lm_head go up at start and their f32 host copies are given
 * back; matmul_res then runs them on the device whatever COLI_VK_DENSE says. The CPU
 * multiplies by one only after a lost device: matmul_res first reads it back here
 * (load_t: the same f32 bytes) and keeps it from there on. What keeps its host copy:
 * the embedding (its rows are gathered on the CPU) and the norms. The chain is decided
 * after the tier (olc_start); its decision is taken here silently first, from the same
 * inputs, so the matrices are placed before the tier sizes its budget. */
typedef struct { void **vk; float **field; int64_t n, rows, columns; char name[96]; } OlmDho;
static OlmDho *g_olm_dho; static int g_olm_dho_n, g_olm_dho_cap;
static Model *g_olm_dho_model;
static pthread_mutex_t g_olm_dho_mx = PTHREAD_MUTEX_INITIALIZER;
static float *olm_dho_reload(void **vk) {
    pthread_mutex_lock(&g_olm_dho_mx);
    OlmDho *e = NULL;
    for (int i = 0; i < g_olm_dho_n && !e; i++) if (g_olm_dho[i].vk == vk) e = &g_olm_dho[i];
    if (!e || !g_olm_dho_model) { fprintf(stderr, "[VK] olmoe: a dense matrix the device held alone cannot be read back\n"); exit(1); }
    if (!*e->field) {
        *e->field = load_t(g_olm_dho_model, e->name, e->rows, e->columns);
        coli_vk_dense_host_reloaded((size_t)e->n * sizeof(float));
    }
    float *w = *e->field;
    pthread_mutex_unlock(&g_olm_dho_mx);
    return w;
}
static void olm_dho_drop(Model *m, float **field, void **vk, const char *name, int I, int O, size_t *bytes) {
    if (!*field || *vk == (void *)&g_vk_refused) return;
    if (!*vk && !coli_vk_tensor_ensure((ColiVkTensor **)vk, *field, NULL, 10, I, O, 0)) { *vk = &g_vk_refused; return; }
    if (g_olm_dho_n == g_olm_dho_cap) {
        g_olm_dho_cap = g_olm_dho_cap ? 2 * g_olm_dho_cap : 64;
        g_olm_dho = realloc(g_olm_dho, (size_t)g_olm_dho_cap * sizeof *g_olm_dho);
        if (!g_olm_dho) { fprintf(stderr, "OOM dense matrix table\n"); exit(1); }
    }
    OlmDho *e = &g_olm_dho[g_olm_dho_n++];
    e->vk = vk; e->field = field; e->n = (int64_t)I * O; e->rows = O; e->columns = I;
    snprintf(e->name, sizeof e->name, "%s", name);
    free(*field); *field = NULL;
    size_t b = (size_t)I * O * sizeof(float);
    coli_vk_dense_host_dropped(b);
    *bytes += b;
    (void)m;
}
/* An automatic cache (cap <= 0) sized with the dense weights in RAM: give the experts
 * what they held. With RAM_GB the process's own resident set shrinks by what was
 * dropped; without it, what the OS offers now is read again (on a discrete GPU it grew
 * by that much; on a device sharing the RAM the device copy took it back). Slots fill
 * lazily, so a layer's cache grows by zeroed slots. */
static void olm_dho_grow_cap(Model *m, size_t dropped) {
    OlmAutoCap *a = &g_olm_auto_cap;
    Cfg *c = &m->c;
    if (!a->on || !m->cache || a->slot_gb <= 0.0) return;
    double resident = a->resident - dropped / 1e9;
    double budget = a->ram_arg > 0.0 ? a->ram_arg : resident + mem_available_gb() * 0.88;
    double room = budget - resident - 0.5;
    /* the room the cache shares with the KV grows with it: kv_room_fit takes slots
     * back from the new size as positions are written */
    if (room > 0.0 && (int64_t)(room * 1e9) > m->room_bytes) m->room_bytes = (int64_t)(room * 1e9);
    int derived = room > 0.0 ? (int)(room / a->slot_gb / (double)a->layers) : 1;
    if (derived < 1) derived = 1;
    if (derived > c->n_experts) derived = c->n_experts;
    int was = m->cache[0].cap;
    if (derived <= was) return;
    pthread_mutex_lock(&g_pilot_mx);
    for (int i = 0; i < c->n_layers; i++) {
        LCache *lc = &m->cache[i];
        if (!lc->slots) continue;
        Slot *s = realloc(lc->slots, (size_t)derived * sizeof(Slot));
        if (!s) { fprintf(stderr, "OOM growing the expert cache\n"); exit(1); }
        memset(s + lc->cap, 0, (size_t)(derived - lc->cap) * sizeof(Slot));
        lc->slots = s; lc->cap = derived;
    }
    pthread_mutex_unlock(&g_pilot_mx);
    fprintf(stderr, "[cache] %d slots/layer of %d experts (was %d): the dense weights on the device only gave "
                    "%.2f GB back, %.2f GB dense resident now\n", derived, c->n_experts, was, dropped / 1e9, resident);
}
/* Layer i's five matrices (i = -1: lm_head) uploaded if they are not yet, their host
 * copies given back; the bytes given back. */
static size_t g_olm_dho_dropped;
static size_t olm_dho_layer(Model *m, int i) {
    Cfg *c = &m->c; int D = c->hidden, E = c->n_experts;
    size_t dropped = 0;
    char nm[96];
    if (i < 0) olm_dho_drop(m, &m->lm_head, &m->vk_lm_head, "lm_head.weight", D, c->vocab, &dropped);
    else {
        Layer *l = &m->L[i];
        snprintf(nm, sizeof nm, "model.layers.%d.self_attn.q_proj.weight", i); olm_dho_drop(m, &l->q, &l->vk_q, nm, D, D, &dropped);
        snprintf(nm, sizeof nm, "model.layers.%d.self_attn.k_proj.weight", i); olm_dho_drop(m, &l->k, &l->vk_k, nm, D, D, &dropped);
        snprintf(nm, sizeof nm, "model.layers.%d.self_attn.v_proj.weight", i); olm_dho_drop(m, &l->v, &l->vk_v, nm, D, D, &dropped);
        snprintf(nm, sizeof nm, "model.layers.%d.self_attn.o_proj.weight", i); olm_dho_drop(m, &l->o, &l->vk_o, nm, D, D, &dropped);
        snprintf(nm, sizeof nm, "model.layers.%d.mlp.gate.weight", i); olm_dho_drop(m, &l->gate, &l->vk_gate, nm, D, E, &dropped);
    }
    g_olm_dho_dropped += dropped;
    return dropped;
}
/* With the fit of a partial chain (olc_fit_start) only the N layers' matrices (and the
 * head's when it goes up) count: the CPU's are refused. The chain's setup then drops each
 * layer's host copies once the layer is on the device (olc_place), and olm_dho_finish
 * says what was given back and grows an automatic cache by it. */
static void olm_dho_start(Model *m) {
    Cfg *c = &m->c;
    int D = c->hidden, E = c->n_experts;
    size_t bytes = 0;
    for (int i = 0; i < c->n_layers; i++)
        if (m->L[i].vk_q != (void *)&g_vk_refused) bytes += (size_t)(4 * (int64_t)D * D + (int64_t)E * D) * sizeof(float);
    if (m->vk_lm_head != (void *)&g_vk_refused) bytes += (size_t)c->vocab * (size_t)D * sizeof(float);
    int chain = coli_vk_chain_decide(NULL, vkt_wanted() && c->n_experts > 0, OLMOE_CHAIN_IGPU);
    if (g_olc_fit_on) chain = chain && g_olc_fit.n > 0;
    if (!coli_vk_dense_host_decide("olmoe", (chain || coli_vk_dense()) && m->lm_head && bytes, bytes)) return;
    g_olm_dho_model = m;
    if (g_olc_fit_on && chain) return;   /* olc_place drops them layer by layer, olm_dho_finish reports */
    for (int i = 0; i < c->n_layers; i++) olm_dho_layer(m, i);
    olm_dho_layer(m, -1);
    coli_vk_dense_host_placed("olmoe", "the embedding (its rows are gathered on the CPU), norms");
    olm_dho_grow_cap(m, g_olm_dho_dropped);
}
static void olm_dho_finish(Model *m) {
    if (!g_olc_fit_on || !g_olc_fit.n || !coli_vk_dense_device_only()) return;
    coli_vk_dense_host_layers(g_olc_fit.n, g_olc_fit.L);
    coli_vk_dense_host_placed("olmoe", "the embedding (its rows are gathered on the CPU), norms");
    olm_dho_grow_cap(m, g_olm_dho_dropped);
}
#endif

static float *step(Model *m, const int *ids, int S, int pos_base) {
    Cfg *c = &m->c; int D = c->hidden;
    kv_room_fit(m, pos_base + S);   /* before the positions' pages are written */
    if (g_pilot && m->token_count > 0) {
        /* Flush stale prefetch requests: clear is_queued so pilot_realload
         * will skip any entries still sitting in pilot_q for the previous
         * token.  We deliberately do NOT move pilot_w backwards; that would
         * break the ring-buffer invariant (pilot_r could exceed pilot_w if
         * the worker consumed an entry concurrently).  The worker will drain
         * the stale slots harmlessly because pilot_realload already exits
         * early when the expert is already cached or is_queued is clear. */
        pthread_mutex_lock(&g_pilot_mx);
        memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts);
        pthread_mutex_unlock(&g_pilot_mx);
    }
    float *x = falloc((int64_t)S*D);
    for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float));
#ifdef COLI_VULKAN
    /* COLI_VK_CHAIN: the layers, the final norm and lm_head on the device; x comes
     * back only when the prefill read-out below needs every row */
    float *chain_logit = NULL;
    int chain_n = 0;   /* layers the chain ran (a partial chain: the CPU runs the rest and the head) */
    if (g_vk_chain) {
        int rows_only = 0;
        chain_logit = falloc(c->vocab);
        chain_n = olc_forward(m, x, S, pos_base, g_echo_k > 0 && g_echo_id && S > 1, chain_logit, &rows_only);
        if (!chain_n) {
            free(chain_logit); chain_logit = NULL;
            olc_cpu_step(m, pos_base);
            /* the primary's layers may have run before the second device's were lost: their
             * residual is in x, and the CPU redoes the whole step from the embedding */
            for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float));
        } else if (rows_only) { free(chain_logit); chain_logit = NULL; }
    }
    if (!chain_logit) layers_forward_range(m, x, S, pos_base, chain_n, c->n_layers, 1);
#else
    layers_forward_range(m, x, S, pos_base, 0, c->n_layers, 1);
#endif
    /* count actual tokens processed (S>1 during prefill) */
    m->token_count += S; m->freq_token_count += S;
    if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens)
        pin_hot_experts(m);
    m->kv_len = pos_base + S;
    /* Recorded HERE, where the tokens entered the cache: fed[0..len-1] are
     * the ids those positions were built from, and that invariant is the
     * whole safety argument for reusing them next turn. */
    kv_prefix_record(&m->kvp, ids, pos_base, S);
    /* Lettura del prefill: un passaggio di lm_head per posizione, pagato solo
     * da chi ha chiesto il canale. La posizione p predice il token p+1; il
     * primo token fresco e predetto dalla fotografia. Cosi ogni token di
     * un'opzione ha il suo logprob, anche fuori dai primi k. */
    if (g_echo_k > 0 && g_echo_id && S > 0) {
        float *erow = falloc(D), *elog = falloc(c->vocab);
        if (g_pin_use_logit && g_pin_logit)
            olmoe_echo(g_echo_id, pos_base, ids[0], g_pin_logit, c->vocab, g_echo_k);
        for (int p = 0; p + 1 < S; p++) {
            rmsnorm_row(erow, x + (int64_t)p*D, m->final_norm, D, c->eps);
            MATMUL_RES(elog, erow, m->lm_head, m->vk_lm_head, 1, D, c->vocab);
            olmoe_echo(g_echo_id, pos_base + p + 1, ids[p+1], elog, c->vocab, g_echo_k);
        }
        free(erow); free(elog);
    }
#ifdef COLI_VULKAN
    if (chain_logit) {   /* the chain's last frame ran the final norm and lm_head */
        g_prof_forwards += 1;
        free(x);
        return chain_logit;
    }
#endif
    float *last = falloc(D);
    rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps);
    float *logit = falloc(c->vocab);
    double t_head = now_s();
    MATMUL_RES(logit, last, m->lm_head, m->vk_lm_head, 1, D, c->vocab);
    g_prof_head_s += now_s() - t_head;
    g_prof_forwards += 1;   /* forward passes, not positions: a prefill of S rows is one */
    free(x); free(last);
    return logit;
}

/* ---- several conversations at once (KV_SLOTS>1, serve_mux) ------------------- */
static void olm_seq_swap(Model *m, OlmSeq *q) {
    float **K = m->K, **V = m->V; int len = m->kv_len; kv_prefix p = m->kvp;
    m->K = q->K; m->V = q->V; m->kv_len = q->kv_len; m->kvp = q->kvp;
    q->K = K; q->V = V; q->kv_len = len; q->kvp = p;
}

/* One decode step of several conversations: row s is the token ids[s] at
 * rows[s].pos of the conversation rows[s].seq, every conversation parked. The
 * matrices, the routed experts and lm_head run once over the S rows; the attention
 * reads and writes each row's own KV (m->mux_rows). The CPU kernels give a row the
 * same bits whatever S is, so each conversation gets the logits it would alone. */
static float *olm_step_rows(Model *m, const OlmRow *rows, const int *ids, int S) {
    Cfg *c = &m->c; int D = c->hidden;
    /* The KV and the expert cache share one room (kv_room_fit): the positions every
     * conversation's KV reaches, summed, before this step's pages are written. */
    for (int s = 0; s < S; s++) if (rows[s].pos + 1 > rows[s].seq->kv_hi) rows[s].seq->kv_hi = rows[s].pos + 1;
    int reach = 0;
    for (int i = 0; i < g_olm_mux_slots; i++) reach += g_olm_mux_seq[i].kv_hi;
    kv_room_fit(m, reach);
    if (g_pilot && m->token_count > 0) {
        pthread_mutex_lock(&g_pilot_mx);
        memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts);
        pthread_mutex_unlock(&g_pilot_mx);
    }
    float *x = falloc((int64_t)S*D);
    for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float));
    m->mux_rows = rows;
    layers_forward_range(m, x, S, 0, 0, c->n_layers, 1);
    m->mux_rows = NULL;
    m->token_count += S; m->freq_token_count += S;
    if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens)
        pin_hot_experts(m);
    for (int s = 0; s < S; s++) {
        OlmSeq *q = rows[s].seq;
        q->kv_len = rows[s].pos + 1;
        kv_prefix_record(&q->kvp, ids + s, rows[s].pos, 1);
    }
    float *last = falloc((int64_t)S*D), *logit = falloc((int64_t)S * c->vocab);
    for (int s = 0; s < S; s++) rmsnorm_row(last + (int64_t)s*D, x + (int64_t)s*D, m->final_norm, D, c->eps);
    double t_head = now_s();
    MATMUL_RES(logit, last, m->lm_head, m->vk_lm_head, S, D, c->vocab);
    g_prof_head_s += now_s() - t_head;
    g_prof_forwards += 1;
    free(x); free(last);
    return logit;
}

static void pilot_realload(Model *m, int layer, int eid) {
    LCache *lc = &m->cache[layer];
    Cfg *c = &m->c;

    pthread_mutex_lock(&g_pilot_mx);
    /* Early-exit if entry was flushed (is_queued cleared) while waiting. */
    if (!m->is_queued[layer * c->n_experts + eid]) {
        pthread_mutex_unlock(&g_pilot_mx);
        return;
    }
    if (slot_indexed(m, layer, eid) || slot_in_flight(m, layer, eid)) {
        m->is_queued[layer * c->n_experts + eid] = 0;
        pthread_mutex_unlock(&g_pilot_mx);
        return;
    }
    Slot *s;
    if (lc->n < lc->cap) {
        s = &lc->slots[lc->n++];
        slot_ensure_allocated(m, s);
    } else {
        /* LRU eviction: skip pinned, in-flight (eid==-1) and busy slots; an expert the
         * Vulkan tier holds goes first (vkt_ram_first: no second copy when RAM is short) */
        int lru = olmoe_victim(lc, layer);
        if (lru < 0) {
            m->is_queued[layer * c->n_experts + eid] = 0;
            pthread_mutex_unlock(&g_pilot_mx);
            return; /* all pinned/in-flight/busy, skip */
        }

        /* LFRU eviction guard: don't displace a warm resident expert with a speculation */
        if (g_pilot_evict_guard && m->freq && m->freq[layer] && m->last_access &&
            lc->slots[lru].eid >= 0 && !vkt_ram_first(layer, lc->slots[lru].eid)) {
            int vid = lc->slots[lru].eid;
            uint64_t vs = lfru_score(m->freq[layer][vid], m->last_access[layer * c->n_experts + vid], m->clock);
            uint64_t cs = lfru_score(m->freq[layer][eid], m->last_access[layer * c->n_experts + eid], m->clock);
            if (cs <= vs + (vs >> 2) + (4u << 8)) {
                m->is_queued[layer * c->n_experts + eid] = 0;
                pthread_mutex_unlock(&g_pilot_mx);
                return; /* drop speculation */
            }
        }

        s = &lc->slots[lru]; s->pinned = 0;
        if (vkt_ram_first(layer, s->eid)) vkt_ram_gave();
    }
    cache_reserve(m, layer, s, eid); s->used = ++m->clock;
    pthread_mutex_unlock(&g_pilot_mx);

    load_expert_merged(m, layer, eid, s);

    pthread_mutex_lock(&g_pilot_mx);
    cache_publish(m, layer, s, eid);
    s->pinned = m->is_pinned[layer * c->n_experts + eid];
    s->used = ++m->clock;
    if (m->last_access) m->last_access[layer * c->n_experts + eid] = m->clock;
    m->is_queued[layer * c->n_experts + eid] = 0;
    pthread_cond_broadcast(&g_pilot_cv);
    pthread_mutex_unlock(&g_pilot_mx);
}

static void *pilot_worker(void *arg) {
    (void)arg;
    while (1) {
        unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
        unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE);
        if (r == w) {
            sleep_ms(1);
            continue;
        }
        int layer = pilot_q[r & 4095].l;
        int eid = pilot_q[r & 4095].e;
        pilot_realload(pilot_m, layer, eid);
        __atomic_store_n(&pilot_r, r + 1, __ATOMIC_RELEASE);
    }
    return NULL;
}

static void pilot_prefetch(Model *m, int lnext, const float *x, int S) {
    if (lnext < 0 || lnext >= m->c.n_layers) return;
    Cfg *c = &m->c; int D = c->hidden, E = c->n_experts;
    ensure_pilot_worker_started(m);
    float *logits = falloc((int64_t)S * E);
    Layer *l = &m->L[lnext];

    // PREDICTION IMPROVEMENT B: Apply RMSNorm to x using destination layer's post_ln
    // This scales inputs to the distribution expected by l->gate.
    float *nrm_x = falloc((int64_t)S * D);
    for (int s = 0; s < S; s++) {
        rmsnorm_row(nrm_x + (int64_t)s * D, x + (int64_t)s * D, l->post_ln, D, c->eps);
    }

    MATMUL_RES(logits, nrm_x, l->gate, l->vk_gate, S, D, E);
    free(nrm_x);

    for (int s = 0; s < S; s++) {
        float *pr = logits + (int64_t)s * E;

        // PREDICTION IMPROVEMENT A: Apply routing momentum (EMA of gate logits)
        float *blended = pr;
        float *ema = m->momentum_logits + (int64_t)lnext * E;
        if (m->pilot_smooth > 0.f) {
            blended = falloc(E);
            int is_zero = 1;
            for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } }
            if (is_zero) {
                for (int e = 0; e < E; e++) {
                    ema[e] = pr[e];
                    blended[e] = pr[e];
                }
            } else {
                for (int e = 0; e < E; e++) {
                    blended[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e];
                    ema[e] = blended[e]; // update EMA
                }
            }
        }

        int cand = 0;
        int idx[128];

        float max_logit = -1e30f;
        for (int e = 0; e < E; e++) { if (blended[e] > max_logit) max_logit = blended[e]; }
        float *exps = falloc(E);
        float sum_exps = 0.f;
        for (int e = 0; e < E; e++) {
            exps[e] = expf(blended[e] - max_logit);
            sum_exps += exps[e];
        }

        float cum_sum = 0.f;
        int min_cand = c->topk;
        int max_cand = c->topk * g_wide;
        if (max_cand < min_cand) max_cand = min_cand;
        if (max_cand > 128) max_cand = 128; /* idx[] buffer bound */
        if (max_cand > E) max_cand = E;

        for (int kk = 0; kk < max_cand; kk++) {
            int best = -1; float bv = -1.f;
            for (int e = 0; e < E; e++) {
                int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken=1; break; }
                if (!taken && exps[e] > bv) { bv = exps[e]; best = e; }
            }
            if (best < 0) break;
            idx[kk] = best;
            cum_sum += bv;
            cand++;
            if (cum_sum >= m->pilot_conf_limit * sum_exps && cand >= min_cand) {
                break;
            }
        }
        free(exps);

        if (blended != pr) free(blended);

        /* IMPROVEMENT 5: sort candidates by eid for sequential SSD read locality */
        for (int a = 0; a < cand-1; a++)
            for (int b = a+1; b < cand; b++)
                if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; }

        for (int kk = 0; kk < cand; kk++) {
            int eid = idx[kk];
            if (eid < 0) continue;
            int found = 0;
            pthread_mutex_lock(&g_pilot_mx);
            found = slot_indexed(m, lnext, eid) != NULL;
            pthread_mutex_unlock(&g_pilot_mx);
#ifdef COLI_VULKAN
            if (vkt_resident(lnext, eid)) found = 1;   /* the device serves it: nothing to read */
#endif
            if (!found) {
                int gidx = lnext * E + eid;
                pthread_mutex_lock(&g_pilot_mx);
                int already_queued = m->is_queued[gidx];
                if (!already_queued) {
                    m->is_queued[gidx] = 1;
                }
                pthread_mutex_unlock(&g_pilot_mx);

                if (!already_queued) {
                    unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED);
                    unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
                    if (w2 - r2 < 4096) {
                        pilot_q[w2 & 4095].l = lnext;
                        pilot_q[w2 & 4095].e = eid;
                        __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE);
                    } else {
                        pthread_mutex_lock(&g_pilot_mx);
                        m->is_queued[gidx] = 0;
                        pthread_mutex_unlock(&g_pilot_mx);
                    }
                }
            }
        }
    }
    free(logits);
}


/* generazione greedy. prompt[np] -> riempie out[np+n_new] */
static void generate(Model *m, const int *prompt, int np, int n_new, int *out) {
    Cfg *c = &m->c;
    m->max_t = np + n_new;
    m->K = calloc(c->n_layers, sizeof(float*)); m->V = calloc(c->n_layers, sizeof(float*));
    for (int i = 0; i < c->n_layers; i++) {
        m->K[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim);
        m->V[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim);
    }
    for (int i = 0; i < np; i++) out[i] = prompt[i];
    /* DUMP=<path>: every forward's logits (vocab raw float32 each, in order), for a
     * comparison of two runs (tests/vulkan_engines.sh compares the dense chain's) */
    const char *dp = getenv("DUMP");
    FILE *df = dp && *dp ? fopen(dp, "wb") : NULL;
    float *logit = step(m, prompt, np, 0);          /* PREFILL */
    int len = np;
    for (int s = 0; s < n_new; s++) {
        if (df) fwrite(logit, sizeof(float), (size_t)c->vocab, df);
        int best = 0; float bv = logit[0];
        for (int i = 1; i < c->vocab; i++) if (logit[i] > bv) { bv = logit[i]; best = i; }
        free(logit);
        out[len++] = best;
        if (s == n_new - 1) break;
        int one = best;
        logit = step(m, &one, 1, len - 1);          /* DECODE */
    }
    if (df) fclose(df);
}

/* teacher-forced NLL of full_ids[np..nfull): feed the REFERENCE token at each step
 * (never the argmax), accumulate -log softmax(logits)[next_ref]. A loss meter for
 * throughput experiments: same engine path as decode, so hit rate/speed stay
 * comparable, but quality is measured as perplexity instead of exact-match.
 * Cross-checked vs HF transformers bf16 on identical token ids: engine (int8
 * experts) 12.11 ppl vs reference 12.25 (#108). Enabled by PPL=1. */
static int tf_nll(Model *m, const int *full, int nfull, int np, double *nll_out) {
    Cfg *c = &m->c;
    m->max_t = nfull;
    m->K = calloc(c->n_layers, sizeof(float*)); m->V = calloc(c->n_layers, sizeof(float*));
    for (int i = 0; i < c->n_layers; i++) {
        m->K[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim);
        m->V[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim);
    }
    double nll = 0; int scored = 0;
    float *logit = step(m, full, np, 0);              /* prefill on the prompt */
    for (int i = np; i < nfull; i++) {
        /* log softmax(logit)[full[i]] without materializing the softmax */
        float mx = logit[0]; for (int v = 1; v < c->vocab; v++) if (logit[v] > mx) mx = logit[v];
        double Z = 0; for (int v = 0; v < c->vocab; v++) Z += exp((double)logit[v] - mx);
        nll += -((double)logit[full[i]] - mx - log(Z));
        scored++;
        free(logit); logit = NULL;
        if (i == nfull - 1) break;
        logit = step(m, &full[i], 1, i);              /* teacher forcing */
    }
    if (logit) free(logit);
    *nll_out = nll / scored;
    return scored;
}

/* ---------- interactive chat mode (CHAT=1) ---------- */
/* OLMoE-Instruct's real template (tokenizer_config.json's chat_template):
 *   {{bos_token}}<|user|>\n{msg}\n<|assistant|>\n{reply}{eos_token}\n<|user|>\n...
 * bos_token == eos_token == "|||IP_ADDRESS|||" (a genuine OLMoE tokenizer quirk,
 * a PII-scrubbing artifact repurposed as the BOS/EOS marker — not a bug here).
 * It's an added special token, so tok_encode() tokenizes the literal string as
 * one atomic id, same as any other added token. <|user|>/<|assistant|> are NOT
 * special tokens in this tokenizer — just plain text the model was trained to
 * treat as turn markers.
 * Turn 1 gets bos_token with no separator before "<|user|>"; every later turn
 * gets eos_token+"\n" first (closing the previous assistant turn we never
 * explicitly appended to history, matching the is_stop-skips-append design
 * below) before its own "<|user|>...". */
static int fmt_user_turn(char *out, int cap, const char *msg, int first_turn) {
    int n = first_turn
        ? snprintf(out, cap, "|||IP_ADDRESS|||<|user|>\n%s\n<|assistant|>\n", msg)
        : snprintf(out, cap, "|||IP_ADDRESS|||\n<|user|>\n%s\n<|assistant|>\n", msg);
    return (n < 0 || n >= cap) ? -1 : n;
}

/* KV cache allocated ONCE for ctx_cap and never reallocated — hist_len only
 * grows (or resets to 0 on /reset), so every turn after the first reuses the
 * previous turns' cached keys/values: real multi-turn context, not a fresh
 * generate() call per message. ctx_cap capped at 4096: attention()'s per-head
 * score buffer (sc[4096]) is fixed-size, any position >= 4096 would overflow it. */
static void run_chat(Model *m, Tok *T, int ctx_cap) {
    Cfg *c = &m->c;
    m->max_t = ctx_cap;
    m->K = calloc(c->n_layers, sizeof(float*)); m->V = calloc(c->n_layers, sizeof(float*));
    for (int i = 0; i < c->n_layers; i++) {
        m->K[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim);
        m->V[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim);
    }

    int tok_eos = tok_id_of(T, "|||IP_ADDRESS|||");
    stops_arm_tok(c, tok_eos, T);

    int max_new = getenv("MAX_NEW") ? atoi(getenv("MAX_NEW")) : 512;
    if (max_new < 1) max_new = 1;

    int *hist = malloc((size_t)ctx_cap * sizeof(int));
    int hist_len = 0;
    int first_turn = 1;

    char *line = malloc(8192);
    char *turn = malloc(8192 + 64);
    int  *newids = malloc(8192 * sizeof(int));
    int  *gen = malloc((size_t)max_new * sizeof(int));
    char *outbuf = malloc(65536);

    fprintf(stderr, "olmoe chat — SNAP=%s, ctx=%d, TEMP=%.2f, NUCLEUS=%.2f\n"
                     "  type a message and press enter; /reset clears context; Ctrl-D exits\n",
            getenv("SNAP"), ctx_cap, g_temp, g_nuc);

    for (;;) {
        printf("\n> "); fflush(stdout);
        if (!fgets(line, 8192, stdin)) break;
        size_t L = strlen(line);
        while (L > 0 && (line[L-1] == '\n' || line[L-1] == '\r')) line[--L] = 0;
        if (L == 0) continue;
        if (!strcmp(line, "/reset")) { hist_len = 0; first_turn = 1; fprintf(stderr, "[chat] context reset\n"); continue; }

        int tn = fmt_user_turn(turn, 8192 + 64, line, first_turn);
        if (tn < 0) { fprintf(stderr, "[chat] message too long, skipped\n"); continue; }
        first_turn = 0;
        int nn = tok_encode(T, turn, tn, newids, 8192);

        if (hist_len + nn + max_new > ctx_cap) {
            fprintf(stderr, "[chat] context window full (%d/%d tokens) — /reset to start over\n",
                    hist_len + nn, ctx_cap);
            continue;
        }

        float *logit = step(m, newids, nn, hist_len);
        hist_len += nn;

        /* Every token counted in hist_len must have gone through step() exactly
         * once (that's what writes its KV-cache slot) — otherwise a later turn's
         * attention would read an uninitialized slot. So even on the turn's LAST
         * token we still call step() once to populate its KV before breaking;
         * only its returned logit (which nothing will consume) is discarded. */
        int ngen = 0;
        for (int s = 0; s < max_new; s++) {
            int nt = pick_tok(logit, c->vocab, -1);
            free(logit); logit = NULL;
            if (is_stop(nt)) break;
            hist[hist_len] = nt; gen[ngen++] = nt; hist_len++;
            int room_left = (s < max_new - 1) && (hist_len < ctx_cap);
            logit = step(m, &nt, 1, hist_len - 1);
            if (!room_left) {
                if (hist_len >= ctx_cap) fprintf(stderr, "\n[chat] context window full mid-reply — /reset to start over\n");
                free(logit); logit = NULL;
                break;
            }
        }

        int outn = tok_decode(T, gen, ngen, outbuf, 65535);
        outbuf[outn] = 0;
        printf("%s\n", outbuf);
        fflush(stdout);
#ifdef COLI_VULKAN
        olmoe_vk_report(); vkt_report("turn", m->hits, m->miss); olc_report(m);
#endif
    }
    free(line); free(turn); free(newids); free(gen); free(outbuf); free(hist);
}

/* ---------- serve mode: openai_server.py engine protocol ----------
 * stdin:  SUBMIT <id> <slot> <len> <max_tokens> <temp> <top_p>\n<payload>\n
 *         CANCEL <id>\n
 * stdout: READY sentinel once loaded, then per request a stream of
 *         DATA <id> <size>\n<bytes>\n frames and a final
 *         DONE <id> STAT <tok> <tps> <hit%> <rss> <prompt_tok> <len_limited>\n
 * Byte-identical to colibri.c's serve protocol (inkling.c documents it in
 * full above its own SUBMIT handling) so the shared openai_server.py gateway
 * drives olmoe unchanged.
 *
 * v1 scope, same as Inkling's first serve mode: one request in flight, full
 * re-prefill every turn, no cross-request KV reuse. The payload arrives
 * already rendered by openai_server.py's render_chat_olmoe (bos/eos turn
 * markers and all) -- this engine tokenizes it as-is, the same way run_chat()
 * feeds fmt_user_turn()'s output to tok_encode() above. Because nothing here
 * persists state across requests, a fresh prefill at pos_base=0 is enough to
 * start clean: attention() only ever reads positions [0, kv_len), so the
 * previous request's leftover K/V contents past the new prompt's length are
 * never touched, the same invariant CHAT mode's /reset already relies on
 * (it clears hist_len, not the K/V buffers themselves). */

typedef struct { char id[64]; int max_tok; float temp, top_p; char *payload; int plen;
                 int logprobs, pin; } SReq;   /* SUBMIT logprobs=k / pin=1 */
#define SRV_QMAX 16
static SReq g_q[SRV_QMAX]; static int g_qn = 0;
static const ColiServeWireProfile olmoe_wire = {
    .max_header_bytes = 511,
    .max_payload_bytes = 1u << 22,
    .max_tokens = 1 << 20,
    .require_exact_lf = 1,
    .require_finite_sampling = 0,
};

/* read one control line (+ payload for SUBMIT). cur_id: request in flight;
 * returns 1 if that request was cancelled, 0 otherwise, -1 on input EOF. */
static int serve_read_cmd(FILE *in, FILE *out, const char *cur_id) {
    ColiServeCommand command;
    ColiServeReadResult result = coli_serve_read_command(in, &olmoe_wire, &command);
    if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) return -1;
    if (result == COLI_SERVE_READ_NOMEM) {
        coli_serve_write_error(out, command.id, "out of memory");
        return -1;
    }
    if (result == COLI_SERVE_READ_BAD_REQUEST &&
        command.kind == COLI_SERVE_COMMAND_SUBMIT) {
        coli_serve_write_error(out, command.id, "bad submit header");
        return -1;
    }
    if (result != COLI_SERVE_READ_OK) return 0;
    if (command.kind == COLI_SERVE_COMMAND_CANCEL) {
        int cancelled = cur_id && !strcmp(command.id, cur_id);
        coli_serve_command_dispose(&command);
        return cancelled;
    }
    if (command.kind == COLI_SERVE_COMMAND_SUBMIT) {
        if (g_qn < SRV_QMAX) {
            SReq *q = &g_q[g_qn++];
            snprintf(q->id, sizeof(q->id), "%s", command.id);
            q->max_tok = command.max_tokens;
            q->logprobs = command.logprobs;
            q->pin = command.pin;
            q->temp = command.temperature;
            q->top_p = command.top_p;
            q->payload = (char *)coli_serve_command_take_payload(&command);
            q->plen = (int)command.payload_bytes;
        } else {
            coli_serve_write_error(out, command.id, "queue full");
        }
    }
    coli_serve_command_dispose(&command);
    return 0;
}

/* HITS rows cols hex: which experts this turn routed, one bit each, every
 * layer (all are MoE here, same rows and columns as EMAP), packed 8 per hex
 * pair. Same line colibri.c emits; the Brain tab lights up from it. */
static void serve_hits(Model *m) {
    Cfg *c = &m->c; int E = c->n_experts, rows = c->n_layers;
    if (!m->ehit) ehit_mark(m, -1, -1);
    int nb = (rows * E + 7) / 8;
    uint8_t *bm = calloc((size_t)nb, 1); int bit = 0;
    for (int i = 0; i < rows; i++)
        for (int e = 0; e < E; e++, bit++)
            if (m->ehit[i][e]) { bm[bit >> 3] |= (uint8_t)(1 << (bit & 7)); m->ehit[i][e] = 0; }
    char *hex = malloc((size_t)nb * 2 + 1); int w = 0;
    for (int b = 0; b < nb; b++) { hex[w++] = "0123456789abcdef"[bm[b] >> 4]; hex[w++] = "0123456789abcdef"[bm[b] & 15]; }
    hex[w] = 0;
    printf("HITS %d %d %s\n", rows, E, hex);
    fflush(stdout); free(hex); free(bm);
}

/* COLI_KV_PREFIX=0: never reuse a previous turn's cache. The escape hatch,
 * and the B arm of the A/B that shows reuse changes nothing but the time. */
static int kv_prefix_off(void){ const char *e = getenv("COLI_KV_PREFIX"); return e && *e == '0'; }

/* The counters when a request's prefill began: DONE and PROF report its share. */
typedef struct { double t0, attn0, moe0, head0; uint64_t h0, m0, disk0; long long fwd0; } OlmReqClock;

/* A request's prompt into the KV the Model holds: its tokens, the budget, the
 * prefix reuse and the pins, the prefill and its read-out. 1 with the prompt's ids
 * and the logits after it; 0 when the request ended here, its ERROR written.
 * serve_one and serve_mux start every request here. */
static int olm_serve_start(Model *m, Tok *T, SReq *q, int ctx_cap, int **ids_out, int *np_out,
                           float **logit_out, OlmReqClock *clk) {
    Cfg *c = &m->c;
    int cap = q->plen + 16;
    int *ids = malloc((size_t)cap * sizeof(int));
    int np = tok_encode(T, q->payload, q->plen, ids, cap);
    if (np <= 0) { coli_serve_write_error(stdout, q->id, "empty prompt"); free(ids); return 0; }
    int budget = coli_serve_budget(np, q->max_tok, ctx_cap, q->logprobs > 0);
    if (budget < 0) {
        char message[128];
        /* The frame the gateway turns into a 400 context_length_exceeded
         * (#506, #1381). Free text here reached the client as a 500. */
        snprintf(message, sizeof(message),
                 "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
                 np, q->max_tok, ctx_cap);
        coli_serve_write_error(stdout, q->id, message); free(ids); return 0;
    }
    if (budget < q->max_tok) {
        fprintf(stderr, "[serve] max_tokens %d clamped to %d (context %d - prompt %d); "
                        "raise CTX for longer answers\n",
                q->max_tok, budget, ctx_cap, np);
        q->max_tok = budget;
    }
    g_temp = q->temp; g_nuc = q->top_p;
    /* A chat client resends the whole transcript every turn. If this prompt
     * begins with the ids the cache was built from, those keys and values ARE
     * the state at those positions -- attention is causal, so a row depends on
     * its prefix and nothing later. Prefill only the tail. Reuse is all or
     * nothing: nothing here can rewind a cache. COLI_KV_PREFIX=0 turns it off,
     * COLI_PREFIX_LOG=1 reports the decision and its reason, because "it did
     * not get faster" is otherwise indistinguishable from "it is not wired up".
     *
     * On a miss the record must be CLEARED and not merely overwritten: a
     * shorter prompt writes fewer positions than the last one recorded, and
     * kv_prefix_record only ever grows the length, so the stale tail would
     * claim coverage the cache no longer has. */
    int reuse = kv_prefix_off() ? 0 : kv_prefix_reuse(&m->kvp, ids, np);
    /* La fotografia si prova sempre, non solo quando il riuso vivo fallisce:
     * altrimenti la prima opzione trova ancora lo stato del prompt, passa dal
     * riuso normale e il suo primo token resta senza predittore. */
    g_pin_use_logit = 0; g_pin_logit = NULL;
    {
        /* Il piu profondo degli scatti che sia un prefisso di questo prompt.
         * Con due livelli (istruzioni, istruzioni+domanda) e il secondo a
         * decidere; se e' morto si ripiega sul primo invece di rifare tutto. */
        int s = coli_pin_best(&g_pins, ids, np);
        while (s >= 0) {
            ColiPin *k = &g_pins.slot[s];
            if (kv_prefix_holds(&m->kvp, k->ids, k->len)) {
                kv_prefix_clear(&m->kvp);
                kv_prefix_record(&m->kvp, k->ids, 0, k->len);
                m->kv_len = k->len;
                reuse = k->len;
                g_pin_logit = k->logit; g_pin_use_logit = k->logit != NULL;
                coli_pin_touch(&g_pins, s);
                break;
            }
            k->len = 0;                    /* le righe non ci sono piu: scatto orfano */
            s = coli_pin_best(&g_pins, ids, np);
        }
    }
    g_echo_k = q->logprobs; g_echo_id = q->id; g_echo_tok = T;
    if (!reuse) kv_prefix_clear(&m->kvp);
    if (getenv("COLI_PREFIX_LOG")) {
        if (reuse)
            fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",
                    reuse, np, 100.0 * reuse / np);
        else
            fprintf(stderr, "[PREFIX] no reuse: held=%d cap=%d prompt=%d\n",
                    m->kvp.len, m->kvp.cap, np);
        fflush(stderr);
    }
    clk->t0 = now_s();
    clk->h0 = m->hits; clk->m0 = m->miss;
    clk->disk0 = __atomic_load_n(&m->disk_ns, __ATOMIC_RELAXED);
    clk->attn0 = g_prof_attn_s; clk->moe0 = g_prof_moe_s; clk->head0 = g_prof_head_s;
    clk->fwd0 = g_prof_forwards;
    /* `reuse` is the ABSOLUTE position of the first fresh token: attention
     * and the KV rows are position-indexed, so this has to be the real
     * offset, not a count of what is left to do. */
    float *logit = step(m, ids + reuse, np - reuse, reuse);
    if (q->pin && logit) {
        coli_pin_pool_init(&g_pins, c->vocab);
        if (coli_pin_store(&g_pins, ids, np, logit)) {
            fprintf(stderr, "[PIN] scatto a %d token\n", np); fflush(stderr);
        }
    }
    g_echo_k = 0; g_echo_id = NULL;   /* la lettura riguarda il prefill, non la decodifica */
    *ids_out = ids; *np_out = np; *logit_out = logit;
    return 1;
}

static int serve_one(Model *m, Tok *T, SReq *q, int ctx_cap) {
    Cfg *c = &m->c;
    int *ids = NULL, np = 0; float *logit = NULL; OlmReqClock clk;
    if (!olm_serve_start(m, T, q, ctx_cap, &ids, &np, &logit, &clk)) return 0;
    double t0 = clk.t0;
    uint64_t h0 = clk.h0, m0 = clk.m0;
    uint64_t disk0 = clk.disk0;
    double attn0 = clk.attn0, moe0 = clk.moe0, head0 = clk.head0;
    long long fwd0 = clk.fwd0;
    int hist_len = np, gen = 0, limited = 1, cancelled = 0;
    char buf[512];
    for (int s = 0; s < q->max_tok && !cancelled; s++) {
        int nt = pick_tok(logit, c->vocab, -1);
        char lptail[1024]; lptail[0] = 0;
        if (q->logprobs > 0) coli_logprob_tail(lptail, sizeof lptail, logit, c->vocab, nt, q->logprobs);
        free(logit); logit = NULL;
        if (is_stop(nt)) { limited = 0; break; }
        int nb = tok_decode(T, &nt, 1, buf, sizeof(buf)-1);
        if (q->logprobs > 0) coli_serve_write_data_lp(stdout, q->id, buf, (size_t)nb, lptail);
        else coli_serve_write_data(stdout, q->id, buf, (size_t)nb);
        gen++; hist_len++;
        while (coli_stdin_readable()) {
            int r = serve_read_cmd(stdin, stdout, q->id);
            if (r < 0) { free(ids); return -1; }
            if (r > 0) { cancelled = 1; limited = 0; }
        }
        /* Unlike run_chat(), we do not step() the final token just to populate
         * its KV slot: nothing in serve mode reads past this request's own
         * reply, so a discarded logit here costs nothing. */
        if (cancelled || s == q->max_tok - 1 || hist_len >= ctx_cap) break;
        logit = step(m, &nt, 1, hist_len - 1);
    }
    free(logit);
    double dt = now_s() - t0;
    double tot = (double)(m->hits - h0 + m->miss - m0);
    ColiServeDone done = {
        .completion_tokens = gen,
        .tokens_per_second = dt > 0 ? gen/dt : 0.0,
        .cache_hit_percent = tot ? 100.0*(m->hits-h0)/tot : 0.0,
        .rss_gb = rss_gb(),
        .prompt_tokens = np,
        .length_limited = limited,
    };
    coli_serve_write_done(stdout, q->id, &done);
    /* PROF: per-turn phase timings for the dashboard, field order the protocol's:
     * disk, wait, matmul, attention, lm_head, forwards. disk is what the expert
     * loads took (#1449 first half); matmul is the MoE block time minus that
     * disk time, clamped at zero (the PILOT worker's reads are counted in disk
     * but happen off the block's clock); attention and lm_head are measured
     * around their calls; forwards counts positions through step(). wait stays
     * zero: this engine has no separate wait phase. */
    double disk_s = (double)(__atomic_load_n(&m->disk_ns, __ATOMIC_RELAXED) - disk0) / 1e9;
    double matmul_s = (g_prof_moe_s - moe0) - disk_s; if (matmul_s < 0.0) matmul_s = 0.0;
    /* microsecond resolution: see qwen36.c, same reason (a sub-millisecond tiny turn read as unmeasured) */
    printf("PROF %.6f %d %d %.6f 0.0 %.6f %.6f %.6f %lld\n", dt, np, gen, disk_s, matmul_s,
           g_prof_attn_s - attn0, g_prof_head_s - head0, g_prof_forwards - fwd0);
    fflush(stdout);
#ifdef COLI_VULKAN
    olmoe_vk_report(); vkt_report("turn", m->hits, m->miss); olc_report(m);
#endif
    serve_hits(m);
    free(ids);
    return 0;
}

/* dashboard HWINFO/TIERS/EMAP: same lines the other serve-capable engines
 * emit for the web dashboard's hardware panel and Brain page. olmoe.c is
 * CPU-only (no CUDA/Metal backend), so the GPU fields are always empty, and
 * every layer is a MoE layer (no dense/sparse split like GLM-5.2), so the
 * tier scan below runs over all n_layers unconditionally. */
static void serve_hwinfo(Model *m) {
    (void)m;
    char cpu[256] = ""; int cores = 0; double rt = 0, ra = 0;
    FILE *ci = fopen("/proc/cpuinfo", "r");
    if (ci) { char ln[256];
        while (fgets(ln, sizeof(ln), ci)) if (!strncmp(ln, "model name", 10)) {
            char *p = strchr(ln, ':'); if (p) { p++; while (*p == ' ') p++;
            int n = (int)strlen(p); if (n > 0 && p[n-1] == '\n') p[--n] = 0;
            snprintf(cpu, sizeof(cpu), "%s", p); } break; }
        fclose(ci); }
#ifdef _SC_NPROCESSORS_ONLN
    cores = (int)sysconf(_SC_NPROCESSORS_ONLN);
#endif
    FILE *mi = fopen("/proc/meminfo", "r");
    if (mi) { char ln[256]; double v = 0;
        while (fgets(ln, sizeof(ln), mi)) {
            if (sscanf(ln, "MemTotal: %lf", &v) == 1) rt = v/1e6;
            if (sscanf(ln, "MemAvailable: %lf", &v) == 1) ra = v/1e6;
        } fclose(mi); }
    /* #1601: neither /proc/cpuinfo nor /proc/meminfo exists on macOS (or
     * Windows), so both reads above fail silently and this line went out as
     * "0.0 0.0 ... unknown" -- the /health hwinfo the dashboard renders. Fill
     * only what /proc could not supply, so the Linux path stays byte-identical.
     * Use the shared one-pass probe on the other platforms too: besides keeping
     * total and available RAM from different definitions, this avoids calling
     * platform-specific helpers that are not available in every build. */
#if defined(__APPLE__)
    if (!cpu[0]) {
        size_t len = sizeof(cpu);
        if (sysctlbyname("machdep.cpu.brand_string", cpu, &len, NULL, 0) != 0)
            cpu[0] = 0;
    }
#endif
#if defined(__APPLE__) || defined(_WIN32)
    if (rt <= 0.0 || ra <= 0.0) {
        double t = 0, a = 0;
        compat_meminfo_gb(&t, &a);
        if (rt <= 0.0) rt = t;
        if (ra <= 0.0) ra = a;
    }
#else
    if (ra <= 0.0) ra = compat_mem_available_gb();
    if (rt <= 0.0) {
#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
        long pg = sysconf(_SC_PHYS_PAGES), ps = sysconf(_SC_PAGESIZE);
        if (pg > 0 && ps > 0) rt = (double)pg * ps / 1e9;
#endif
    }
#endif
    printf("HWINFO %d %.1f %.1f 0 0.0 %s|\n", cores, rt, ra, cpu[0] ? cpu : "unknown");
    fflush(stdout);
}

static void serve_tiers_emap(Model *m) {
    Cfg *c = &m->c; int E = c->n_experts;
    int filled = 0;
    for (int i = 0; i < c->n_layers; i++) filled += m->cache[i].n;
    int64_t I = c->inter, D = c->hidden;
    /* per-expert resident bytes: int8 gate/up/down + one f32 scale per row */
    int64_t slotb = 3*I*D + (2*I+D)*4;
    printf("TIERS 0 %d %d 0.00 %.2f\n", filled, c->n_layers*E - filled, filled*(double)slotb/1e9);
    /* EMAP: 1 byte/expert hex — tier(2b: 0=disk 1=RAM 2=Vulkan device)<<6 | heat(6b: log2 usage) */
    char *hex = malloc((size_t)c->n_layers*E*2 + 1); int w = 0;
    for (int i = 0; i < c->n_layers; i++) {
        LCache *lc = &m->cache[i];
        for (int e = 0; e < E; e++) {
            int tier = 0;
            for (int z = 0; z < lc->n; z++) if (lc->slots[z].eid == e) { tier = 1; break; }
#ifdef COLI_VULKAN
            if (vkt_resident(i, e)) tier = 2;   /* on the Vulkan device */
#endif
            uint32_t u = m->freq[i] ? m->freq[i][e] : 0;
            int heat = 0; while (u) { heat++; u >>= 1; } if (heat > 63) heat = 63;
            int b = (tier << 6) | heat;
            hex[w++] = "0123456789abcdef"[b >> 4];
            hex[w++] = "0123456789abcdef"[b & 15];
        }
    }
    hex[w] = 0;
    printf("EMAP %d %d %s\n", c->n_layers, E, hex);
    fflush(stdout); free(hex);
}

/* ---- several conversations at once (KV_SLOTS>1) -------------------------------
 * The gateway's cache slots, each a conversation with KV rows of its own (OlmSeq).
 * A SUBMIT on a free slot starts its request at once through olm_serve_start, on
 * that slot's KV: its prefix reuse and pins work as a lone serve's. Then every step
 * picks the next token of each active request and runs one forward over a row of
 * each (olm_step_rows): the matrices and the experts are read once for all of them.
 * A request's frames are a lone request's; they interleave by id. As alone, CANCEL
 * ends a request with DONE and STOP is not read. */

typedef struct {
    SReq q;
    int active, cancel, limited;
    int *ids, np, gen;
    float *lo;                  /* the logits the next pick reads */
    OlmReqClock clk;
} OlmMuxReq;

static void olm_mux_bind(Model *m, int slot) {
    if (g_olm_mux_cur == slot) return;
    if (g_olm_mux_cur >= 0) olm_seq_swap(m, &g_olm_mux_seq[g_olm_mux_cur]);
    if (slot >= 0) olm_seq_swap(m, &g_olm_mux_seq[slot]);
    g_olm_mux_cur = slot;
}

/* A request's end, as serve_one ends one. */
static void olm_mux_finish(Model *m, OlmMuxReq *r) {
    free(r->lo); r->lo = NULL; free(r->ids); r->ids = NULL; r->active = 0;
    if (r->cancel) r->limited = 0;
    double dt = now_s() - r->clk.t0;
    double tot = (double)(m->hits - r->clk.h0 + m->miss - r->clk.m0);
    ColiServeDone done = {
        .completion_tokens = r->gen,
        .tokens_per_second = dt > 0 ? r->gen/dt : 0.0,
        .cache_hit_percent = tot ? 100.0*(m->hits-r->clk.h0)/tot : 0.0,
        .rss_gb = rss_gb(),
        .prompt_tokens = r->np,
        .length_limited = r->limited,
    };
    coli_serve_write_done(stdout, r->q.id, &done);
    double disk_s = (double)(__atomic_load_n(&m->disk_ns, __ATOMIC_RELAXED) - r->clk.disk0) / 1e9;
    double matmul_s = (g_prof_moe_s - r->clk.moe0) - disk_s; if (matmul_s < 0.0) matmul_s = 0.0;
    printf("PROF %.6f %d %d %.6f 0.0 %.6f %.6f %.6f %lld\n", dt, r->np, r->gen, disk_s, matmul_s,
           g_prof_attn_s - r->clk.attn0, g_prof_head_s - r->clk.head0, g_prof_forwards - r->clk.fwd0);
    fflush(stdout);
    serve_hits(m);
}

/* The next token of an active request, as serve_one's loop picks and sends it: 1
 * with the token when the request goes on, 0 when it ended. */
static int olm_mux_pick(Model *m, Tok *T, OlmMuxReq *r, int ctx_cap, int *tk_out) {
    Cfg *c = &m->c;
    if (r->cancel || r->gen >= r->q.max_tok) { olm_mux_finish(m, r); return 0; }
    g_temp = r->q.temp; g_nuc = r->q.top_p;
    int nt = pick_tok(r->lo, c->vocab, -1);
    char lptail[1024]; lptail[0] = 0;
    if (r->q.logprobs > 0) coli_logprob_tail(lptail, sizeof lptail, r->lo, c->vocab, nt, r->q.logprobs);
    free(r->lo); r->lo = NULL;
    if (is_stop(nt)) { r->limited = 0; olm_mux_finish(m, r); return 0; }
    char buf[512];
    int nb = tok_decode(T, &nt, 1, buf, sizeof(buf)-1);
    if (r->q.logprobs > 0) coli_serve_write_data_lp(stdout, r->q.id, buf, (size_t)nb, lptail);
    else coli_serve_write_data(stdout, r->q.id, buf, (size_t)nb);
    r->gen++;
    if (r->gen >= r->q.max_tok || r->np + r->gen >= ctx_cap) { olm_mux_finish(m, r); return 0; }
    *tk_out = nt; return 1;
}

static void serve_mux(Model *m, Tok *T, int ctx_cap) {
    int n = g_olm_mux_slots, V = m->c.vocab, input_eof = 0;
    OlmMuxReq *rq = calloc((size_t)n, sizeof *rq);
    OlmRow *rows = malloc((size_t)n * sizeof *rows);
    int *tok = malloc((size_t)n * sizeof(int)), *who = malloc((size_t)n * sizeof(int));
    if (!rq || !rows || !tok || !who) { fprintf(stderr, "[serve] out of memory\n"); exit(1); }
    unsigned long long steps = 0, nrows = 0;
    fprintf(stderr, "[olmoe] serving %d conversations at once (KV_SLOTS)\n", n);
    for (;;) {
        int active = 0; for (int i = 0; i < n; i++) active += rq[i].active;
        /* idle: wait for a command; decoding: take one only when one is there */
        if (!input_eof && (!active || coli_stdin_readable())) {
            ColiServeCommand command;
            ColiServeReadResult result = coli_serve_read_command(stdin, &olmoe_wire, &command);
            if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) input_eof = 1;
            else if (result == COLI_SERVE_READ_NOMEM) { coli_serve_write_error(stdout, command.id, "out of memory"); input_eof = 1; }
            else if (result == COLI_SERVE_READ_BAD_REQUEST) {
                if (command.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, command.id, "bad submit header");
                coli_serve_command_dispose(&command);
            } else if (result == COLI_SERVE_READ_OK) {
                if (command.kind == COLI_SERVE_COMMAND_CANCEL) {
                    for (int i = 0; i < n; i++) if (rq[i].active && !strcmp(rq[i].q.id, command.id)) rq[i].cancel = 1;
                } else if (command.kind == COLI_SERVE_COMMAND_SUBMIT) {
                    if (command.slot < 0 || command.slot >= n) coli_serve_write_error(stdout, command.id, "invalid cache slot");
                    else if (rq[command.slot].active) coli_serve_write_error(stdout, command.id, "SLOT_BUSY");
                    else {
                        OlmMuxReq *t = &rq[command.slot];
                        memset(t, 0, sizeof *t);
                        snprintf(t->q.id, sizeof(t->q.id), "%s", command.id);
                        t->q.max_tok = command.max_tokens; t->q.logprobs = command.logprobs; t->q.pin = command.pin;
                        t->q.temp = command.temperature; t->q.top_p = command.top_p;
                        t->q.payload = (char *)coli_serve_command_take_payload(&command);
                        t->q.plen = (int)command.payload_bytes;
                        olm_mux_bind(m, command.slot);
                        int ok = olm_serve_start(m, T, &t->q, ctx_cap, &t->ids, &t->np, &t->lo, &t->clk);
                        free(t->q.payload); t->q.payload = NULL;
                        if (ok) {
                            t->active = 1; t->limited = 1;
                            if (t->np > g_olm_mux_seq[command.slot].kv_hi) g_olm_mux_seq[command.slot].kv_hi = t->np;
                        }
                    }
                }
                coli_serve_command_dispose(&command);
            }
        }
        active = 0; for (int i = 0; i < n; i++) active += rq[i].active;
        if (!active) { if (input_eof) break; continue; }
        int S = 0, ended = 0;
        for (int i = 0; i < n; i++) if (rq[i].active) {
            int tk;
            if (!olm_mux_pick(m, T, &rq[i], ctx_cap, &tk)) { ended = 1; continue; }
            rows[S] = (OlmRow){&g_olm_mux_seq[i], rq[i].np + rq[i].gen - 1}; tok[S] = tk; who[S] = i; S++;
        }
        if (S) {
            olm_mux_bind(m, -1);   /* every conversation parked: the rows read theirs */
            float *lo = olm_step_rows(m, rows, tok, S);
            steps++; nrows += (unsigned long long)S;
            for (int s = 0; s < S; s++) {
                OlmMuxReq *t = &rq[who[s]];
                t->lo = falloc(V); memcpy(t->lo, lo + (int64_t)s * V, (size_t)V * sizeof(float));
            }
            free(lo);
        }
        if (ended) {
#ifdef COLI_VULKAN
            olmoe_vk_report(); vkt_report("turn", m->hits, m->miss);
#endif
            serve_tiers_emap(m);
        }
    }
    fprintf(stderr, "[olmoe] KV_SLOTS=%d: %llu decode steps, %llu rows (%.2f a step)\n", n, steps, nrows,
            steps ? (double)nrows / (double)steps : 0.0);
    olm_mux_bind(m, 0);
    free(rq); free(rows); free(tok); free(who);
}

static void serve_loop(Model *m, Tok *T, int ctx_cap) {
    coli_serve_stdio_init();
    int tok_eos = tok_id_of(T, "|||IP_ADDRESS|||");
    stops_arm_tok(&m->c, tok_eos, T);
    coli_serve_write_ready(stdout, rss_gb());
    serve_hwinfo(m);
    serve_tiers_emap(m);
    if (g_olm_mux_slots > 1) { serve_mux(m, T, ctx_cap); return; }
    for (;;) {
        while (!g_qn) if (serve_read_cmd(stdin, stdout, NULL) < 0) return;
        SReq q = g_q[0];
        memmove(g_q, g_q + 1, (size_t)(--g_qn) * sizeof(SReq));
        int fatal = serve_one(m, T, &q, ctx_cap);
        free(q.payload);
        if (fatal < 0) return;
        /* Resend the grid after EVERY turn, not only after READY: at boot the
         * expert cache is empty by definition, and that cold snapshot stayed
         * the only one the dashboard ever saw -- all grey, RAM 0, everything
         * on disk, forever. HITS was already per turn, which is why the white
         * "routed now" flash worked while the residency colour never moved.
         * inkling.c, kimi_k3.c, qwen38.c, deepseek_v41.c and colibri.c
         * already do this. */
        serve_tiers_emap(m);
    }
}

/* ---------- lettura ref.json ---------- */
static int *read_int_array(jval *o, const char *key, int *n_out) {
    jval *a = json_get(o, key);
    int *r = malloc(a->len * sizeof(int));
    for (int i = 0; i < a->len; i++) r[i] = (int)a->kids[i]->num;
    *n_out = a->len; return r;
}

/* The reference is input, and the run indexes with it: generate() and tf_nll()
 * size the token buffer and the KV cache from its lengths, attention() scores
 * every position into its sc[4096] stack buffer, and every id selects an
 * embedding row (a logit too, under PPL=1). Refuse one that does not fit. */
static int ref_fits(const int *prompt, int np, const int *full, int nfull, int vocab) {
    if (np < 1 || nfull <= np) {
        fprintf(stderr, "reference: prompt_ids holds %d tokens and full_ids %d; "
                        "need 1 <= prompt < full\n", np, nfull);
        return 0;
    }
    if (nfull > 4096) {
        fprintf(stderr, "reference: full_ids holds %d tokens, more than the 4096 "
                        "positions attention() holds\n", nfull);
        return 0;
    }
    const int *ids[2] = { prompt, full };
    const int n[2] = { np, nfull };
    const char *name[2] = { "prompt_ids", "full_ids" };
    for (int a = 0; a < 2; a++)
        for (int i = 0; i < n[a]; i++)
            if (ids[a][i] < 0 || ids[a][i] >= vocab) {
                fprintf(stderr, "reference: %s[%d] = %d is not a token id (vocab %d)\n",
                        name[a], i, ids[a][i], vocab);
                return 0;
            }
    return 1;
}

#ifndef OLMOE_NO_MAIN
int main(int argc, char **argv) {
    coli_omp_tune_threads("olmoe");   /* squadra sui core fisici, niente spin-wait: vedi omp_tune.h */
    const char *snap = getenv("SNAP");
    if (!snap) { coli_print_launcher_help("OLMoE", "SNAP=<model directory> ./olmoe ..."); return 1; }
    g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0;
    g_wide  = getenv("WIDE")  ? atoi(getenv("WIDE"))  : 1;
    g_pilot_evict_guard = getenv("PILOT_EVICT_GUARD") ? atoi(getenv("PILOT_EVICT_GUARD")) : 1;
    g_expert_drop = getenv("EXPERT_DROP") ? atoi(getenv("EXPERT_DROP")) : 0;
    g_fused3     = getenv("FUSED3") ? atoi(getenv("FUSED3")) : 0;
    if (g_wide < 1) g_wide = 1;
    if (g_wide > 4) g_wide = 4;
    int hot_n  = getenv("HOT")   ? atoi(getenv("HOT"))   : 0;
    int cap    = argc > 1 ? coli_arg_int(argv[1], "cache/layer") : 16;
    int bits   = argc > 2 ? coli_arg_int(argv[2], "expert bits") : 8;
    if (bits < 2 || bits > 8) {
        fprintf(stderr, "quant_bits must be 2..8 (got %d)\n", bits);
        return 1;
    }

    /* SERVE=1: openai_server.py drives the engine over stdin/stdout (READY
     * handshake, SUBMIT/CANCEL, DATA/DONE/PROF frames, HWINFO/TIERS/EMAP for
     * the dashboard) — same protocol as colibri.c/inkling.c/kimi_k3.c. v1:
     * one request at a time, full re-prefill every turn (see serve_one()). */
    if (getenv("SERVE") && getenv("SERVE")[0] == '1') {
        const char *ks = getenv("KV_SLOTS");   /* the conversations decoded at once */
        if (ks && *ks) {
            char *end = NULL; long v = strtol(ks, &end, 10);
            if (end == ks || *end || v < 1 || v > 16) { fprintf(stderr, "KV_SLOTS must be between 1 and 16\n"); return 2; }
            g_olm_mux_slots = (int)v;
        }
        int ctx_cap = getenv("CTX") ? atoi(getenv("CTX")) : 4096;
        if (ctx_cap < 1 || ctx_cap > 4096) {   /* attention()'s sc[4096] score buffer hard-caps this */
            fprintf(stderr, "CTX must be 1..4096 (got %d)\n", ctx_cap);
            return 1;
        }
    /* static, not a stack local: the PILOT prefetch worker is detached and
     * loops forever, and it keeps this address in the global pilot_m. A stack
     * Model dies when main returns while that thread is still dereferencing
     * it -- ASan: stack-use-after-return, READ of size 8, in a worker thread,
     * with the run's tokens already correct (#1262). Static storage outlives
     * every thread, so the pointer the worker holds stays valid. */
        static Model m; model_init(&m, snap, cap, bits);
        m.max_t = ctx_cap;
        m.K = calloc(m.c.n_layers, sizeof(float*)); m.V = calloc(m.c.n_layers, sizeof(float*));
        for (int i = 0; i < m.c.n_layers; i++) {
            m.K[i] = falloc((int64_t)m.c.n_heads * m.max_t * m.c.head_dim);
            m.V[i] = falloc((int64_t)m.c.n_heads * m.max_t * m.c.head_dim);
        }
        /* Serve mode only: the record is sized with the cache it describes, and
         * the cache here is allocated once for ctx_cap and never reallocated,
         * so a recorded position stays valid for the life of the process. The
         * CLI paths (generate/run_chat/PPL) leave it unallocated, which makes
         * every kv_prefix call there a no-op. */
        kv_prefix_alloc(&m.kvp, m.max_t);
        if (g_olm_mux_slots > 1) {   /* slot 0 is the Model's own KV */
            g_olm_mux_seq = calloc((size_t)g_olm_mux_slots, sizeof *g_olm_mux_seq);
            for (int s = 1; g_olm_mux_seq && s < g_olm_mux_slots; s++) {
                OlmSeq *q = &g_olm_mux_seq[s];
                q->K = calloc(m.c.n_layers, sizeof(float*)); q->V = calloc(m.c.n_layers, sizeof(float*));
                for (int i = 0; i < m.c.n_layers; i++) {
                    q->K[i] = falloc((int64_t)m.c.n_heads * m.max_t * m.c.head_dim);
                    q->V[i] = falloc((int64_t)m.c.n_heads * m.max_t * m.c.head_dim);
                }
                kv_prefix_alloc(&q->kvp, m.max_t);
            }
            if (!g_olm_mux_seq) { fprintf(stderr, "[serve] out of memory for %d conversations\n", g_olm_mux_slots); return 1; }
            g_olm_mux_cur = 0;
            fprintf(stderr, "[olmoe] KV_SLOTS=%d: nothing drafts, each conversation has %d positions of KV\n",
                    g_olm_mux_slots, m.max_t);
        }
        Tok T;
        char tokpath[2048]; snprintf(tokpath, sizeof(tokpath), "%s/tokenizer.json", snap);
        tok_load(&T, tokpath);
        coli_rt_term_arm();   /* SIGTERM must reach the save below (#1629) */
        serve_loop(&m, &T, ctx_cap);
        { const char *up = getenv("COLI_USAGE");
          if (up && *up) rt_save(up, 0); }
        return 0;
    }

    if (getenv("CHAT")) {   /* interactive mode: bypasses the ref.json harness entirely */
        /* #509 convention, same as the GLM engine: COLI_TEMP is the primary channel;
         * TEMP stays a legacy alias ONLY when it is fully numeric — on Windows and under
         * ROCm stacks %TEMP% names a directory, and atof("C:\...") == 0.0 would silently
         * force greedy decoding for every olmoe chat on those hosts. */
        if (getenv("COLI_TEMP")) g_temp = (float)atof(getenv("COLI_TEMP"));
        else if (getenv("TEMP") && *getenv("TEMP")) {
            char *tend; double tv = strtod(getenv("TEMP"), &tend);
            if (tend != getenv("TEMP") && *tend == '\0') g_temp = (float)tv;
        }
        g_nuc  = getenv("NUCLEUS") ? (float)atof(getenv("NUCLEUS")) : g_nuc;
        int ctx_cap = getenv("CTX") ? atoi(getenv("CTX")) : 4096;
        if (ctx_cap < 1 || ctx_cap > 4096) {   /* attention()'s sc[4096] score buffer hard-caps this */
            fprintf(stderr, "CTX must be 1..4096 (got %d)\n", ctx_cap);
            return 1;
        }
    /* static, not a stack local: the PILOT prefetch worker is detached and
     * loops forever, and it keeps this address in the global pilot_m. A stack
     * Model dies when main returns while that thread is still dereferencing
     * it -- ASan: stack-use-after-return, READ of size 8, in a worker thread,
     * with the run's tokens already correct (#1262). Static storage outlives
     * every thread, so the pointer the worker holds stays valid. */
        static Model m; model_init(&m, snap, cap, bits);
        printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());
        Tok T;
        char tokpath[2048]; snprintf(tokpath, sizeof(tokpath), "%s/tokenizer.json", snap);
        tok_load(&T, tokpath);
        run_chat(&m, &T, ctx_cap);
        { const char *up = getenv("COLI_USAGE");
          if (up && *up) rt_save(up, 0); }
        return 0;
    }

    const char *refpath = argc > 3 ? argv[3] : "ref.json";

    float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;
    float conf   = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;

    printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d guard=%d hot=%d smooth=%.2f conf=%.2f fused3=%d ==\n",
           cap, bits, g_pilot, g_wide, g_pilot_evict_guard, hot_n, smooth, conf, g_fused3);

    FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; }
    fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
    char *buf=malloc(n+1); if (fread(buf,1,n,f)!=(size_t)n) {} buf[n]=0; fclose(f);
    char *arena=NULL; jval *ref = json_parse(buf, &arena);
    int np, nfull; int *prompt = read_int_array(ref,"prompt_ids",&np); int *full = read_int_array(ref,"full_ids",&nfull);
    int n_new = nfull - np;

    /* static, not a stack local: the PILOT prefetch worker is detached and
     * loops forever, and it keeps this address in the global pilot_m. A stack
     * Model dies when main returns while that thread is still dereferencing
     * it -- ASan: stack-use-after-return, READ of size 8, in a worker thread,
     * with the run's tokens already correct (#1262). Static storage outlives
     * every thread, so the pointer the worker holds stays valid. */
    static Model m; model_init(&m, snap, cap, bits);
    printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());
    if (!ref_fits(prompt, np, full, nfull, m.c.vocab)) { free(buf); free(arena); return 1; }

    if (getenv("PPL") && atoi(getenv("PPL")) == 1) {   /* loss-meter mode: teacher-forced NLL */
        double nll; double t = now_s();
        int scored = tf_nll(&m, full, nfull, np, &nll);
        double dt = now_s() - t;
        double tot = m.hits + m.miss;
        printf("TF-NLL: %.4f nats/token over %d tokens  |  ppl = %.2f\n", nll, scored, exp(nll));
        printf("Expert cache hit rate: %.1f%%  (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,
               (unsigned long long)m.hits, (unsigned long long)m.miss);
        printf("Speed: %.2f tok/s (%.1fs for %d tokens) | PEAK RSS: %.2f GB\n", scored/dt, dt, scored, rss_gb());
#ifdef COLI_VULKAN
        fflush(stdout); olmoe_vk_report(); vkt_report("run", m.hits, m.miss); olc_report(&m);
#endif
        free(buf); free(arena);
        return 0;      /* PPL is a measurement run: no rt_save on purpose, so a loss
                        * sweep cannot fold its own tokens into the persisted ranking */
    }

    int *out = malloc((np + n_new) * sizeof(int));
    double t = now_s();
    generate(&m, prompt, np, n_new, out);
    double dt = now_s() - t;

    int match = 0;
    printf("\nReference: ");  for (int i=np;i<nfull;i++) printf("%d ", full[i]);
    printf("\nC engine : ");  for (int i=np;i<nfull;i++) { printf("%d ", out[i]); if (out[i]==full[i]) match++; }
    printf("\nMatching tokens: %d/%d\n", match, n_new);
    double tot = m.hits + m.miss;
    printf("\nPEAK RSS: %.2f GB\n", rss_gb());
    printf("Expert cache hit rate: %.1f%%  (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,
           (unsigned long long)m.hits, (unsigned long long)m.miss);


    // Persistent Hot Pinning: save dynamic pinning if newly created
    if (m.hot_pinned) {
        char pinpath[512];
        snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap);
        FILE *pinf_chk = fopen(pinpath, "rb");
        if (!pinf_chk) {
            FILE *pinf_save = coli_own_fopen(pinpath, "wb");   /* never through a planted link */
            if (pinf_save) {
                size_t expected_size = (size_t)m.c.n_layers * m.c.n_experts;
                fwrite(m.is_pinned, 1, expected_size, pinf_save);
                fclose(pinf_save);
                printf("[HOT] Saved persistent pinning to %s\n", pinpath);
            }
        } else {
            fclose(pinf_chk);
        }
    }

    { const char *up = getenv("COLI_USAGE");
      if (up && *up) rt_save(up, 0); }              /* same bytes as every other engine */
    printf("Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new);
    /* One line, every engine, one format: `coli tune` sweeps scheduling knobs and
     * needs tokens-and-elapsed to compare candidates. Before this only colibri
     * emitted a parseable throughput line (REPLAY decode), so the tuner was
     * GLM-only and bannered the right model while launching the wrong engine
     * (#898). Printed to stdout, which is what autotune captures.
     * Tokens and seconds, not tok/s: the ratio is derived by the caller at full
     * precision (#852 -- two decimals of tok/s is one significant digit at the
     * rates this engine runs at). */
    printf("TUNE decode: %d tokens in %.3fs\n", n_new, dt);
#ifdef COLI_VULKAN
    fflush(stdout); olmoe_vk_report(); vkt_report("run", m.hits, m.miss); olc_report(&m);
#endif
    free(buf); free(arena);
    return 0;
}
#endif /* OLMOE_NO_MAIN */

#ifdef COLI_SEGMENT_ADAPTER
/* ---------- engine-owned Segment adapter ------------------------------ */

typedef struct {
    Model model;
    uint32_t layer_begin, layer_end, context_tokens;
    pthread_mutex_t run_lock;
} OlmoeSegmentEngine;

typedef struct {
    OlmoeSegmentEngine *engine;
    float **K, **V;
    uint32_t context_tokens, position;
} OlmoeSegmentSession;

static void olmoe_segment_model_destroy(OlmoeSegmentEngine *engine) {
    if (!engine) return;
    Model *model = &engine->model;
    for (uint32_t layer = engine->layer_begin; layer < engine->layer_end;
         layer++) {
        Layer *weights = &model->L[layer];
#ifdef COLI_VULKAN
        vk_res_free(&weights->vk_q); vk_res_free(&weights->vk_k); vk_res_free(&weights->vk_v);
        vk_res_free(&weights->vk_o); vk_res_free(&weights->vk_gate);
#endif
        free(weights->in_ln); free(weights->post_ln);
        free(weights->q); free(weights->k); free(weights->v); free(weights->o);
        free(weights->qn); free(weights->kn); free(weights->gate);
        LCache *cache = &model->cache[layer];
        for (int slot = 0; slot < cache->n; slot++) {
            free(cache->slots[slot].g);
            free(cache->slots[slot].gs);
        }
        free(cache->slot_by_expert);
        free(cache->slots);
    }
    free(model->last_access); free(model->is_queued); free(model->is_pinned);
    free(model->freq);
    free(model->momentum_logits);
    free(model->cache); free(model->L);
    st_destroy(&model->S);
}

static int olmoe_segment_engine_open(
    void **engine_impl, ColiSegmentCapabilities *capabilities,
    const ColiSegmentEngineOptions *options, char *error, size_t error_size) {
    if (!engine_impl || !capabilities || !options)
        return coli_segment_adapter_error(error, error_size,
                                           "invalid OLMoE Segment open");
    *engine_impl = NULL;
    if (options->backend_mask &&
        (options->backend_mask & ~COLI_SEGMENT_CAP_CPU))
        return coli_segment_adapter_error(error, error_size,
                                           "OLMoE Segment supports CPU only");
    if (options->context_tokens > 4096)
        return coli_segment_adapter_error(error, error_size,
                                           "OLMoE Segment context exceeds 4096");

    Cfg config;
    memset(&config, 0, sizeof(config));
    load_cfg(&config, options->model_dir);
    if (options->layer_end > (uint32_t)config.n_layers)
        return coli_segment_adapter_error(error, error_size,
                                           "OLMoE Segment range exceeds model");
    int range_layers = (int)(options->layer_end - options->layer_begin);
    int cap = 16;
    if (options->memory_limit_bytes) {
        uint64_t weights = (uint64_t)config.hidden * config.inter * 3u;
        uint64_t scales = (uint64_t)(config.inter * 2 + config.hidden) *
                          sizeof(float);
        uint64_t per_slot = weights + scales;
        uint64_t slots = per_slot && range_layers > 0
            ? options->memory_limit_bytes / per_slot / (uint64_t)range_layers
            : 0;
        cap = slots > (uint64_t)config.n_experts ? config.n_experts : (int)slots;
        if (cap < 1) cap = 1;
    }

    OlmoeSegmentEngine *engine = calloc(1, sizeof(*engine));
    if (!engine)
        return coli_segment_adapter_error(error, error_size,
                                           "out of memory opening OLMoE Segment");
    engine->layer_begin = options->layer_begin;
    engine->layer_end = options->layer_end;
    engine->context_tokens = options->context_tokens;
    if (pthread_mutex_init(&engine->run_lock, NULL)) {
        free(engine);
        return coli_segment_adapter_error(error, error_size,
                                           "cannot initialize OLMoE Segment lock");
    }
    model_init_range(&engine->model, options->model_dir, cap, 8,
                     (int)options->layer_begin, (int)options->layer_end, 0, 0);

    memset(capabilities, 0, sizeof(*capabilities));
    capabilities->struct_size = sizeof(*capabilities);
    capabilities->abi_version = COLI_SEGMENT_ABI_VERSION;
    capabilities->flags = COLI_SEGMENT_CAP_SNAPSHOT |
                          COLI_SEGMENT_CAP_RANGE_NATIVE |
                          COLI_SEGMENT_CAP_MULTI_SESSION |
                          COLI_SEGMENT_CAP_CPU;
    coli_segment_capability_string(capabilities->engine_id,
                                   sizeof(capabilities->engine_id), "olmoe");
    coli_segment_capability_string(capabilities->state_schema,
                                   sizeof(capabilities->state_schema),
                                   "olmoe/kv-f32-v1");
    coli_segment_capability_string(capabilities->numeric_class,
                                   sizeof(capabilities->numeric_class),
                                   "olmoe/f32-int8/cpu-v1");
    capabilities->state_dtype = COLI_SEGMENT_DTYPE_F32;
    capabilities->state_width = (uint32_t)config.hidden;
    capabilities->max_batch_rows = 128;
    capabilities->max_context_tokens = 4096;
    capabilities->num_layers = (uint32_t)config.n_layers;
    *engine_impl = engine;
    return 0;
}

static void olmoe_segment_engine_destroy(void *engine_impl) {
    OlmoeSegmentEngine *engine = (OlmoeSegmentEngine *)engine_impl;
    if (!engine) return;
    olmoe_segment_model_destroy(engine);
    pthread_mutex_destroy(&engine->run_lock);
    free(engine);
}

static int olmoe_segment_session_create(
    void *engine_impl, void **session_impl,
    const ColiSegmentSessionOptions *options, char *error, size_t error_size) {
    OlmoeSegmentEngine *engine = (OlmoeSegmentEngine *)engine_impl;
    if (!engine || !session_impl || !options)
        return coli_segment_adapter_error(error, error_size,
                                           "invalid OLMoE Segment session");
    *session_impl = NULL;
    OlmoeSegmentSession *session = calloc(1, sizeof(*session));
    if (!session)
        return coli_segment_adapter_error(error, error_size,
                                           "out of memory creating OLMoE session");
    session->engine = engine;
    session->context_tokens = options->context_tokens;
    int layers = engine->model.c.n_layers;
    session->K = calloc((size_t)layers, sizeof(*session->K));
    session->V = calloc((size_t)layers, sizeof(*session->V));
    if (!session->K || !session->V) goto oom;
    size_t cells;
    if (coli_segment_size_mul((size_t)engine->model.c.n_heads,
                              options->context_tokens, &cells) ||
        coli_segment_size_mul(cells, (size_t)engine->model.c.head_dim,
                              &cells)) goto oom;
    for (uint32_t layer = engine->layer_begin; layer < engine->layer_end;
         layer++) {
        session->K[layer] = calloc(cells, sizeof(float));
        session->V[layer] = calloc(cells, sizeof(float));
        if (!session->K[layer] || !session->V[layer]) goto oom;
    }
    *session_impl = session;
    return 0;

oom:
    if (session->K && session->V)
        for (uint32_t layer = engine->layer_begin; layer < engine->layer_end;
             layer++) {
            free(session->K[layer]); free(session->V[layer]);
        }
    free(session->K); free(session->V); free(session);
    return coli_segment_adapter_error(error, error_size,
                                       "out of memory allocating OLMoE KV");
}

static void olmoe_segment_session_destroy(void *session_impl) {
    OlmoeSegmentSession *session = (OlmoeSegmentSession *)session_impl;
    if (!session) return;
    for (uint32_t layer = session->engine->layer_begin;
         layer < session->engine->layer_end; layer++) {
        free(session->K[layer]); free(session->V[layer]);
    }
    free(session->K); free(session->V); free(session);
}

static int olmoe_segment_session_run(void *session_impl,
                                     const ColiSegmentRunRequest *request,
                                     char *error, size_t error_size) {
    OlmoeSegmentSession *session = (OlmoeSegmentSession *)session_impl;
    if (!session || !request || request->position != session->position)
        return coli_segment_adapter_error(
            error, error_size, "OLMoE Segment requires contiguous positions");
    if (request->should_cancel &&
        request->should_cancel(request->cancel_user_data))
        return coli_segment_adapter_error(error, error_size,
                                           "OLMoE Segment run cancelled");
    OlmoeSegmentEngine *engine = session->engine;
    if (request->output != request->input)
        memcpy(request->output, request->input, request->input_bytes);
    pthread_mutex_lock(&engine->run_lock);
    Model *model = &engine->model;
    model->K = session->K; model->V = session->V;
    model->max_t = (int)session->context_tokens;
    model->kv_len = (int)session->position;
    layers_forward_range(model, (float *)request->output, (int)request->rows,
                         (int)request->position, (int)engine->layer_begin,
                         (int)engine->layer_end, 0);
    model->K = NULL; model->V = NULL; model->max_t = 0; model->kv_len = 0;
    pthread_mutex_unlock(&engine->run_lock);
    session->position += request->rows;
    return 0;
}

static int olmoe_segment_payload_size(const OlmoeSegmentSession *session,
                                      uint32_t position, size_t *bytes) {
    size_t cells = (size_t)(session->engine->layer_end -
                            session->engine->layer_begin);
    if (coli_segment_size_mul(cells, 2, &cells) ||
        coli_segment_size_mul(cells,
                              (size_t)session->engine->model.c.n_heads,
                              &cells) ||
        coli_segment_size_mul(cells, position, &cells) ||
        coli_segment_size_mul(cells,
                              (size_t)session->engine->model.c.head_dim,
                              &cells) ||
        coli_segment_size_mul(cells, sizeof(float), bytes)) return -1;
    return 0;
}

static uint64_t olmoe_segment_state_hash(const OlmoeSegmentSession *session) {
    uint64_t hash = COLI_SEGMENT_HASH_INIT;
    size_t row_bytes = (size_t)session->position *
                       session->engine->model.c.head_dim * sizeof(float);
    int heads = session->engine->model.c.n_heads;
    size_t stride = (size_t)session->context_tokens *
                    session->engine->model.c.head_dim;
    for (uint32_t layer = session->engine->layer_begin;
         layer < session->engine->layer_end; layer++)
        for (int kv = 0; kv < 2; kv++) {
            float *state = kv ? session->V[layer] : session->K[layer];
            for (int head = 0; head < heads; head++)
                hash = coli_segment_hash_update(hash, state + head * stride,
                                                row_bytes);
        }
    return hash;
}

static int olmoe_segment_session_snapshot(
    void *session_impl, ColiSegmentWriteFn write_fn, void *write_user_data,
    char *error, size_t error_size) {
    OlmoeSegmentSession *session = (OlmoeSegmentSession *)session_impl;
    size_t payload_bytes;
    if (!session || olmoe_segment_payload_size(session, session->position,
                                               &payload_bytes))
        return coli_segment_adapter_error(error, error_size,
                                           "OLMoE snapshot size overflow");
    ColiSegmentSnapshotHeader header;
    coli_segment_snapshot_header_init(
        &header, "olmoe", session->engine->layer_begin,
        session->engine->layer_end, session->context_tokens, session->position,
        payload_bytes, olmoe_segment_state_hash(session));
    if (coli_segment_stream_write(write_fn, write_user_data, &header,
                                  sizeof(header), error, error_size)) return -1;
    size_t row_bytes = (size_t)session->position *
                       session->engine->model.c.head_dim * sizeof(float);
    int heads = session->engine->model.c.n_heads;
    size_t stride = (size_t)session->context_tokens *
                    session->engine->model.c.head_dim;
    for (uint32_t layer = session->engine->layer_begin;
         layer < session->engine->layer_end; layer++)
        for (int kv = 0; kv < 2; kv++) {
            float *state = kv ? session->V[layer] : session->K[layer];
            for (int head = 0; head < heads; head++)
                if (coli_segment_stream_write(
                        write_fn, write_user_data, state + head * stride,
                        row_bytes, error, error_size)) return -1;
        }
    return 0;
}

static int olmoe_segment_session_restore(
    void *session_impl, ColiSegmentReadFn read_fn, void *read_user_data,
    char *error, size_t error_size) {
    OlmoeSegmentSession *session = (OlmoeSegmentSession *)session_impl;
    ColiSegmentSnapshotHeader header;
    if (!session || coli_segment_stream_read(read_fn, read_user_data, &header,
                                             sizeof(header), error, error_size))
        return -1;
    size_t payload_bytes;
    if (olmoe_segment_payload_size(session, header.position, &payload_bytes) ||
        coli_segment_snapshot_header_valid(
            &header, "olmoe", session->engine->layer_begin,
            session->engine->layer_end, session->context_tokens, payload_bytes,
            error, error_size)) return -1;
    unsigned char *payload = payload_bytes ? malloc(payload_bytes) : NULL;
    if (payload_bytes && !payload)
        return coli_segment_adapter_error(error, error_size,
                                           "out of memory restoring OLMoE KV");
    if (coli_segment_stream_read(read_fn, read_user_data, payload, payload_bytes,
                                 error, error_size)) {
        free(payload);
        return -1;
    }
    if (coli_segment_hash_update(COLI_SEGMENT_HASH_INIT, payload,
                                 payload_bytes) != header.payload_hash) {
        free(payload);
        return coli_segment_adapter_error(error, error_size,
                                           "OLMoE snapshot checksum mismatch");
    }
    size_t row_bytes = (size_t)header.position *
                       session->engine->model.c.head_dim * sizeof(float);
    int heads = session->engine->model.c.n_heads;
    size_t stride = (size_t)session->context_tokens *
                    session->engine->model.c.head_dim;
    unsigned char *cursor = payload;
    for (uint32_t layer = session->engine->layer_begin;
         layer < session->engine->layer_end; layer++)
        for (int kv = 0; kv < 2; kv++) {
            float *state = kv ? session->V[layer] : session->K[layer];
            for (int head = 0; head < heads; head++) {
                memcpy(state + head * stride, cursor, row_bytes);
                cursor += row_bytes;
            }
        }
    session->position = header.position;
    free(payload);
    return 0;
}

static const ColiSegmentAdapter olmoe_segment_adapter = {
    sizeof(ColiSegmentAdapter), COLI_SEGMENT_ABI_VERSION, "olmoe",
    olmoe_segment_engine_open, olmoe_segment_engine_destroy,
    olmoe_segment_session_create, olmoe_segment_session_destroy,
    olmoe_segment_session_run, olmoe_segment_session_snapshot,
    olmoe_segment_session_restore, {0}
};

int coli_olmoe_segment_adapter_register(void) {
    return coli_segment_adapter_register(&olmoe_segment_adapter);
}
#endif /* COLI_SEGMENT_ADAPTER */

#ifdef COLI_EDGE_ADAPTER
/* ---------- engine-owned model Edge adapter --------------------------- */

typedef struct {
    Model model;
    Tok tokenizer;
} OlmoeEdgeEngine;

static void olmoe_edge_engine_destroy(void *engine_impl) {
    OlmoeEdgeEngine *engine = (OlmoeEdgeEngine *)engine_impl;
    if (!engine) return;
    free(engine->model.embed);
    free(engine->model.lm_head);
    free(engine->model.final_norm);
    st_destroy(&engine->model.S);
    tok_free(&engine->tokenizer);
    free(engine);
}

static int olmoe_edge_engine_open(
    void **engine_impl, ColiEdgeCapabilities *capabilities,
    const ColiEdgeEngineOptions *options, char *error, size_t error_size) {
    if (!engine_impl || !capabilities || !options)
        return coli_edge_adapter_error(error, error_size,
                                       "invalid OLMoE Edge open");
    *engine_impl = NULL;
    if (options->backend_mask &&
        (options->backend_mask & ~COLI_EDGE_CAP_CPU))
        return coli_edge_adapter_error(error, error_size,
                                       "OLMoE Edge supports CPU only");
    OlmoeEdgeEngine *engine = calloc(1, sizeof(*engine));
    if (!engine)
        return coli_edge_adapter_error(error, error_size,
                                       "out of memory opening OLMoE Edge");
    load_cfg(&engine->model.c, options->model_dir);
    st_init(&engine->model.S, options->model_dir);
    const Cfg *ec = &engine->model.c;
    engine->model.embed = load_t(&engine->model, "model.embed_tokens.weight", ec->vocab, ec->hidden);
    engine->model.lm_head = load_t(&engine->model, "lm_head.weight", ec->vocab, ec->hidden);
    engine->model.final_norm = load_t(&engine->model, "model.norm.weight", ec->hidden, 0);
    char tokenizer_path[4096];
    snprintf(tokenizer_path, sizeof(tokenizer_path), "%s/tokenizer.json",
             options->model_dir);
    tok_load(&engine->tokenizer, tokenizer_path);

    Cfg *config = &engine->model.c;
    uint64_t cells = (uint64_t)config->vocab * config->hidden;
    uint64_t resident = (2u * cells + (uint64_t)config->hidden) * sizeof(float);
    if (options->memory_limit_bytes && resident > options->memory_limit_bytes) {
        olmoe_edge_engine_destroy(engine);
        return coli_edge_adapter_error(error, error_size,
                                       "OLMoE Edge exceeds memory limit");
    }
    memset(capabilities, 0, sizeof(*capabilities));
    capabilities->struct_size = sizeof(*capabilities);
    capabilities->abi_version = COLI_EDGE_ABI_VERSION;
    capabilities->flags = COLI_EDGE_CAP_TOKENIZE |
                          COLI_EDGE_CAP_DETOKENIZE |
                          COLI_EDGE_CAP_GREEDY | COLI_EDGE_CAP_LOGITS |
                          COLI_EDGE_CAP_CPU;
    coli_edge_capability_string(capabilities->engine_id,
                                sizeof(capabilities->engine_id), "olmoe");
    coli_edge_capability_string(capabilities->state_schema,
                                sizeof(capabilities->state_schema),
                                "olmoe/kv-f32-v1");
    coli_edge_capability_string(capabilities->numeric_class,
                                sizeof(capabilities->numeric_class),
                                "olmoe/f32-int8/cpu-v1");
    coli_edge_capability_string(capabilities->tokenizer_class,
                                sizeof(capabilities->tokenizer_class),
                                "olmoe/byte-bpe-v1");
    capabilities->state_dtype = COLI_EDGE_DTYPE_F32;
    capabilities->state_width = (uint32_t)config->hidden;
    capabilities->vocab_size = (uint32_t)config->vocab;
    capabilities->max_batch_rows = 128;
    capabilities->max_context_tokens = 4096;
    capabilities->num_layers = (uint32_t)config->n_layers;
    capabilities->bos_token_id = -1;
    capabilities->eos_token_id = -1;
    capabilities->resident_bytes = resident;
    *engine_impl = engine;
    return 0;
}

static int olmoe_edge_tokenize(
    void *engine_impl, const char *text, size_t text_bytes,
    int32_t *token_ids, size_t token_capacity, size_t *token_count,
    char *error, size_t error_size) {
    OlmoeEdgeEngine *engine = (OlmoeEdgeEngine *)engine_impl;
    return coli_edge_tok_tokenize(&engine->tokenizer, text, text_bytes,
                                  token_ids, token_capacity, token_count,
                                  error, error_size);
}

static int olmoe_edge_detokenize(
    void *engine_impl, const int32_t *token_ids, size_t token_count,
    char *text, size_t text_capacity, size_t *text_bytes,
    char *error, size_t error_size) {
    OlmoeEdgeEngine *engine = (OlmoeEdgeEngine *)engine_impl;
    return coli_edge_tok_detokenize(&engine->tokenizer, token_ids, token_count,
                                    text, text_capacity, text_bytes,
                                    error, error_size);
}

static int olmoe_edge_embed(void *engine_impl,
                            const ColiEdgeEmbedRequest *request,
                            char *error, size_t error_size) {
    OlmoeEdgeEngine *engine = (OlmoeEdgeEngine *)engine_impl;
    Cfg *config = &engine->model.c;
    float *output = (float *)request->output;
    for (uint32_t row = 0; row < request->rows; row++) {
        int token = request->token_ids[row];
        if (token < 0 || token >= config->vocab)
            return coli_edge_adapter_error(error, error_size,
                                           "OLMoE token ID is out of range");
        memcpy(output + (size_t)row * config->hidden,
               engine->model.embed + (size_t)token * config->hidden,
               (size_t)config->hidden * sizeof(float));
    }
    return 0;
}

static int olmoe_edge_select(void *engine_impl,
                             const ColiEdgeSelectRequest *request,
                             char *error, size_t error_size) {
    OlmoeEdgeEngine *engine = (OlmoeEdgeEngine *)engine_impl;
    Cfg *config = &engine->model.c;
    float *normalized = falloc(config->hidden);
    float *logits = falloc(config->vocab);
    const float *input = (const float *)request->input;
    for (uint32_t row = 0; row < request->rows; row++) {
        if (request->should_cancel &&
            request->should_cancel(request->cancel_user_data)) {
            free(logits); free(normalized);
            return coli_edge_adapter_error(error, error_size,
                                           "OLMoE Edge selection cancelled");
        }
        rmsnorm_row(normalized, input + (size_t)row * config->hidden,
                    engine->model.final_norm, config->hidden, config->eps);
        matmul(logits, normalized, engine->model.lm_head,
               1, config->hidden, config->vocab);
        if (coli_edge_argmax(logits, (uint32_t)config->vocab,
                            &request->token_ids[row],
                            request->scores ? &request->scores[row] : NULL)) {
            free(logits); free(normalized);
            return coli_edge_adapter_error(error, error_size,
                                           "OLMoE Edge head failed");
        }
    }
    free(logits); free(normalized);
    return 0;
}

static int olmoe_edge_logits(void *engine_impl,
                             const ColiEdgeLogitsRequest *request,
                             char *error, size_t error_size) {
    OlmoeEdgeEngine *engine = (OlmoeEdgeEngine *)engine_impl;
    Cfg *config = &engine->model.c;
    float *normalized = falloc(config->hidden);
    const float *input = (const float *)request->input;
    for (uint32_t row = 0; row < request->rows; row++) {
        if (request->should_cancel &&
            request->should_cancel(request->cancel_user_data)) {
            free(normalized);
            return coli_edge_adapter_error(error, error_size,
                                           "OLMoE Edge logits cancelled");
        }
        rmsnorm_row(normalized, input + (size_t)row * config->hidden,
                    engine->model.final_norm, config->hidden, config->eps);
        matmul(request->logits + (size_t)row * config->vocab,
               normalized, engine->model.lm_head,
               1, config->hidden, config->vocab);
    }
    free(normalized);
    return 0;
}

static const ColiEdgeAdapter olmoe_edge_adapter = {
    sizeof(ColiEdgeAdapter), COLI_EDGE_ABI_VERSION, "olmoe",
    olmoe_edge_engine_open, olmoe_edge_engine_destroy,
    olmoe_edge_tokenize, olmoe_edge_detokenize,
    olmoe_edge_embed, olmoe_edge_select, olmoe_edge_logits, {0}
};

int coli_olmoe_edge_adapter_register(void) {
    return coli_edge_adapter_register(&olmoe_edge_adapter);
}
#endif /* COLI_EDGE_ADAPTER */