Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
91.3 KB
Raw Blame
/* vk_tier.c -- the Vulkan routed-expert tier shared by the MoE engines. See
 * vk_tier.h for what it does and how an engine calls it.
 *
 * Threads. Everything runs on the engine thread except two things: the uploader
 * thread, which turns staged copies of promoted experts into device tensors, and
 * vkt_put, which a warm start may call from many loader threads. Neither touches
 * a slot: they hand finished tensors back through the `done` list, and the engine
 * thread makes them resident at a quiescent point (no batch in flight), where it
 * also frees evicted experts. So the slot table, the counters and every Vulkan
 * call but the pool's allocation are single-threaded; the lock guards the queue,
 * the done list, the room-made signal and the count of victims not freed yet
 * only.
 *
 * Two devices. With COLI_VK_DEV2 the tier holds experts on a second device too
 * (backend_vulkan.c's batch on that device, its own budget): a slot says which
 * device holds it, the warm start fills the primary device with the hottest experts
 * and the second with the next ones, a promotion goes where there is room or takes
 * the coldest resident's place on either device, and a step sends each device its
 * own batch, both in flight at once. Streaming (big prefill steps' cold experts)
 * stays on the primary device. A second device that fails gives its experts back
 * to the CPU and the tier goes on with the primary one. */
#ifdef COLI_VULKAN
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <time.h>
#include "compat.h"
#include <errno.h>
#include "vk_tier.h"
#include "backend_vulkan.h"
#include "tier.h"

#define VKT_QCAP 32                 /* promotions staged and waiting for the uploader */
#define VKT_CAND 64                 /* victim candidates kept per refresh */
#define VKT_DECAY_TOKENS 1024u      /* tier.h's decay period, in tokens */

enum { VS_NONE = 0, VS_QUEUED, VS_RESIDENT, VS_EVICT };

typedef struct {
    ColiVkExpert *ex;
    uint32_t heat, last;
    uint8_t state;
    uint8_t dev;                       /* the device it is on or on its way to (0, or 1: COLI_VK_DEV2's) */
} VSlot;
typedef struct { int layer, eid, dev; uint8_t *buf; } VQ;
typedef struct { int layer, eid, ok, copy_failed; ColiVkTensor *g, *u, *d; } VDone;
/* streaming: a staging slot (an expert's three tensors on the device, filled again for
 * every expert it carries) and a sub-batch in flight */
enum { SS_FREE = 0, SS_FILLED, SS_BUSY };
typedef struct { ColiVkTensor *t[3]; ColiVkExpert *ex; int layer, eid, state, pending, pref; } VStage;
typedef struct { int n, nslot, cap, *slot; float **yout; } VSub;

static struct {
    int on;
    VktConfig c;
    char engine[32];
    /* the forms of the experts, as the engine holds them and on the device: [0] the main
     * layers', [1] the extra layers' (VktConfig.extra_layers, an MTP head's), from
     * main_layers on */
    int main_layers;
    struct { VktFmt gu, dn; int gu_fmt, gu_gs, dn_fmt, dn_gs; size_t gu_codes, gu_scales, d_codes, d_scales, stage_bytes; } form[2];
    /* the extra layers' experts: on the primary device, in a pool of their own (another
     * size than the main ones', so neither leaves holes the other cannot use), at most
     * xmax of x_bytes each in x_budget */
    int xmax, xres, xque;
    size_t x_bytes, x_budget;
    unsigned long long extra_served, extra_routed;
    size_t exp_bytes, budget;
    int max_resident, resident, queued, rate, uma;   /* totals over the devices */
    /* per device: the most it holds, resident, queued (dmax[1] = 0 without a second one) */
    int dmax[2], dres[2], dque[2];
    /* the second device (COLI_VK_DEV2) */
    int d2;                            /* the tier holds experts there */
    char d2_name[64];
    size_t d2_budget, d2_exp_bytes;
    int d2_rows;                       /* the most rows of a big step its one batch takes */
    int d0_inflight, d2_inflight, d2_rows0;   /* a batch in flight on each; where the second's rows start in by[] */
    int nd2; int *d2_idx; int cd2;     /* a big step's rows on it: their assignments */
    const float **d2_y; ColiVkExpert **d2_bex; int *d2_brows; const float **d2_bx; float *d2_bw; int d2_cap, d2_cexp;
    unsigned long long d2_served, d2_steps, d2_big, d2_uploads, d2_lost;
    /* exclusive RAM/VRAM (vkt_ram_first): on, and the RAM slots given up for it. routed[l*E+e]
     * is the issue (seq) that last routed the expert, layer_seq[l] the layer's last issue: an
     * expert routed in its layer's current step may be one the CPU is computing from its RAM
     * slot (handed back by the balance, past a batch's rows), so it is never offered */
    int excl; unsigned long long ram_gave;
    uint32_t *routed_at, *layer_seq, seq;
    double d2_ms;
    VSlot *s;
    uint32_t tick, decay_at;          /* tokens seen (rows of the forward's first layer) */
    int last_layer, first_layer, promos[2], promo_cap;   /* promotions this forward, the main layers' and the extra ones', each up to the cap */
    int begin;                         /* vkt_begin_forward: the next issue starts a forward */
    /* uploader */
    pthread_t th; int th_on, stop;
    pthread_mutex_t mx; pthread_cond_t cv, cv_room, cv_done;
    int busy;                          /* the uploader is between taking an entry and push_done */
    int sync;                          /* COLI_VK_TIER_SYNC=1: quiescent points wait for the uploader */
    int evict_pending;                 /* victims chosen and not freed yet (a batch is in flight) */
    VQ q[VKT_QCAP]; int qh, qn;
    VDone *done; int ndone, cdone;
    unsigned long room_gen;
    /* evicted, freed at the next quiescent point */
    int *evict; int nevict, cevict;
    /* the coldest residents, refreshed once per forward */
    int cand[2][VKT_CAND]; uint64_t cand_score[2][VKT_CAND]; int ncand[2];   /* per kind: main, extra */
    /* the step in flight */
    int inflight, S, K;
    int *map; int cmap;
    int *grp; int *touched;              /* expert -> group of this step, and the groups' experts */
    ColiVkExpert **bex; int *brows; int *bfirst; int cb;
    const float **bx, **by; int *rowsrc; int cbx;
    float *bw;                           /* the rows' route weights (vkt_issue_w), beside bx */
    int max_dev_rows;
    double t_issued;
    /* the balance: the share of a step's resident experts the device takes, moved by
     * what each join waited (the device was the slower side) or did not */
    int balance, can_balance; float share; unsigned long long handed;
    /* accounting */
    unsigned long long routed, served, steps, uploads, upload_bytes, evictions, qfull, rated, refused, failed, warm;
    double dev_ms, cpu_ms, wait_ms;
    unsigned long long rep_routed, rep_served;
    /* streaming (big prefill steps; see "streaming" below) */
    int st_ok, st_slots, st_n, st_per, st_half, st_rows_env, gemm_rows, st_par;
    VStage *st;
    double bw_up, cpu_row_ms;             /* measured: bytes per ms into a slot; CPU ms per (row, expert) */
    int big, big_fail, big_n, big_taken, said_fwd;
    unsigned sk;                          /* sub-batches issued in this step */
    VSub hf[2];                           /* the two in flight */
    float *sy; size_t csy;                /* the step's device rows, S*K*hidden */
    float *sy_step;                       /* the backend's whole-step rows instead (coli_vk_xb_step_begin), or NULL */
    int big_valid;                        /* the step's routed assignments (idx >= 0) */
    int *bcnt, *bofs, *blist, *bcls, cblist; /* per expert of the step: rows, first row, class; the rows grouped */
    uint32_t *pred; uint8_t *pred_ok;     /* per layer: the last big step's rows per expert */
    unsigned long long st_steps, st_experts, st_rows, st_bytes, st_subs, st_sums, st_kept, st_kept_rows, pf_n, pf_used;
    double st_up_ms;
} T;

/* "2.31 GiB", "35.0 MiB", "9.5 KiB": the budget lines read on a 24 GB card and a tiny fixture alike */
static const char *human(double b, char *buf, size_t n) {
    if (b >= 1073741824.0) snprintf(buf, n, "%.2f GiB", b / 1073741824.0);
    else if (b >= 1048576.0) snprintf(buf, n, "%.1f MiB", b / 1048576.0);
    else snprintf(buf, n, "%.1f KiB", b / 1024.0);
    return buf;
}
static double vkt_now_ms(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec * 1e3 + t.tv_nsec / 1e6; }
static VSlot *slot(int layer, int eid) { return &T.s[(size_t)layer * T.c.experts + eid]; }
static int kind_of(int layer) { return layer >= T.main_layers; }   /* the form of a layer's experts */

/* ---- source formats ---------------------------------------------------------- */
static int src_per_row(VktSrc k) {
    return k == VKT_SRC_I8_ROW || k == VKT_SRC_I8_AS_I4_ROW || k == VKT_SRC_I4S_PAIRS_ROW || k == VKT_SRC_I4U_PAIRS_ROW;
}
static size_t src_row_bytes(VktSrc k, int I) {
    switch (k) {
    case VKT_SRC_I8_ROW: case VKT_SRC_I8_GS: case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I8_AS_I4_GS:
    case VKT_SRC_FP8_GS: case VKT_SRC_FP8_BLOCK: return (size_t)I;
    case VKT_SRC_I3_G64: return ((size_t)I + 63) / 64 * 24;
    case VKT_SRC_BF16: return (size_t)I * 2;
    case VKT_SRC_F32: return (size_t)I * 4;
    default: return ((size_t)I + 1) / 2;                 /* the int4 and fp4 kinds */
    }
}
static size_t src_scale_bytes(VktFmt f, int I, int O) {
    if (f.kind == VKT_SRC_BF16 || f.kind == VKT_SRC_F32) return 0;
    if (src_per_row(f.kind)) return (size_t)O * 4;
    if (f.kind == VKT_SRC_I4U_PLANAR64 || f.kind == VKT_SRC_I3_G64) return (size_t)O * (((size_t)I + 63) / 64) * 4;
    size_t ng = ((size_t)I + f.gs - 1) / f.gs;
    if (f.kind == VKT_SRC_MXFP4_E8M0) return (size_t)O * ng;
    if (f.kind == VKT_SRC_FP8_BLOCK) return (((size_t)O + f.gs - 1) / f.gs) * ng * 4;
    return (size_t)O * ng * 4;
}
/* The backend format a source kind lands in; 0 when the combination is not one. */
static int dev_fmt(VktFmt f, int *fmt, int *gs) {
    int g = f.gs;
    switch (f.kind) {
    case VKT_SRC_I8_ROW: *fmt = 1; *gs = 0; return 1;
    case VKT_SRC_I8_GS: *fmt = 13; *gs = g; return g >= 4 && g % 4 == 0;
    case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I4S_PAIRS_ROW: case VKT_SRC_I4U_PAIRS_ROW: *fmt = 2; *gs = 0; return 1;
    case VKT_SRC_I8_AS_I4_GS: case VKT_SRC_I4S_PAIRS_GS: case VKT_SRC_I4U_PAIRS_GS: *fmt = 4; *gs = g; return g >= 8 && g % 8 == 0;
    case VKT_SRC_I4U_PLANAR64: *fmt = 4; *gs = 64; return 1;
    case VKT_SRC_I3_G64: *fmt = 5; *gs = 0; return 1;
    case VKT_SRC_MXFP4_F32: case VKT_SRC_MXFP4_E8M0: *fmt = 7; *gs = g; return g >= 8 && g % 8 == 0;
    case VKT_SRC_FP8_GS: case VKT_SRC_FP8_BLOCK: *fmt = 12; *gs = g; return g >= 4 && g % 4 == 0;
    case VKT_SRC_BF16: *fmt = 11; *gs = 0; return 1;
    case VKT_SRC_F32: *fmt = 10; *gs = 0; return 1;
    default: return 0;
    }
}

/* One matrix [O x I] from its RAM form into a device tensor's mapping: rows at
 * `stride` bytes (already zeroed past the row), scales as the backend reads them.
 * Pure byte work; the values are never requantized. */
static void convert_par(VktFmt f, int I, int O, const uint8_t *codes, const void *scales,
                        uint8_t *rows, size_t stride, float *sc, int par);
static void convert(VktFmt f, int I, int O, const uint8_t *codes, const void *scales,
                    uint8_t *rows, size_t stride, float *sc) {
    convert_par(f, I, O, codes, scales, rows, stride, sc, 0);
}
/* par: the rows split over the OpenMP threads (streaming, on the engine thread) */
static void convert_par(VktFmt f, int I, int O, const uint8_t *codes, const void *scales,
                        uint8_t *rows, size_t stride, float *sc, int par) {
    size_t rb = src_row_bytes(f.kind, I);
    par = par && O >= 64;
    switch (f.kind) {
    case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I8_AS_I4_GS:   /* int8 in [-8,7] -> v+8 nibble pairs */
        #pragma omp parallel for schedule(static) if (par)
        for (int o = 0; o < O; o++) {
            const int8_t *src = (const int8_t *)codes + (size_t)o * I;
            uint8_t *dst = rows + (size_t)o * stride;
            for (int j = 0; j < I / 2; j++)
                dst[j] = (uint8_t)(((src[2 * j] + 8) & 15) | (((src[2 * j + 1] + 8) & 15) << 4));
            if (I & 1) dst[I / 2] = (uint8_t)((src[I - 1] + 8) & 15);
        }
        break;
    case VKT_SRC_I4S_PAIRS_ROW: case VKT_SRC_I4S_PAIRS_GS:  /* two's complement nibble -> v+8: xor 8 */
        #pragma omp parallel for schedule(static) if (par)
        for (int o = 0; o < O; o++) {
            const uint8_t *src = codes + (size_t)o * rb;
            uint8_t *dst = rows + (size_t)o * stride;
            for (size_t j = 0; j < rb; j++) dst[j] = src[j] ^ 0x88u;
        }
        break;
    case VKT_SRC_I4U_PLANAR64:   /* planar blocks (byte k = elements k, k+32) -> pairs (2j, 2j+1) */
        #pragma omp parallel for schedule(static) if (par)
        for (int o = 0; o < O; o++) {
            const uint8_t *src = codes + (size_t)o * rb;
            uint8_t *dst = rows + (size_t)o * stride;
            int nb = I / 64;
            for (int b = 0; b < nb; b++) {
                const uint8_t *blk = src + (size_t)b * 32;
                for (int p = 0; p < 32; p++) {
                    int e0 = 2 * p, e1 = 2 * p + 1;
                    unsigned a = e0 < 32 ? (blk[e0] & 15u) : (unsigned)(blk[e0 - 32] >> 4);
                    unsigned c = e1 < 32 ? (blk[e1] & 15u) : (unsigned)(blk[e1 - 32] >> 4);
                    dst[(size_t)b * 32 + p] = (uint8_t)(a | (c << 4));
                }
            }
            memcpy(dst + (size_t)nb * 32, src + (size_t)nb * 32, rb - (size_t)nb * 32);   /* tail: pairs already */
        }
        break;
    default:   /* the same bytes */
        #pragma omp parallel for schedule(static) if (par)
        for (int o = 0; o < O; o++) memcpy(rows + (size_t)o * stride, codes + (size_t)o * rb, rb);
        break;
    }
    if (f.kind == VKT_SRC_BF16 || f.kind == VKT_SRC_F32) { sc[0] = 1.0f; return; }
    if (f.kind == VKT_SRC_MXFP4_E8M0) {        /* ue8m0 -> f32 2^(s-127), quant.h's mx4_scale */
        size_t n = (size_t)O * (((size_t)I + f.gs - 1) / f.gs);
        const uint8_t *e = scales;
        for (size_t i = 0; i < n; i++) { uint32_t u = (uint32_t)e[i] << 23; memcpy(&sc[i], &u, 4); }
        return;
    }
    if (f.kind == VKT_SRC_FP8_BLOCK) {         /* [O/bs][I/bs] blocks -> one row each */
        size_t nb = ((size_t)I + f.gs - 1) / f.gs;
        const float *b = scales;
        for (int o = 0; o < O; o++) memcpy(sc + (size_t)o * nb, b + (size_t)(o / f.gs) * nb, nb * 4);
        return;
    }
    memcpy(sc, scales, src_scale_bytes(f, I, O));
}

/* Upload one expert from its RAM form to device dev's pool (any thread). 0 when the pool
 * refused, -1 when the copy to the device failed (staged uploads: the room was there,
 * and is given back). */
static int upload(int dev, int kind, const uint8_t *g, const uint8_t *u, const uint8_t *d,
                  const void *gs, const void *us, const void *ds, ColiVkTensor *t[3]) {
    const int H = T.c.hidden, F = T.c.inter;
    const __typeof__(T.form[0]) *fm = &T.form[kind];
    const uint8_t *codes[3] = {g, u, d}; const void *sc[3] = {gs, us, ds};
    t[0] = t[1] = t[2] = NULL;
    for (int k = 0; k < 3; k++) {
        VktFmt f = k < 2 ? fm->gu : fm->dn;
        int fmt = k < 2 ? fm->gu_fmt : fm->dn_fmt, fgs = k < 2 ? fm->gu_gs : fm->dn_gs;
        int I = k < 2 ? H : F, O = k < 2 ? F : H;
        uint8_t *rows; size_t stride; float *scales;
        if (!(kind ? coli_vk_tier_tensor_extra(&t[k], fmt, I, O, fgs, &rows, &stride, &scales)   /* its own pool */
                   : coli_vk_tier_tensor_dev(dev, &t[k], fmt, I, O, fgs, &rows, &stride, &scales))) {
            for (int j = 0; j < k; j++) { coli_vk_tensor_free(t[j]); t[j] = NULL; }
            return 0;
        }
        convert(f, I, O, codes[k], sc[k], rows, stride, scales);
    }
    /* staged uploads: the three host images go to device memory now (nothing to do on
     * mapped memory) */
    if (!coli_vk_tensor_commit(t, 3)) {
        for (int k = 0; k < 3; k++) { coli_vk_tensor_free(t[k]); t[k] = NULL; }
        return -1;
    }
    return 1;
}

/* One expert's bytes on a device whose weight ranges start at `a` bytes. */
static size_t expert_bytes_at(int H, int F, VktFmt gu, VktFmt dn, size_t a) {
    int gf, gg, df, dg;
    if (!dev_fmt(gu, &gf, &gg) || !dev_fmt(dn, &df, &dg)) return 0;
    size_t b = 0;
    for (int k = 0; k < 3; k++) {
        int fmt = k < 2 ? gf : df, gs = k < 2 ? gg : dg, I = k < 2 ? H : F, O = k < 2 ? F : H;
        size_t rows = (coli_vk_tensor_row_bytes(fmt, I) + 3) / 4 * 4 * (size_t)O;
        size_t sc = coli_vk_tensor_scale_count(fmt, I, O, gs) * 4;
        b += (rows + a - 1) / a * a + (sc + a - 1) / a * a;   /* two ranges, each aligned */
    }
    return b;
}
size_t vkt_expert_bytes(int H, int F, VktFmt gu, VktFmt dn) {
    return expert_bytes_at(H, F, gu, dn, coli_vk_buffer_alignment());
}

/* ---- the uploader -------------------------------------------------------------- */
static void push_done(int layer, int eid, int ok, int copy_failed, ColiVkTensor *t[3]) {
    pthread_mutex_lock(&T.mx);
    if (T.ndone == T.cdone) {
        int nc = T.cdone ? 2 * T.cdone : 64;
        VDone *n = realloc(T.done, (size_t)nc * sizeof(*n));
        if (!n) {   /* cannot even record it: the slot stays queued and is never used */
            T.busy = 0; pthread_cond_broadcast(&T.cv_done);
            pthread_mutex_unlock(&T.mx);
            for (int k = 0; k < 3; k++) if (t[k]) coli_vk_tensor_free(t[k]);
            return;
        }
        T.done = n; T.cdone = nc;
    }
    T.done[T.ndone++] = (VDone){layer, eid, ok, copy_failed, t[0], t[1], t[2]};
    T.busy = 0; pthread_cond_broadcast(&T.cv_done);
    pthread_mutex_unlock(&T.mx);
}

static void *uploader(void *arg) {
    (void)arg;
    for (;;) {
        pthread_mutex_lock(&T.mx);
        while (!T.qn && !T.stop) pthread_cond_wait(&T.cv, &T.mx);
        /* COLI_VK_TIER_SYNC=1: a victim chosen while a batch is in flight is freed at that
         * batch's join, before the join waits for the uploads staged so far: wait for
         * that free here, so the promotion that displaced it finds its room. Uploading
         * first would meet the full pool, and a refusal is final in this mode. */
        while (T.sync && T.evict_pending && !T.stop) pthread_cond_wait(&T.cv_room, &T.mx);
        if (T.stop) { pthread_mutex_unlock(&T.mx); return NULL; }
        VQ e = T.q[T.qh]; T.qh = (T.qh + 1) % VKT_QCAP; T.qn--; T.busy = 1;
        pthread_mutex_unlock(&T.mx);
        const uint8_t *b = e.buf;
        const int kd = kind_of(e.layer);
        const size_t guc = T.form[kd].gu_codes, dc = T.form[kd].d_codes, gus = T.form[kd].gu_scales;
        const uint8_t *g = b, *u = g + guc, *d = u + guc;
        const uint8_t *gs = d + dc, *us = gs + gus, *ds = us + gus;
        ColiVkTensor *t[3];
        int ok = 0, r = 0;
        /* Refused: the pool is at its budget until the engine thread frees the victim
         * this promotion displaced, at its next quiescent point (a join: every layer
         * in decode, after a chunk's CPU work in prefill). Wait for room to be made
         * and try again; refused again after three frees, or nothing freed for a
         * minute (the engine stopped stepping), the pool is full for real. */
        for (int frees = 0, waited = 0; !ok; ) {
            r = upload(e.dev, kd, g, u, d, gs, us, ds, t);
            ok = r > 0;
            /* sync: every free decided so far came first (above), and the engine now waits
             * on us, not we on it: the pool is full for real. A failed copy is no question
             * of room: no waiting for one. */
            if (ok || r < 0 || frees >= 3 || waited >= 600 || T.sync) break;
            pthread_mutex_lock(&T.mx);
            unsigned long gen = T.room_gen;
            while (T.room_gen == gen && !T.stop && waited < 600) {
                struct timespec until; clock_gettime(CLOCK_REALTIME, &until);
                until.tv_nsec += 100 * 1000000L; if (until.tv_nsec >= 1000000000L) { until.tv_sec++; until.tv_nsec -= 1000000000L; }
                if (pthread_cond_timedwait(&T.cv_room, &T.mx, &until) == ETIMEDOUT) waited++;
            }
            if (T.room_gen != gen) frees++;
            int stop = T.stop;
            pthread_mutex_unlock(&T.mx);
            if (stop) break;
        }
        free(e.buf);
        if (!ok) t[0] = t[1] = t[2] = NULL;
        push_done(e.layer, e.eid, ok, r < 0, t);
    }
}

/* ---- quiescent points: nothing in flight ---------------------------------------- */
static void quiesce(void) {
    if (T.inflight) return;
    int freed = 0, had = T.nevict;
    for (int i = 0; i < T.nevict; i++) {
        VSlot *v = &T.s[T.evict[i]];
        if (v->ex) { coli_vk_xb_expert_free(v->ex); v->ex = NULL; freed = 1; }
        v->state = VS_NONE;
    }
    T.nevict = 0;
    if (had) {   /* room made: an upload may be waiting for it */
        pthread_mutex_lock(&T.mx);
        if (freed) T.room_gen++;
        T.evict_pending = 0;
        pthread_cond_broadcast(&T.cv_room);
        pthread_mutex_unlock(&T.mx);
    }
    /* finished uploads become resident; COLI_VK_TIER_SYNC=1 waits for every staged one
     * first, so what is resident depends on the routing alone, not on thread timing
     * (tests: a fixture's whole run can be over before the uploader is scheduled) */
    pthread_mutex_lock(&T.mx);
    if (T.sync && T.th_on) while ((T.qn || T.busy) && !T.stop) pthread_cond_wait(&T.cv_done, &T.mx);
    int n = T.ndone; VDone *d = n ? malloc((size_t)n * sizeof(*d)) : NULL;
    if (d) { memcpy(d, T.done, (size_t)n * sizeof(*d)); T.ndone = 0; } else n = 0;
    pthread_mutex_unlock(&T.mx);
    for (int i = 0; i < n; i++) {
        VSlot *v = slot(d[i].layer, d[i].eid);
        int dv = v->dev, kd = kind_of(d[i].layer);
        T.queued--; if (kd) T.xque--; else T.dque[dv]--;
        /* an upload to a second device that has stopped meanwhile (d2_stop) is dropped */
        ColiVkExpert *e = d[i].ok && (dv == 0 || T.d2) ? coli_vk_xb_expert(d[i].g, d[i].u, d[i].d) : NULL;
        if (e) {
            v->ex = e; v->state = VS_RESIDENT; T.resident++; if (kd) T.xres++; else T.dres[dv]++;
            T.uploads++; T.upload_bytes += coli_vk_tensor_bytes(d[i].g) + coli_vk_tensor_bytes(d[i].u) + coli_vk_tensor_bytes(d[i].d);
            if (dv) T.d2_uploads++;
        } else {
            for (int k = 0; k < 3; k++) { ColiVkTensor *t = k == 0 ? d[i].g : k == 1 ? d[i].u : d[i].d; if (t) coli_vk_tensor_free(t); }
            v->state = VS_NONE; v->dev = 0;
            if (dv && !T.d2) continue;
            T.failed++;
            /* refused: the budget holds fewer than the count said (fragmentation, or the
             * device ran out first): stop planning past what is there. A failed copy gave
             * its room back: the budget stands, the expert may come again. */
            if (!d[i].copy_failed && kd && T.xmax > T.xres + T.xque) T.xmax = T.xres + T.xque;
            if (!d[i].copy_failed && !kd && T.dmax[dv] > T.dres[dv] + T.dque[dv]) {
                T.max_resident -= T.dmax[dv] - (T.dres[dv] + T.dque[dv]);
                T.dmax[dv] = T.dres[dv] + T.dque[dv];
            }
        }
    }
    free(d);
}

/* The second device stopped (a batch there failed, or the device did): its residents
 * are freed (nothing is in flight there when this runs) and go back to the CPU, uploads
 * on their way there are dropped when they land, and the tier goes on with the primary
 * device. */
static void d2_stop(const char *why) {
    if (!T.d2) return;
    T.d2 = 0; T.d2_lost++;
    size_t n = (size_t)T.c.layers * T.c.experts;
    for (size_t i = 0; i < n; i++) {
        VSlot *v = &T.s[i];
        if (v->dev != 1 || v->state != VS_RESIDENT) continue;
        if (v->ex) { coli_vk_xb_expert_free(v->ex); v->ex = NULL; }
        v->state = VS_NONE; v->dev = 0; T.resident--; T.dres[1]--;
    }
    T.max_resident -= T.dmax[1]; T.dmax[1] = 0;
    fprintf(stderr, "[VK] tier %s: %s, its experts go back to the CPU and the tier goes on with %s\n",
            T.engine, why, coli_vk_device_name());
}

/* The device a newcomer of layer `layer` goes to when there is room: a main layer's to
 * the primary device first, then the second one; an extra layer's to its pool on the
 * primary device; -1 when they are full. */
static int room_dev(int layer) {
    if (kind_of(layer)) return T.xres + T.xque < T.xmax ? 0 : -1;
    if (T.dres[0] + T.dque[0] < T.dmax[0]) return 0;
    if (T.d2 && T.dres[1] + T.dque[1] < T.dmax[1]) return 1;
    return -1;
}

/* ---- placement ------------------------------------------------------------------ */
static uint64_t score(const VSlot *v) { return tier_lfru_score(v->heat, v->last, T.tick); }

/* The coldest residents of each kind, ascending, once per forward. */
static void refresh_candidates(void) {
    T.ncand[0] = T.ncand[1] = 0;
    size_t n = (size_t)T.c.layers * T.c.experts;
    for (size_t i = 0; i < n; i++) {
        VSlot *v = &T.s[i];
        if (v->state != VS_RESIDENT) continue;
        int k = kind_of((int)(i / (size_t)T.c.experts)), *c = T.cand[k];
        uint64_t sc = score(v), *cs = T.cand_score[k];
        if (T.ncand[k] == VKT_CAND && sc >= cs[VKT_CAND - 1]) continue;
        int at = T.ncand[k] < VKT_CAND ? T.ncand[k]++ : VKT_CAND - 1;
        while (at > 0 && cs[at - 1] > sc) { c[at] = c[at - 1]; cs[at] = cs[at - 1]; at--; }
        c[at] = (int)i; cs[at] = sc;
    }
}
/* The resident a newcomer of kind k and score hs would displace, or -1: the coldest
 * candidate of its kind (each kind has its own pool) still resident, when the newcomer
 * beats it by tier.h's LFRU margin (25% + 4 counts). It stays first in the list until
 * drop_victim takes it. */
static void drop_victim(int k) {
    memmove(T.cand[k], T.cand[k] + 1, (size_t)--T.ncand[k] * sizeof(int));
    memmove(T.cand_score[k], T.cand_score[k] + 1, (size_t)T.ncand[k] * sizeof(uint64_t));
}
static int peek_victim(int k, uint64_t hs) {
    while (T.ncand[k]) {
        int i = T.cand[k][0];
        VSlot *v = &T.s[i];
        if (v->state != VS_RESIDENT) { drop_victim(k); continue; }
        uint64_t cs = score(v);
        return hs > cs + (cs >> 2) + (4u << 8) ? i : -1;
    }
    return -1;
}

/* A forward starts when the layer index goes back (a layer that issues several
 * blocks of rows, or rows one by one, is the same forward). The tick counts tokens:
 * the rows of the forward's first layer, every block of them. */
static void new_forward(void) {
    T.promos[0] = T.promos[1] = 0; T.promo_cap = 0; T.said_fwd = 0;
    if (T.tick >= T.decay_at) {
        size_t n = (size_t)T.c.layers * T.c.experts;
        for (size_t i = 0; i < n; i++) T.s[i].heat = tier_decay_value(T.s[i].heat);
        T.decay_at = T.tick + VKT_DECAY_TOKENS;
    }
    refresh_candidates();
}

/* The bytes of an expert the CPU just computed pass by: promote it when there is
 * room, or when it is hot enough to displace the coldest resident. The copy is the
 * only cost on this thread; the uploader does the rest. */
void vkt_note(int layer, int eid, const VktExpertSrc *src) {
    if (!T.on || !src || layer < 0 || layer >= T.c.layers || eid < 0 || eid >= T.c.experts) return;
    VSlot *v = slot(layer, eid);
    if (v->state != VS_NONE || !src->g || !src->u || !src->d) return;
    /* the extra layers (an MTP head's) come last in a forward: their promotions are
     * counted apart, or the model's layers would always have spent the rate first */
    const int kd = kind_of(layer);
    if (T.promos[kd] >= T.promo_cap) { T.rated++; return; }
    const __typeof__(T.form[0]) *fm = &T.form[kd];
    if (fm->gu_scales && (!src->gs || !src->us || !src->ds)) return;
    int victim = -1, dev = room_dev(layer);
    if (dev < 0 && (!v->heat || (victim = peek_victim(kd, score(v))) < 0)) return;
    if (victim >= 0) dev = T.s[victim].dev;   /* the newcomer takes its place, on its device */
    pthread_mutex_lock(&T.mx);
    int qfull = T.qn >= VKT_QCAP;
    pthread_mutex_unlock(&T.mx);
    if (qfull) { T.qfull++; return; }
    uint8_t *buf = malloc(fm->stage_bytes);
    if (!buf) return;
    if (victim >= 0) drop_victim(kd);
    uint8_t *p = buf;
    memcpy(p, src->g, fm->gu_codes); p += fm->gu_codes;
    memcpy(p, src->u, fm->gu_codes); p += fm->gu_codes;
    memcpy(p, src->d, fm->d_codes);  p += fm->d_codes;
    if (fm->gu_scales) { memcpy(p, src->gs, fm->gu_scales); p += fm->gu_scales; memcpy(p, src->us, fm->gu_scales); p += fm->gu_scales; }
    if (fm->d_scales) memcpy(p, src->ds, fm->d_scales);
    if (victim >= 0) {
        VSlot *w = &T.s[victim];
        w->state = VS_EVICT; T.resident--; if (kd) T.xres--; else T.dres[w->dev]--; T.evictions++;
        if (T.nevict == T.cevict) {
            int nc = T.cevict ? 2 * T.cevict : 64;
            int *n = realloc(T.evict, (size_t)nc * sizeof(int));
            if (!n) { w->state = VS_RESIDENT; T.resident++; if (kd) T.xres++; else T.dres[w->dev]++; T.evictions--; free(buf); return; }
            T.evict = n; T.cevict = nc;
        }
        T.evict[T.nevict++] = victim;
        pthread_mutex_lock(&T.mx); T.evict_pending = T.nevict; pthread_mutex_unlock(&T.mx);
        quiesce();   /* nothing in flight: free it right away; else at the join */
    }
    v->state = VS_QUEUED; v->dev = (uint8_t)dev; T.queued++; if (kd) T.xque++; else T.dque[dev]++; T.promos[kd]++;
    pthread_mutex_lock(&T.mx);
    T.q[(T.qh + T.qn) % VKT_QCAP] = (VQ){layer, eid, dev, buf}; T.qn++;
    pthread_cond_signal(&T.cv);
    pthread_mutex_unlock(&T.mx);
}

/* vkt_note's decision without its copy: the same tests, in the same order. */
int vkt_wants(int layer, int eid) {
    if (!T.on || layer < 0 || layer >= T.c.layers || eid < 0 || eid >= T.c.experts) return 0;
    VSlot *v = slot(layer, eid);
    if (v->state != VS_NONE || T.promos[kind_of(layer)] >= T.promo_cap) return 0;
    if (room_dev(layer) < 0 && (!v->heat || peek_victim(kind_of(layer), score(v)) < 0)) return 0;
    pthread_mutex_lock(&T.mx);
    int qfull = T.qn >= VKT_QCAP;
    pthread_mutex_unlock(&T.mx);
    return !qfull;
}

/* ---- warm start ------------------------------------------------------------------ */
static const uint32_t *const *g_plan_heat;
static int g_plan_E;
static int plan_cmp(const void *a, const void *b) {
    int x = *(const int *)a, y = *(const int *)b;
    uint32_t hx = g_plan_heat[x / g_plan_E][x % g_plan_E], hy = g_plan_heat[y / g_plan_E][y % g_plan_E];
    return hx < hy ? 1 : hx > hy ? -1 : (x > y) - (x < y);
}
static uint32_t *const *g_heat_hist;
int vkt_plan(int *layers, int *eids, int max) {
    if (!T.on || !g_heat_hist) return 0;
    int L = T.main_layers, E = T.c.experts, n = 0;   /* the history: the main layers' */
    int *order = malloc((size_t)L * E * sizeof(int));
    if (!order) return 0;
    for (int l = 0; l < L; l++)
        if (g_heat_hist[l]) for (int e = 0; e < E; e++) if (g_heat_hist[l][e]) order[n++] = l * E + e;
    g_plan_heat = (const uint32_t *const *)g_heat_hist; g_plan_E = E;
    qsort(order, (size_t)n, sizeof(int), plan_cmp);
    int k = 0;
    for (int i = 0; i < n && k < max && room_dev(0) >= 0; i++) {
        VSlot *v = &T.s[order[i]];
        if (v->state != VS_NONE) continue;
        int dev = room_dev(0);   /* the hottest fill the primary device, the next ones the second */
        v->state = VS_QUEUED; v->dev = (uint8_t)dev; T.queued++; T.dque[dev]++;
        layers[k] = order[i] / E; eids[k] = order[i] % E; k++;
    }
    free(order);
    return k;
}
int vkt_put(int layer, int eid, const VktExpertSrc *src) {
    if (!T.on || layer < 0 || layer >= T.c.layers || eid < 0 || eid >= T.c.experts) return 0;
    ColiVkTensor *t[3] = {NULL, NULL, NULL};
    int dev = slot(layer, eid)->dev;   /* vkt_plan's choice */
    int r = src && src->g && src->u && src->d ? upload(dev, kind_of(layer), src->g, src->u, src->d, src->gs, src->us, src->ds, t) : 0;
    int ok = r > 0;
    push_done(layer, eid, ok, r < 0, t);
    if (ok) __atomic_add_fetch(&T.warm, 1, __ATOMIC_RELAXED);
    return ok;
}
void vkt_put_done(void) { if (T.on) quiesce(); }

/* ---- one layer step -------------------------------------------------------------- */
/* The per-step arrays: assignments (map), groups (bex, brows, bfirst, touched) and
 * rows (bx, by), each family grown together. */
static int grow_to(int need, int *cap) {
    int nc = *cap ? *cap : 64;
    while (nc < need) nc *= 2;
    return nc;
}
static int grow_map(int need) {
    if (need <= T.cmap) return 1;
    int nc = grow_to(need, &T.cmap);
    int *m = realloc(T.map, (size_t)nc * sizeof(int));
    if (!m) return 0;
    T.map = m; T.cmap = nc;
    return 1;
}
static int grow_groups(int need) {
    if (need <= T.cb) return 1;
    int nc = grow_to(need, &T.cb);
    ColiVkExpert **a = realloc(T.bex, (size_t)nc * sizeof(*a)); if (a) T.bex = a;
    int *b = realloc(T.brows, (size_t)nc * sizeof(int)); if (b) T.brows = b;
    int *c = realloc(T.bfirst, (size_t)nc * sizeof(int)); if (c) T.bfirst = c;
    int *d = realloc(T.touched, (size_t)nc * sizeof(int)); if (d) T.touched = d;
    if (!a || !b || !c || !d) return 0;
    T.cb = nc;
    return 1;
}
static int grow_rows(int need) {
    if (need <= T.cbx) return 1;
    int nc = grow_to(need, &T.cbx);
    const float **a = realloc(T.bx, (size_t)nc * sizeof(*a)); if (a) T.bx = a;
    const float **b = realloc(T.by, (size_t)nc * sizeof(*b)); if (b) T.by = b;
    float *c = realloc(T.bw, (size_t)nc * sizeof(*c)); if (c) T.bw = c;
    if (!a || !b || !c) return 0;
    T.cbx = nc;
    return 1;
}

/* ---- streaming: a big prefill step's cold experts on the device -------------------------
 * A prompt chunk of thousands of rows routes most of a layer's experts, each to dozens of
 * rows or more. The resident ones run on the device as always; a cold one (not resident)
 * the CPU would read from its RAM cache or the disk and multiply for every one of its
 * rows. Streaming hands it to the device instead: the engine loads it as its CPU path
 * would (VktConfig.load), the tier copies it into one of a few staging slots (st_n of
 * them, carved out of the budget) and runs its rows there with the tiled GEMM, then the
 * slot is free for the next one. The resident set and its LFRU never see a streamed
 * expert: it is not offered for promotion (a prompt's whole working set passes through
 * the slots and would flush the residents), and its routings count as heat, as every
 * routing does.
 *
 * Such a step runs as sub-batches (backend_vulkan.c's coli_vk_xb_sub_*): the resident
 * experts first, then the streamed ones, at most st_per streamed experts and st_half
 * rows a sub-batch; sub-batch k goes to half k % 2 of the scratch, so while the device
 * computes one the engine thread loads and uploads the experts of the next (double
 * buffering). An expert with more rows than a sub-batch holds is cut into parts.
 *
 * The rule, per cold expert of a step: stream it when it has at least R rows,
 *     R = max(G, ceil(upload / cpu_row)),
 * G the rows from which the device takes the tiled GEMM (COLI_VK_TIER_GEMM_ROWS, 16:
 * fewer rows would run the per-row GEMV), upload the time one expert takes to reach a
 * slot (its bytes at the bandwidth measured over the uploads so far), cpu_row the time
 * the CPU spent per (row, expert) pair it computed in the prefill steps so far (its share
 * of a step between issue and join, loads included). Below R the CPU keeps the expert:
 * its rows cost the CPU less than the upload. Until both are measured R = G.
 * COLI_VK_TIER_STREAM_ROWS=n fixes R at n (tests). A [VK] line per forward says what the
 * rule gave, and the run's line how much streamed. */
static double vkt_now_ms(void);
static int stream_min_rows(void) {
    if (T.st_rows_env > 0) return T.st_rows_env;
    int r = T.gemm_rows;
    if (T.bw_up > 0 && T.cpu_row_ms > 0) {
        double up = (double)T.exp_bytes / T.bw_up, need = up / T.cpu_row_ms;
        if (need > 1e6) need = 1e6;
        int n = (int)need + (need > (int)need);
        if (n > r) r = n;
    }
    return r;
}
int vkt_step_rows(int S, int block) {
    if (T.on && T.st_ok && S >= T.gemm_rows) return S;
    return S < block ? S : block;
}

/* the slots, made at the first step that streams */
static int st_alloc(void) {
    if (T.st) return T.st_n >= 2;
    T.st = calloc((size_t)T.st_slots, sizeof(VStage));
    if (!T.st) { T.st_ok = 0; return 0; }
    int H = T.c.hidden, F = T.c.inter, n = 0;
    for (int i = 0; i < T.st_slots; i++) {
        VStage *v = &T.st[n];
        int ok = 1;
        for (int k = 0; k < 3 && ok; k++) {
            int fmt = k < 2 ? T.form[0].gu_fmt : T.form[0].dn_fmt, gs = k < 2 ? T.form[0].gu_gs : T.form[0].dn_gs;
            uint8_t *rows; size_t stride; float *sc;
            ok = coli_vk_tier_tensor(&v->t[k], fmt, k < 2 ? H : F, k < 2 ? F : H, gs, &rows, &stride, &sc);
            if (!ok) v->t[k] = NULL;
        }
        if (ok) v->ex = coli_vk_xb_expert(v->t[0], v->t[1], v->t[2]);
        if (!ok || !v->ex) {
            for (int k = 0; k < 3; k++) if (v->t[k]) coli_vk_tensor_free(v->t[k]);
            memset(v, 0, sizeof *v);
            break;
        }
        n++;
    }
    T.st_n = n;
    if (n < 2) {
        fprintf(stderr, "[VK] tier %s: no device memory for the streaming slots, cold experts stay on the CPU\n", T.engine);
        T.st_ok = 0;
        return 0;
    }
    if (n < T.st_slots) fprintf(stderr, "[VK] tier %s: %d of %d streaming slots\n", T.engine, n, T.st_slots);
    T.st_per = n / 2;
    return 1;
}
/* Experts' bytes into slots: eids[i] into T.st[slots[i]], ok[i] = 1 when it is there
 * (else the CPU computes it). The engine loads them on this thread, several at once
 * when it can (load_batch: its parallel reads); the conversions into the slots run in
 * parallel, each expert into its own slot. */
static int st_put(int layer, int eid, VStage *v, const VktExpertSrc *src, int par) {
    const int H = T.c.hidden, F = T.c.inter;
    if (!src->g || !src->u || !src->d || (T.form[0].gu_scales && (!src->gs || !src->us || !src->ds))) return 0;
    const uint8_t *codes[3] = {src->g, src->u, src->d}; const void *scs[3] = {src->gs, src->us, src->ds};
    for (int k = 0; k < 3; k++) {
        uint8_t *rows; size_t stride; float *sc;
        if (!coli_vk_tensor_refill(v->t[k], &rows, &stride, &sc)) return 0;
        convert_par(k < 2 ? T.form[0].gu : T.form[0].dn, k < 2 ? H : F, k < 2 ? F : H, codes[k], scs[k], rows, stride, sc, par);
    }
    (void)layer; (void)eid;
    return 1;
}
static void st_fill_group(int layer, const int *eids, const int *slots, int n, uint8_t *ok) {
    VktExpertSrc srcs[64]; void *hs[64];
    if (n > 64) n = 64;
    double ms = 0;
    /* A device upload group can exceed the RAM cache. Consume each cache-sized
     * batch completely before loading the next, which may reuse those RAM slots.
     * A partial batch is capacity, not a reason to serialize all remaining reads. */
    for (int base = 0; base < n;) {
        int left = n - base;
        int got = T.c.load_batch && left >= 2
                ? T.c.load_batch(T.c.load_ctx, layer, eids + base, left, srcs, hs) : 0;
        if (got < 0 || got > left) got = 0;
        if (got) {
            double t0 = vkt_now_ms();
            #pragma omp parallel for schedule(dynamic, 1) if (T.st_par)
            for (int j = 0; j < got; j++)
                ok[base + j] = (uint8_t)st_put(layer, eids[base + j], &T.st[slots[base + j]], &srcs[j], 0);
            ms += vkt_now_ms() - t0;
            for (int j = 0; j < got; j++) T.c.release(T.c.load_ctx, hs[j]);
            base += got;
            continue;
        }
        /* A lone expert, or an engine without a usable batch hook. Its rows can
         * still convert in parallel (COLI_VK_TIER_STREAM_PAR=0 keeps one thread). */
        void *h = NULL; VktExpertSrc src;
        ok[base] = 0;
        if (T.c.load(T.c.load_ctx, layer, eids[base], &src, &h)) {
            double t0 = vkt_now_ms();
            ok[base] = (uint8_t)st_put(layer, eids[base], &T.st[slots[base]], &src, T.st_par);
            ms += vkt_now_ms() - t0;
            T.c.release(T.c.load_ctx, h);
        }
        base++;
    }
    double t0 = vkt_now_ms();
    /* Fill the uploader's bounded staging window across the complete group.
     * Committing each expert separately drains that window and waits for a
     * fence after just three matrices, serializing transfers on discrete GPUs.
     * A failed group is wholly left to the CPU; no partial expert is published. */
    ColiVkTensor *tensors[3 * 64]; int nt = 0;
    for (int i = 0; i < n; i++) if (ok[i])
        for (int k = 0; k < 3; k++) tensors[nt++] = T.st[slots[i]].t[k];
    if (nt && !coli_vk_tensor_commit(tensors, nt)) memset(ok, 0, (size_t)n);
    int done = 0;
    for (int i = 0; i < n; i++) {
        VStage *v = &T.st[slots[i]];
        if (!ok[i]) { v->state = SS_FREE; continue; }
        v->layer = layer; v->eid = eids[i]; v->state = SS_FILLED; v->pending = 0; v->pref = 0;
        done++;
    }
    ms += vkt_now_ms() - t0;
    T.st_up_ms += ms; T.st_bytes += (unsigned long long)done * T.exp_bytes;
    if (done && ms > 0) { double bw = (double)done * T.exp_bytes / ms; T.bw_up = T.bw_up > 0 ? 0.8 * T.bw_up + 0.2 * bw : bw; }
}
/* a sub-batch in flight joined: its rows copied to their places, its slots free */
static int st_join_half(int h) {
    VSub *b = &T.hf[h];
    if (!coli_vk_xb_sub_busy(h)) return 1;
    double dms = 0, t0 = vkt_now_ms();
    int ok = coli_vk_xb_sub_join(h, b->yout, &dms);
    T.wait_ms += vkt_now_ms() - t0;
    T.dev_ms += dms;
    for (int i = 0; i < b->nslot; i++) {
        VStage *v = &T.st[b->slot[i]];
        if (--v->pending <= 0) { v->pending = 0; v->state = SS_FREE; }
    }
    b->n = b->nslot = 0;
    if (!ok) T.big_fail = 1;
    return ok;
}
static int grow_int(int **p, int *cap, int need) {
    if (need <= *cap) return 1;
    int nc = *cap ? *cap : 64; while (nc < need) nc *= 2;
    int *n = realloc(*p, (size_t)nc * sizeof(int));
    if (!n) return 0;
    *p = n; *cap = nc;
    return 1;
}

/* One big step: every resident expert and every cold one the rule streams, as
 * sub-batches; the CPU gets the rest (taken stays 0). The second device's residents run
 * there as one batch beside them, up to d2_rows rows (past that the rule decides). */
enum { BC_NONE = 0, BC_DEV, BC_STREAM, BC_CPU, BC_DEV2 };
/* the second device's batch of a big step: its experts' rows, in its one batch */
static int d2_issue_big(int layer, const float *x, int K, const float *w, const int *cnt, const int *ofs,
                        const int *cls, uint8_t *taken) {
    const int E = T.c.experts, H = T.c.hidden;
    int ne = 0, nr = 0;
    for (int e = 0; e < E; e++) if (cls[e] == BC_DEV2) { ne++; nr += cnt[e]; }
    if (!ne) return 0;
    if (ne > T.d2_cexp) {
        ColiVkExpert **a = realloc(T.d2_bex, (size_t)ne * sizeof(*a)); if (a) T.d2_bex = a;
        int *b = realloc(T.d2_brows, (size_t)ne * sizeof(int)); if (b) T.d2_brows = b;
        if (!a || !b) return 0;
        T.d2_cexp = ne;
    }
    if (nr > T.d2_cap) {
        const float **a = realloc(T.d2_bx, (size_t)nr * sizeof(*a)); if (a) T.d2_bx = a;
        const float **b = realloc(T.d2_y, (size_t)nr * sizeof(*b)); if (b) T.d2_y = b;
        float *c = realloc(T.d2_bw, (size_t)nr * sizeof(float)); if (c) T.d2_bw = c;
        if (!a || !b || !c) return 0;
        T.d2_cap = nr;
    }
    if (!grow_int(&T.d2_idx, &T.cd2, nr)) return 0;
    int c = 0, j = 0;
    for (int e = 0; e < E; e++) {
        if (cls[e] != BC_DEV2) continue;
        T.d2_bex[c] = slot(layer, e)->ex; T.d2_brows[c] = cnt[e]; c++;
        for (int r = 0; r < cnt[e]; r++, j++) {
            int i = T.blist[ofs[e] + r];
            T.d2_bx[j] = x + (size_t)(i / K) * H; T.d2_bw[j] = w ? w[i] : 1.0f; T.d2_idx[j] = i;
        }
    }
    if (!coli_vk_xb_issue_dev(1, T.d2_bex, T.d2_brows, c, T.d2_bx, w ? T.d2_bw : NULL)) {
        if (!coli_vk_xb_ready_dev(1)) d2_stop("the second device's batch failed");
        return 0;
    }
    for (int q = 0; q < j; q++) { taken[T.d2_idx[q]] = 1; T.map[T.d2_idx[q]] = T.d2_idx[q]; }
    T.d2_inflight = 1; T.nd2 = j; T.inflight = 1;
    T.d2_served += (unsigned long long)j; T.d2_steps++; T.d2_big++;
    return j;
}
static int issue_big(int layer, const float *x, int S, int K, const int *idx, const float *w, uint8_t *taken) {
    const int E = T.c.experts, H = T.c.hidden, n = S * K;
    int *cnt = T.bcnt, *ofs = T.bofs, *cls = T.bcls;
    memset(cnt, 0, (size_t)E * sizeof(int));
    for (int i = 0; i < n; i++) {
        int e = idx[i];
        if (e < 0 || e >= E) continue;
        T.routed++;
        VSlot *v = slot(layer, e);
        if (v->heat < 0xFFFFFFFFu) v->heat++;
        v->last = T.tick;
        cnt[e]++;
    }
    /* the rows grouped by expert, each expert's in routing order; its class */
    if (!grow_int(&T.blist, &T.cblist, n) || !grow_map(n)) return 0;
    int at = 0, R = stream_min_rows();
    for (int e = 0; e < E; e++) {
        ofs[e] = at; at += cnt[e];
        VSlot *v = slot(layer, e);
        cls[e] = !cnt[e] ? BC_NONE : v->state == VS_RESIDENT && v->ex ? (v->dev ? BC_DEV2 : BC_DEV)
               : T.st_ok && cnt[e] >= R ? BC_STREAM : BC_CPU;
    }
    /* the second device's one batch holds d2_rows rows; past them the rule decides */
    for (int e = 0, d2r = 0; e < E; e++)
        if (cls[e] == BC_DEV2) {
            if (d2r + cnt[e] > T.d2_rows) cls[e] = T.st_ok && cnt[e] >= R ? BC_STREAM : BC_CPU;
            else d2r += cnt[e];
        }
    int *fill = malloc((size_t)E * sizeof(int));
    if (!fill) return 0;
    memcpy(fill, ofs, (size_t)E * sizeof(int));
    for (int i = 0; i < n; i++) { int e = idx[i]; if (e >= 0 && e < E) T.blist[fill[e]++] = i; }
    free(fill);
    if (T.pred) {   /* the prediction for the next big step at this layer */
        for (int e = 0; e < E; e++) T.pred[(size_t)layer * E + e] = (uint32_t)cnt[e];
        T.pred_ok[layer] = 1;
    }
    int ndev = 0, nst = 0, nkept = 0, kept_rows = 0, nd2 = 0;
    for (int e = 0; e < E; e++) {
        ndev += cls[e] == BC_DEV; nst += cls[e] == BC_STREAM; nd2 += cls[e] == BC_DEV2;
        if (cls[e] == BC_CPU) { nkept++; kept_rows += cnt[e]; }
    }
    /* prefetched slots this step does not stream go back */
    for (int i = 0; T.st && i < T.st_n; i++) {
        VStage *v = &T.st[i];
        if (v->state == SS_FILLED && (v->layer != layer || cls[v->eid] != BC_STREAM)) v->state = SS_FREE;
    }
    if (nst && !st_alloc()) {
        for (int e = 0; e < E; e++) if (cls[e] == BC_STREAM) { cls[e] = BC_CPU; nkept++; kept_rows += cnt[e]; }
        nst = 0;
    }
    if (!T.said_fwd) {
        T.said_fwd = 1;
        char hb[32];
        fprintf(stderr, "[VK] tier %s stream: a step of %d rows; cold experts with %d rows or more go to the device (",
                T.engine, S, R);
        if (T.st_rows_env > 0) fprintf(stderr, "COLI_VK_TIER_STREAM_ROWS");
        else if (T.bw_up > 0 && T.cpu_row_ms > 0)
            fprintf(stderr, "an upload of %s at %.1f GB/s takes %.3f ms, the CPU %.4f ms a row; the GEMM from %d rows",
                    human((double)T.exp_bytes, hb, sizeof hb), T.bw_up / 1e6, (double)T.exp_bytes / T.bw_up,
                    T.cpu_row_ms, T.gemm_rows);
        else fprintf(stderr, "the GEMM's %d rows: the upload and the CPU's rows not measured yet", T.gemm_rows);
        fprintf(stderr, "); layer %d: %d resident, %d streamed, %d kept on the CPU (%d rows)", layer, ndev, nst, nkept, kept_rows);
        if (T.d2) fprintf(stderr, ", %d on the second device", nd2);
        fprintf(stderr, "\n");
    }
    T.st_kept += (unsigned long long)nkept; T.st_kept_rows += (unsigned long long)kept_rows;
    if (!ndev && !nst && !nd2) return 0;
    /* The backend counts both halves, activation buffers, descriptor windows and
     * growth before allocating. Leave half of available RAM for CPU work too. */
    int half = T.st_half < n ? T.st_half : n;
    if (ndev || nst) {
        size_t ram = (size_t)(compat_mem_available_gb() * 1e9);
        half = coli_vk_xb_sub_fit(half, E + 1, ram ? ram / 2 : SIZE_MAX);
        if (half < 1 || !coli_vk_xb_sub_reserve(half, E + 1)) {
            fprintf(stderr, "[VK] tier %s: no scratch for a step of %d rows, its experts stay on the CPU\n", T.engine, S);
            return 0;
        }
    }
    if ((size_t)n * H > T.csy) {
        float *ny = realloc(T.sy, (size_t)n * H * sizeof(float));
        if (!ny) return 0;
        T.sy = ny; T.csy = (size_t)n * H;
    }
    for (int i = 0; i < n; i++) T.map[i] = -1;
    T.big = 1; T.big_fail = 0; T.big_n = n; T.big_taken = 0; T.sk = 0;
    T.big_valid = 0;
    for (int e = 0; e < E; e++) T.big_valid += cnt[e];
    /* the step's token rows once and its output rows in place, where the backend can
     * (not beside the second device: its rows come back as before) */
    T.sy_step = !nd2 && (ndev || nst) ? coli_vk_xb_step_begin(S, K, x) : NULL;
    float *SY = T.sy_step ? T.sy_step : T.sy;
    T.can_balance = 0;
    /* the order: resident experts, then the streamed ones already in a slot, then the rest */
    int *order = malloc((size_t)E * sizeof(int)), no = 0;
    ColiVkExpert **bex = malloc((size_t)(E + 1) * sizeof(*bex));
    int *brows = malloc((size_t)(E + 1) * sizeof(int));
    const float **bx = malloc((size_t)half * sizeof(*bx)); float *bw = malloc((size_t)half * sizeof(float));
    int *bidx = malloc((size_t)half * sizeof(int));
    int *eslot = malloc((size_t)E * sizeof(int));
    int ok = order && bex && brows && bx && bw && bidx && eslot;
    for (int h = 0; h < 2 && ok; h++) {   /* a sub-batch's slots and output places */
        if (T.hf[h].cap < half) {
            float **ny = realloc(T.hf[h].yout, (size_t)half * sizeof(float *));
            if (ny) { T.hf[h].yout = ny; T.hf[h].cap = half; } else ok = 0;
        }
        if (ok && !T.hf[h].slot) ok = (T.hf[h].slot = malloc((size_t)(E + 1) * sizeof(int))) != NULL;
    }
    if (!ok) { free(order); free(bex); free(brows); free(bx); free(bw); free(bidx); free(eslot); T.big = 0; return 0; }
    for (int e = 0; e < E; e++) { eslot[e] = -1; if (cls[e] == BC_DEV) order[no++] = e; }
    for (int i = 0; T.st && i < T.st_n; i++) {
        VStage *v = &T.st[i];
        if (v->state == SS_FILLED && v->layer == layer && cls[v->eid] == BC_STREAM && eslot[v->eid] < 0) {
            eslot[v->eid] = i; order[no++] = v->eid;
            if (v->pref) T.pf_used++;
        }
    }
    for (int e = 0; e < E; e++) if (cls[e] == BC_STREAM && eslot[e] < 0) order[no++] = e;
    /* the second device's batch first, so it runs beside every sub-batch */
    int d2_taken = nd2 ? d2_issue_big(layer, x, K, w, cnt, ofs, cls, taken) : 0;
    /* sub-batches */
    int cnt_b = 0, rows_b = 0, nst_b = 0, total = 0;
    VSub *cur = NULL;
    int curh = 0;
#define SUBMIT_CURRENT() do { \
        if (cnt_b) { \
            int okk = !T.big_fail && (T.sy_step ? coli_vk_xb_sub_issue_step(curh, bex, brows, cnt_b, bidx, bw) \
                                                : coli_vk_xb_sub_issue(curh, bex, brows, cnt_b, bx, bw)); \
            if (okk) { \
                for (int q = 0; q < rows_b; q++) { taken[bidx[q]] = 1; T.map[bidx[q]] = bidx[q]; } \
                total += rows_b; T.st_subs++; T.sk++; \
                T.inflight = 1; \
            } else { \
                T.big_fail = 1; \
                for (int q = 0; q < cur->nslot; q++) { VStage *vv = &T.st[cur->slot[q]]; if (--vv->pending <= 0) { vv->pending = 0; vv->state = SS_FREE; } } \
                cur->nslot = 0; cur->n = 0; \
            } \
        } \
        cnt_b = rows_b = nst_b = 0; cur = NULL; \
    } while (0)
    for (int o = 0; o < no && !T.big_fail; o++) {
        int e = order[o];
        /* An earlier group fill can have reassigned this expert to the CPU.
         * Its entry remains in order[], but has no resident device tensor. */
        if (cls[e] == BC_CPU) continue;
        int sl = -1;
        if (cls[e] == BC_STREAM) {
            if (nst_b >= T.st_per) SUBMIT_CURRENT();
            if (T.big_fail) break;
            if (eslot[e] < 0) {
                /* this expert and the next ones without a slot, as many as the sub-batch has
                 * room for, loaded and uploaded together; free slots first, joining the
                 * oldest sub-batch in flight when there are not enough */
                int room = T.st_per - nst_b, gid[64], gsl[64], gn = 0;
                if (room > 64) room = 64;
                for (int o2 = o; o2 < no && gn < room; o2++)
                    if (cls[order[o2]] == BC_STREAM && eslot[order[o2]] < 0) gid[gn++] = order[o2];
                int nfree = 0;
                for (int tries = 0; tries < 3; tries++) {
                    nfree = 0;
                    for (int i = 0; i < T.st_n && nfree < gn; i++) if (T.st[i].state == SS_FREE) gsl[nfree++] = i;
                    if (nfree >= gn) break;
                    int old = coli_vk_xb_sub_busy(T.sk % 2) ? (int)(T.sk % 2) : (int)((T.sk + 1) % 2);
                    if (cur && old == curh && cnt_b) old = !curh;   /* never the half being filled */
                    if (!coli_vk_xb_sub_busy(old)) break;
                    st_join_half(old);
                }
                if (T.big_fail) break;
                if (nfree < gn) gn = nfree;
                uint8_t gok[64];
                st_fill_group(layer, gid, gsl, gn, gok);
                for (int i = 0; i < gn; i++) {
                    if (gok[i]) eslot[gid[i]] = gsl[i];
                    else { cls[gid[i]] = BC_CPU; T.st_kept++; T.st_kept_rows += (unsigned long long)cnt[gid[i]]; }
                }
                if (cls[e] != BC_STREAM || eslot[e] < 0) {   /* not loaded (or no slot): the CPU's */
                    if (cls[e] == BC_STREAM) { cls[e] = BC_CPU; T.st_kept++; T.st_kept_rows += (unsigned long long)cnt[e]; }
                    continue;
                }
            }
            sl = eslot[e];
            T.st_experts++; T.st_rows += (unsigned long long)cnt[e];
        }
        ColiVkExpert *ex = sl >= 0 ? T.st[sl].ex : slot(layer, e)->ex;
        for (int r0 = 0; r0 < cnt[e] && !T.big_fail; ) {
            if (rows_b && rows_b + (cnt[e] - r0 < half ? cnt[e] - r0 : half) > half) SUBMIT_CURRENT();
            if (T.big_fail) break;
            if (!cur) {   /* a new sub-batch in half sk % 2: the one there before is joined first */
                curh = (int)(T.sk % 2);
                if (coli_vk_xb_sub_busy(curh) && !st_join_half(curh)) break;
                cur = &T.hf[curh]; cur->n = cur->nslot = 0;
            }
            int take = cnt[e] - r0; if (take > half - rows_b) take = half - rows_b;
            bex[cnt_b] = ex; brows[cnt_b] = take; cnt_b++;
            for (int q = 0; q < take; q++) {
                int i = T.blist[ofs[e] + r0 + q];
                bx[rows_b + q] = x + (size_t)(i / K) * H;
                bw[rows_b + q] = w ? w[i] : 1.0f;
                bidx[rows_b + q] = i;
                cur->yout[rows_b + q] = SY + (size_t)i * H;
            }
            rows_b += take; cur->n = rows_b;
            if (sl >= 0) {
                if (!T.st[sl].pending || T.st[sl].state != SS_BUSY) { T.st[sl].state = SS_BUSY; }
                T.st[sl].pending++;
                cur->slot[cur->nslot++] = sl;
                if (r0 == 0) nst_b++;
            }
            r0 += take;
        }
    }
    SUBMIT_CURRENT();
#undef SUBMIT_CURRENT
    free(order); free(bex); free(brows); free(bx); free(bw); free(bidx); free(eslot);
    total += d2_taken;
    T.big_taken = total;
    if (!total && !T.inflight) {   /* nothing went (or everything failed before a submit) */
        T.big = 0;
        if (T.sy_step) { coli_vk_xb_step_end(); T.sy_step = NULL; }
        if (T.big_fail) {
            fprintf(stderr, "[VK] tier %s: a batch failed, its experts are recomputed on the CPU and the tier stops\n", T.engine);
            T.on = 0;
        }
        return 0;
    }
    T.steps++; T.served += (unsigned long long)total; T.st_steps++;
    T.S = S; T.K = K;
    T.t_issued = vkt_now_ms();
    return total;
}
static int join_big(const float **rows) {
    double t0 = vkt_now_ms(), tc = t0 - T.t_issued;
    T.cpu_ms += tc;
    /* the oldest first */
    int a = (int)(T.sk % 2), b = !a;
    st_join_half(a); st_join_half(b);
    int ok1 = 1;
    if (T.d2_inflight) {   /* the second device's rows to their places */
        double dms = 0, t1 = vkt_now_ms();
        ok1 = coli_vk_xb_join_dev(1, T.d2_y, &dms);
        T.wait_ms += vkt_now_ms() - t1; T.dev_ms += dms; T.d2_ms += dms;
        if (ok1) for (int q = 0; q < T.nd2; q++) memcpy((T.sy_step ? T.sy_step : T.sy) + (size_t)T.d2_idx[q] * T.c.hidden, T.d2_y[q], (size_t)T.c.hidden * sizeof(float));
        T.d2_inflight = 0;
    }
    int cpu_pairs = T.big_n - T.big_taken;
    if (cpu_pairs >= 32 && tc > 0) {
        double r = tc / cpu_pairs;
        T.cpu_row_ms = T.cpu_row_ms > 0 ? 0.8 * T.cpu_row_ms + 0.2 * r : r;
    }
    T.inflight = 0; T.big = 0;
    int ok = !T.big_fail;
    const float *SY = T.sy_step ? T.sy_step : T.sy;
    for (int i = 0; i < T.big_n; i++) rows[i] = ok && ok1 && T.map[i] >= 0 ? SY + (size_t)i * T.c.hidden : NULL;
    for (int i = 0; T.st && i < T.st_n; i++) if (T.st[i].state == SS_BUSY) { T.st[i].state = SS_FREE; T.st[i].pending = 0; }
    if (!ok) {
        fprintf(stderr, "[VK] tier %s: a batch failed, its experts are recomputed on the CPU and the tier stops\n", T.engine);
        T.on = 0;
        return 0;
    }
    if (!ok1) {
        d2_stop("a batch on the second device failed");
        return 0;
    }
    quiesce();
    return 1;
}

/* layer's likely streamed experts into the free slots, while the device runs its attention */
int vkt_stream_prefetch(int layer, int S) {
    if (!T.on || !T.st_ok || T.inflight || layer < 0 || layer >= T.main_layers || S < T.gemm_rows) return 0;
    if (!coli_vk_xb_ready() || !st_alloc()) return 0;
    const int E = T.c.experts, K = T.c.topk;
    int R = stream_min_rows();
    /* the slots a step before left filled are free again */
    for (int i = 0; i < T.st_n; i++) if (T.st[i].state != SS_BUSY) T.st[i].state = SS_FREE;
    double *est = malloc((size_t)E * sizeof(double));
    int *cand = malloc((size_t)E * sizeof(int));
    if (!est || !cand) { free(est); free(cand); return 0; }
    double tot = 0;
    if (T.pred && T.pred_ok[layer]) {
        double sum = 0;
        for (int e = 0; e < E; e++) sum += T.pred[(size_t)layer * E + e];
        for (int e = 0; e < E; e++) est[e] = sum > 0 ? (double)T.pred[(size_t)layer * E + e] * S * K / sum : 0;
        tot = sum;
    } else {
        for (int e = 0; e < E; e++) tot += slot(layer, e)->heat;
        for (int e = 0; e < E; e++) est[e] = tot > 0 ? (double)slot(layer, e)->heat * S * K / tot : 0;
    }
    int nc = 0;
    if (tot > 0)
        for (int e = 0; e < E; e++) {
            VSlot *v = slot(layer, e);
            if (v->state == VS_RESIDENT || est[e] < R) continue;
            int at = nc++;
            while (at > 0 && est[cand[at - 1]] < est[e]) { cand[at] = cand[at - 1]; at--; }
            cand[at] = e;
        }
    int got = 0, used = 0;
    if (nc > T.st_n) nc = T.st_n;
    while (used < nc) {   /* groups of a sub-batch's worth, into the slots in order */
        int gn = nc - used < T.st_per ? nc - used : T.st_per, gsl[64]; uint8_t gok[64];
        if (gn > 64) gn = 64;
        for (int i = 0; i < gn; i++) gsl[i] = used + i;
        st_fill_group(layer, cand + used, gsl, gn, gok);
        for (int i = 0; i < gn; i++) if (gok[i]) { T.st[gsl[i]].pref = 1; got++; }
        used += gn;
    }
    T.pf_n += (unsigned long long)got;
    free(est); free(cand);
    return got;
}

static int issue(int layer, const float *x, int S, int K, const int *idx, const float *w, uint8_t *taken) {
    if (S > 0 && K > 0) memset(taken, 0, (size_t)S * K);
    if (!T.on || T.inflight || layer < 0 || layer >= T.c.layers || S < 1 || K < 1) return 0;
    if (!coli_vk_xb_ready()) {
        if (T.on) { fprintf(stderr, "[VK] tier %s: the device stopped answering, the experts stay on the CPU\n", T.engine); T.on = 0; }
        return 0;
    }
    if (T.d2 && !coli_vk_xb_ready_dev(1)) d2_stop("the second device stopped answering");
    if (layer < T.last_layer || T.begin) { T.begin = 0; T.first_layer = layer; new_forward(); }
    if (T.routed_at) {   /* this step's routing: its experts are not offered to the RAM cache's eviction */
        uint32_t q = ++T.seq;
        for (int i = 0; i < S * K; i++)
            if (idx[i] >= 0 && idx[i] < T.c.experts) __atomic_store_n(&T.routed_at[(size_t)layer * T.c.experts + idx[i]], q, __ATOMIC_RELAXED);
        __atomic_store_n(&T.layer_seq[layer], q, __ATOMIC_RELEASE);
    }
    if (layer == T.first_layer) {   /* COLI_VK_TIER_RATE promotions per token of the forward */
        T.tick += (uint32_t)S;
        long cap = (long)T.promo_cap + (long)T.rate * S;
        T.promo_cap = cap > (1 << 30) ? 1 << 30 : (int)cap;
    }
    T.last_layer = layer;
    quiesce();
    if (T.st_ok && S >= T.gemm_rows && layer < T.main_layers) return issue_big(layer, x, S, K, idx, w, taken);
    int E = T.c.experts, H = T.c.hidden, n = S * K;
    if (!grow_map(n)) return 0;
    /* heat for every routed expert, groups for the resident ones (first seen first) */
    int ng = 0, total = 0, rows1 = 0;
    for (int i = 0; i < n; i++) {
        T.map[i] = -1;
        int e = idx[i];
        if (e < 0 || e >= E) continue;
        T.routed++;
        if (layer >= T.main_layers) T.extra_routed++;
        VSlot *v = slot(layer, e);
        if (v->heat < 0xFFFFFFFFu) v->heat++;
        v->last = T.tick;
        if (v->state != VS_RESIDENT) continue;
        if (v->dev && rows1 >= T.d2_rows) continue;    /* the second device's batch is as large as it gets */
        int g = T.grp[e];
        if (g < 0) {
            if (total >= T.max_dev_rows || !grow_groups(ng + 1)) continue;   /* the batch is as large as it gets */
            g = ng++; T.grp[e] = g; T.touched[g] = e; T.bex[g] = v->ex; T.brows[g] = 0;
        } else if (total >= T.max_dev_rows) continue;
        T.brows[g]++; total++; rows1 += v->dev;
        T.map[i] = g;                                   /* group for now, row index below */
    }
    if (!ng) return 0;
    /* The balance: when the device has been the slower side, the experts the CPU
     * also holds in RAM beyond the device's share of the step's rows go back to the
     * CPU, whole experts (the CPU reads one once for all its rows); the ones only
     * the device holds stay (the CPU would read them from disk). */
    T.can_balance = 0;
    if (T.balance) {
        int cap = (int)(T.share * total + 0.999f), kept = 0, keep_n = 0;
        uint8_t *gone = NULL;
        for (int g = 0; g < ng; g++) {
            if (!T.c.in_ram(T.c.ram_ctx, layer, T.touched[g])) { kept += T.brows[g]; continue; }
            T.can_balance = 1;
            /* the first expert always stays: a step with nothing on the device would
             * measure nothing, and the share could never come back up */
            if (!kept || kept + T.brows[g] <= cap) { kept += T.brows[g]; continue; }
            if (!gone && !(gone = calloc((size_t)ng, 1))) break;
            gone[g] = 1;
        }
        if (gone) {
            int *newg = malloc((size_t)ng * sizeof(int));
            if (newg) {
                for (int g = 0; g < ng; g++) {
                    if (gone[g]) { newg[g] = -1; T.grp[T.touched[g]] = -1; T.handed += (unsigned long long)T.brows[g]; total -= T.brows[g]; continue; }
                    newg[g] = keep_n;
                    T.bex[keep_n] = T.bex[g]; T.brows[keep_n] = T.brows[g]; T.touched[keep_n] = T.touched[g]; keep_n++;
                }
                for (int i = 0; i < n; i++) if (T.map[i] >= 0) T.map[i] = newg[T.map[i]];
                ng = keep_n;
                free(newg);
            }
            free(gone);
        }
        if (!ng) return 0;
    }
    /* With a second device: the primary device's groups first, then the second's, so each
     * device's batch is a contiguous run of groups and of rows (rows0 rows on the first). */
    int n0 = ng, rows0 = total;
    if (T.d2) {
        n0 = rows0 = 0;
        for (int g = 0; g < ng; g++) if (!slot(layer, T.touched[g])->dev) { n0++; rows0 += T.brows[g]; }
        if (n0 && n0 < ng) {
            int *newg = malloc((size_t)ng * sizeof(int)), *tr = malloc((size_t)ng * sizeof(int)), *tt = malloc((size_t)ng * sizeof(int));
            ColiVkExpert **tb = malloc((size_t)ng * sizeof(*tb));
            if (!newg || !tr || !tt || !tb) {
                free(newg); free(tr); free(tt); free(tb);
                for (int g = 0; g < ng; g++) T.grp[T.touched[g]] = -1;
                for (int i = 0; i < n; i++) T.map[i] = -1;
                return 0;
            }
            for (int g = 0, a0 = 0, a1 = n0; g < ng; g++) newg[g] = slot(layer, T.touched[g])->dev ? a1++ : a0++;
            for (int g = 0; g < ng; g++) { tb[newg[g]] = T.bex[g]; tr[newg[g]] = T.brows[g]; tt[newg[g]] = T.touched[g]; }
            memcpy(T.bex, tb, (size_t)ng * sizeof(*tb)); memcpy(T.brows, tr, (size_t)ng * sizeof(int));
            memcpy(T.touched, tt, (size_t)ng * sizeof(int));
            for (int i = 0; i < n; i++) if (T.map[i] >= 0) T.map[i] = newg[T.map[i]];
            free(newg); free(tr); free(tt); free(tb);
        }
    }
    /* rows expert by expert, each expert's in routing order */
    if (!grow_rows(total)) {
        for (int g = 0; g < ng; g++) T.grp[T.touched[g]] = -1;
        for (int i = 0; i < n; i++) T.map[i] = -1;
        return 0;
    }
    int at = 0;
    for (int g = 0; g < ng; g++) { T.bfirst[g] = at; at += T.brows[g]; T.brows[g] = 0; }
    for (int i = 0; i < n; i++) {
        int g = T.map[i];
        if (g < 0) continue;
        int j = T.bfirst[g] + T.brows[g]++;
        T.bx[j] = x + (size_t)(i / K) * H;
        T.bw[j] = w ? w[i] : 1.0f;
        T.map[i] = j;
    }
    for (int g = 0; g < ng; g++) T.grp[T.touched[g]] = -1;
    /* one batch per device, both in flight at once */
    T.d0_inflight = n0 > 0 && coli_vk_xb_issue_w(T.bex, T.brows, n0, T.bx, w ? T.bw : NULL);
    T.d2_inflight = 0; T.d2_rows0 = rows0;
    if (n0 < ng) {
        T.d2_inflight = coli_vk_xb_issue_dev(1, T.bex + n0, T.brows + n0, ng - n0, T.bx + rows0, w ? T.bw + rows0 : NULL);
        if (!T.d2_inflight && !coli_vk_xb_ready_dev(1)) d2_stop("the second device's batch failed");
    }
    int took = 0;
    for (int i = 0; i < n; i++) {
        int j = T.map[i];
        if (j < 0) continue;
        if (j < rows0 ? !T.d0_inflight : !T.d2_inflight) { T.map[i] = -1; continue; }   /* not submitted: the CPU's */
        taken[i] = 1; took++;
    }
    if (!took) return 0;
    T.inflight = 1; T.S = S; T.K = K; T.steps++; T.served += (unsigned long long)took;
    if (layer >= T.main_layers) T.extra_served += (unsigned long long)took;
    if (T.d2_inflight) { T.d2_served += (unsigned long long)(total - rows0); T.d2_steps++; }
    T.t_issued = vkt_now_ms();
    return took;
}
int vkt_issue(int layer, const float *x, int S, int K, const int *idx, uint8_t *taken) {
    return issue(layer, x, S, K, idx, NULL, taken);
}
int vkt_issue_w(int layer, const float *x, int S, int K, const int *idx, const float *w, uint8_t *taken) {
    return issue(layer, x, S, K, idx, w, taken);
}

int vkt_join(const float **rows) {
    if (!T.inflight) return 0;
    if (T.big) {
        int ok = join_big(rows);
        if (T.sy_step) coli_vk_xb_step_end();   /* rows stay readable until the next step */
        return ok;
    }
    double t0 = vkt_now_ms(), dms = 0, dms1 = 0, tc = t0 - T.t_issued;
    T.cpu_ms += tc;
    int ok = !T.d0_inflight || coli_vk_xb_join(T.by, &dms);
    int ok1 = !T.d2_inflight || coli_vk_xb_join_dev(1, T.by + T.d2_rows0, &dms1);
    T.d0_inflight = T.d2_inflight = 0;
    double tw = vkt_now_ms() - t0;
    T.wait_ms += tw;
    T.dev_ms += dms + dms1; T.d2_ms += dms1;
    if (dms1 > dms) dms = dms1;   /* the two ran side by side: the step waited for the slower */
    /* The balance moves only on steps where it had a choice. Waiting for more than
     * a tenth of the CPU's own time: give the CPU more. The device done well before
     * the CPU (its timestamps say so; without them, nothing waited): take more. */
    if (T.balance && T.can_balance) {
        if (tw > 0.1 * tc + 0.02) T.share *= 0.92f;
        else if (dms > 0 ? dms < 0.8 * tc : tw < 0.01) T.share = T.share * 1.04f + 0.01f;
        if (T.share > 1.f) T.share = 1.f;
        if (T.share < 0.05f) T.share = 0.05f;
    }
    T.inflight = 0;
    int n = T.S * T.K;
    for (int i = 0; i < n; i++) rows[i] = ok && ok1 && T.map[i] >= 0 ? T.by[T.map[i]] : NULL;
    if (!ok) {
        fprintf(stderr, "[VK] tier %s: a batch failed, its experts are recomputed on the CPU and the tier stops\n", T.engine);
        T.on = 0;
        return 0;
    }
    if (!ok1) {   /* the step's device rows are recomputed on the CPU (join's 0), the tier goes on */
        d2_stop("a batch on the second device failed");
        return 0;
    }
    quiesce();
    return 1;
}

/* vkt_join, and for a big step that ran wholly on the device (every routed assignment
 * taken) on the backend's whole-step buffers, the routed sum per row on the device:
 * *sum = S rows of sum over k in rank order of w[s*K+k] * expert(s, k), as an fma chain
 * from 0 (the engines' own loop); else *sum = NULL and rows as vkt_join gives them. */
int vkt_join_sum(const float **rows, const float *w, const float **sum) {
    *sum = NULL;
    if (!T.inflight) return 0;
    if (!T.big || !T.sy_step) return vkt_join(rows);
    int ok = join_big(rows);
    if (ok && T.big_taken == T.big_valid && T.big_valid > 0) {
        uint8_t *use = malloc((size_t)T.big_n);
        if (use) {
            for (int i = 0; i < T.big_n; i++) use[i] = T.map[i] >= 0;
            double dms = 0, t0 = vkt_now_ms();
            *sum = coli_vk_xb_step_sum(w, use, &dms);
            T.wait_ms += vkt_now_ms() - t0; T.dev_ms += dms;
            if (*sum) T.st_sums++;
            free(use);
        }
    }
    coli_vk_xb_step_end();
    return ok;
}

void vkt_begin_forward(void) { if (T.on) T.begin = 1; }
int vkt_resident(int layer, int eid) {
    return T.on && layer >= 0 && layer < T.c.layers && eid >= 0 && eid < T.c.experts &&
           slot(layer, eid)->state == VS_RESIDENT;
}
int vkt_ready(void) { return T.on; }

/* ---- budget ---------------------------------------------------------------------- */
static size_t mem_available(void) { return (size_t)(compat_mem_available_gb() * 1e9); }
static double env_gb(const char *name, double def) {
    const char *e = getenv(name);
    return e && *e ? atof(e) : def;
}

int vkt_wanted(void) {
    const char *on = getenv("COLI_VK_TIER");
    return !(on && *on == '0');
}

/* A form of the experts: the device formats and the bytes of one expert as the engine
 * holds it. 0 when it has no device form. */
static int form_init(int k, VktFmt gu, VktFmt dn) {
    __typeof__(T.form[0]) *fm = &T.form[k];
    const int H = T.c.hidden, F = T.c.inter;
    if (!dev_fmt(gu, &fm->gu_fmt, &fm->gu_gs) || !dev_fmt(dn, &fm->dn_fmt, &fm->dn_gs)) return 0;
    fm->gu = gu; fm->dn = dn;
    fm->gu_codes = src_row_bytes(gu.kind, H) * (size_t)F;
    fm->d_codes = src_row_bytes(dn.kind, F) * (size_t)H;
    fm->gu_scales = src_scale_bytes(gu, H, F);
    fm->d_scales = src_scale_bytes(dn, F, H);
    fm->stage_bytes = 2 * fm->gu_codes + fm->d_codes + 2 * fm->gu_scales + fm->d_scales;
    return 1;
}

/* COLI_VK_DEV2: the second device's share. Its budget is what it has free less
 * COLI_VK_RESERVE2_GB (0.5 GiB: its batch's scratch, the driver), an integrated GPU's a
 * quarter of what the RAM leaves (as the primary's), capped by COLI_VK_EXPERTS2 experts
 * when set; it holds at most the experts the primary device does not. */
static void d2_init(long long fit0, long long all, int act) {
    if (!getenv("COLI_VK_DEV2") || !*getenv("COLI_VK_DEV2")) return;
    const char *eng = T.engine;
    if (!coli_vk_dev2_open_env()) {
        fprintf(stderr, "[VK] tier %s: COLI_VK_DEV2=%s brought no second device up, the tier stays on %s\n",
                eng, getenv("COLI_VK_DEV2"), coli_vk_device_name());
        return;
    }
    ColiVkDevInfo di;
    if (!coli_vk_dev_info(1, &di) || !coli_vk_xb_init_dev(1, T.c.hidden, T.c.inter, act, T.c.act_limit, T.c.act_a, T.c.act_b)) {
        fprintf(stderr, "[VK] tier %s: no expert batch on the second device (shaders?), the tier stays on %s\n",
                eng, coli_vk_device_name());
        return;
    }
    snprintf(T.d2_name, sizeof T.d2_name, "%s", di.name ? di.name : "?");
    const double GiB = 1073741824.0;
    T.d2_exp_bytes = expert_bytes_at(T.c.hidden, T.c.inter, T.c.gate_up, T.c.down, di.buf_align ? di.buf_align : 256);
    double heap_free = di.has_budget ? (double)di.free_bytes : (double)di.local_bytes * 0.8;
    double room = heap_free - env_gb("COLI_VK_RESERVE2_GB", 0.5) * GiB, want = room;
    if (di.shares_ram) {
        double avail = (double)mem_available();
        double ram_room = avail > 0 ? avail - (double)T.c.ram_reserve - (double)T.c.dense_bytes - 2.0 * GiB : room;
        if (ram_room / 4 < want) want = ram_room / 4;
    }
    if (want < 0) want = 0;
    size_t blk = coli_vk_block_bytes_dev(1, (size_t)256 << 20);
    if ((size_t)want < blk) blk = (size_t)want;
    long long fit = 0;
    if (T.d2_exp_bytes && blk >= T.d2_exp_bytes) {
        size_t full = (size_t)want / blk, rest = (size_t)want - full * blk;
        fit = (long long)full * (long long)(blk / T.d2_exp_bytes) + (long long)(rest / T.d2_exp_bytes);
    }
    long long fit_raw = fit, left = all - fit0;
    const char *cap = getenv("COLI_VK_EXPERTS2");
    if (cap && *cap && atoll(cap) >= 0 && fit > atoll(cap)) fit = atoll(cap);
    if (fit > left) fit = left;
    char hb[32], he[32];
    if (fit < 1) {
        fprintf(stderr, "[VK] tier %s: second device %s: %s, the tier stays on %s\n", eng, T.d2_name,
                left < 1 ? "the primary device holds every expert" : "no room for experts (COLI_VK_RESERVE2_GB sets its reserve)",
                coli_vk_device_name());
        return;
    }
    /* a pool no bigger than its experts when they all fit in one block (a small model) */
    size_t lim = (size_t)want, need = (size_t)(fit + 1) * T.d2_exp_bytes;
    if (fit_raw >= left && need < lim && need <= coli_vk_block_bytes_dev(1, (size_t)256 << 20)) lim = need;
    coli_vk_tier_pool_limit_dev(1, lim);
    T.d2 = 1; T.dmax[1] = (int)fit; T.max_resident += (int)fit; T.d2_budget = (size_t)want;
    /* a big step's rows on it: their x rows within 128 MiB */
    long hr = (128L << 20) / ((long)T.c.hidden * 4);
    T.d2_rows = hr < 64 ? 64 : hr > 65535 ? 65535 : (int)hr;
    fprintf(stderr, "[VK] tier %s: second device %s, budget %s = %d experts of %s (%s), the experts after the primary device's\n",
            eng, T.d2_name, human((double)T.d2_budget, hb, sizeof hb), T.dmax[1], human((double)T.d2_exp_bytes, he, sizeof he),
            di.shares_ram ? "shared RAM" : "device memory");
}

int vkt_init(const VktConfig *cfg, uint32_t *const *heat) {
    if (T.on || !cfg) return 0;
    if (!vkt_wanted()) return 0;
    if (!coli_vk_available()) return 0;
    const char *eng = cfg->engine ? cfg->engine : "engine";
    memset(&T, 0, sizeof T);
    T.c = *cfg;
    snprintf(T.engine, sizeof T.engine, "%s", eng);
    if (T.c.layers < 1 || T.c.experts < 1 || T.c.hidden < 1 || T.c.inter < 1 || T.c.topk < 1) return 0;
    T.main_layers = T.c.layers;
    if (!form_init(0, T.c.gate_up, T.c.down)) {
        fprintf(stderr, "[VK] tier %s: expert format %d/%d (gs %d/%d) has no device form, the experts stay on the CPU\n",
                eng, T.c.gate_up.kind, T.c.down.kind, T.c.gate_up.gs, T.c.down.gs);
        return 0;
    }
    int act = T.c.act == VKT_ACT_SITU ? COLI_VK_ACT_SITU
            : T.c.act == VKT_ACT_SWIGLU_V4 ? COLI_VK_ACT_SWIGLU_V4 : COLI_VK_ACT_SWIGLU;
    if (!coli_vk_xb_init(T.c.hidden, T.c.inter, act, T.c.act_limit, T.c.act_a, T.c.act_b)) {
        fprintf(stderr, "[VK] tier %s: no expert batch on this device (shaders?), the experts stay on the CPU\n", eng);
        return 0;
    }
    int H = T.c.hidden, F = T.c.inter;
    T.exp_bytes = vkt_expert_bytes(H, F, T.c.gate_up, T.c.down);
    /* the extra layers (an MTP head's): on the tier when their form has a device form */
    if (T.c.extra_layers > 0) {
        if (form_init(1, T.c.extra_gate_up, T.c.extra_down) && T.exp_bytes &&
            (T.x_bytes = vkt_expert_bytes(H, F, T.c.extra_gate_up, T.c.extra_down)))
            T.c.layers += T.c.extra_layers;
        else
            fprintf(stderr, "[VK] tier %s: the extra layers' expert format %d/%d (gs %d/%d) has no device form, their experts stay on the CPU\n",
                    eng, T.c.extra_gate_up.kind, T.c.extra_down.kind, T.c.extra_gate_up.gs, T.c.extra_down.gs);
    }
    if (T.main_layers == T.c.layers) T.c.extra_layers = 0;

    /* The budget. Discrete: what VK_EXT_memory_budget says is free on the device,
     * minus a reserve (scratch, KV mirrors, driver) and the dense weights still to
     * come. Integrated (device memory is host RAM): a quarter of what the RAM has
     * left after the engine's expert cache has grown to its size and the dense
     * weights are placed, so the tier never takes what the CPU cache needs.
     * COLI_VK_TIER_GB replaces either, within what the device can hold. */
    const double GiB = 1073741824.0;
    double used = 0, bud = 0;
    int have = coli_vk_mem_budget(&used, &bud);
    double heap_free = have ? (bud - used) * 1e9 : (double)coli_vk_device_local_bytes() * 0.8;
    T.uma = coli_vk_device_shares_ram();
    double reserve = env_gb("COLI_VK_TIER_RESERVE_GB", 1.0) * GiB;
    double room = heap_free - reserve - (double)T.c.dense_bytes;
    double want;
    const char *cap = getenv("COLI_VK_TIER_GB");
    if (cap && *cap) want = atof(cap) * GiB;
    else if (T.uma) {
        double avail = (double)mem_available();
        double ram_room = avail > 0 ? avail - (double)T.c.ram_reserve - (double)T.c.dense_bytes - 2.0 * GiB : room;
        want = ram_room / 4;
    } else want = room;
    if (want > room) want = room;
    if (want < 0) want = 0;
    /* whole experts per pool block, so the count matches what the blocks can hold */
    size_t blk = coli_vk_block_bytes((size_t)256 << 20);   /* the pool's blocks (smaller under COLI_VK_DEVICE_CAP_MB) */
    if ((size_t)want < blk) blk = (size_t)want;
    long long fit = 0;
    if (T.exp_bytes && blk >= T.exp_bytes) {
        size_t full = (size_t)want / blk, rest = (size_t)want - full * blk;
        fit = (long long)full * (long long)(blk / T.exp_bytes) + (long long)(rest / T.exp_bytes);
    }
    long long all = (long long)T.main_layers * T.c.experts, fit_raw = fit;
    /* The extra layers' pool: all their experts when the budget holds them beside every
     * main one, else their layers' share of the budget (at least one expert, when two
     * main ones still fit beside it). Its room, in main experts, leaves the main count. */
    if (T.c.extra_layers) {
        long long xall = (long long)T.c.extra_layers * T.c.experts, xn;
        double bud = (double)fit_raw * T.exp_bytes;
        if (bud >= (double)all * T.exp_bytes + (double)xall * T.x_bytes) xn = xall;
        else xn = (long long)(bud * T.c.extra_layers / (T.main_layers + T.c.extra_layers) / T.x_bytes);
        if (xn < 1 && bud >= (double)T.x_bytes + 2.0 * T.exp_bytes) xn = 1;
        if (xn > xall) xn = xall;
        if (xn < 1) {
            char hx[32];
            fprintf(stderr, "[VK] tier %s: no room for an extra layer's expert (%s), they stay on the CPU\n",
                    eng, human((double)T.x_bytes, hx, sizeof hx));
            T.c.layers = T.main_layers; T.c.extra_layers = 0;
        } else {
            long long take = (long long)(((size_t)xn * T.x_bytes + T.exp_bytes - 1) / T.exp_bytes);
            T.xmax = (int)xn; T.x_budget = (size_t)take * T.exp_bytes; fit_raw -= take;
        }
    }
    if (fit > all) fit = all;
    if (fit > fit_raw) fit = fit_raw;
    if (fit < 2) {
        char hb[32], he[32];
        fprintf(stderr, "[VK] tier %s: no room for experts (budget %s, %s each; COLI_VK_TIER_GB sets it), the experts stay on the CPU\n",
                eng, human(want, hb, sizeof hb), human((double)T.exp_bytes, he, sizeof he));
        return 0;
    }
    if (T.c.max_experts > 0 && fit > T.c.max_experts) fit = T.c.max_experts;   /* the engine's count cap */
    /* Streaming (big prefill steps, see "streaming" above): COLI_VK_TIER_STREAM=0 turns
     * it off; it needs the engine's load hook and the GEMM route. Its staging slots come
     * out of the budget: the residents give up what the budget cannot hold beside them. */
    {
        ColiVkXbStats xs; coli_vk_xb_stats(&xs);
        T.gemm_rows = xs.gemm_rows;
        const char *se = getenv("COLI_VK_TIER_STREAM"), *sn = getenv("COLI_VK_TIER_STREAM_SLOTS");
        const char *sr = getenv("COLI_VK_TIER_STREAM_ROWS");
        int want_st = !(se && *se == '0') && T.c.load && T.c.release && T.gemm_rows > 0;
        int slots = sn && *sn ? atoi(sn) : 64;
        if (slots > fit_raw / 2) slots = (int)(fit_raw / 2);
        if (want_st && slots >= 2) {
            if (fit > fit_raw - slots) fit = fit_raw - slots;
            T.st_ok = fit >= 2;
            if (!T.st_ok) fit = fit_raw > all ? all : fit_raw;
        }
        if (T.st_ok) {
            T.st_slots = slots;
            T.st_rows_env = sr && *sr ? atoi(sr) : 0;
            /* a sub-batch's rows: its x rows within 32 MiB (a card without Resizable BAR
             * keeps them in its host-visible window) */
            long hr = (32L << 20) / ((long)T.c.hidden * 4);
            T.st_half = hr < 1 ? 1 : hr > 65535 ? 65535 : (int)hr;
            const char *sp = getenv("COLI_VK_TIER_STREAM_PAR");
            T.st_par = !(sp && *sp == '0');
            const char *sh = getenv("COLI_VK_TIER_STREAM_HALF");   /* tests: small sub-batches */
            if (sh && *sh && atoi(sh) > 0) T.st_half = atoi(sh) > 65535 ? 65535 : atoi(sh);
            T.bcnt = malloc((size_t)T.c.experts * sizeof(int)); T.bofs = malloc((size_t)T.c.experts * sizeof(int));
            T.bcls = malloc((size_t)T.c.experts * sizeof(int));
            T.pred = calloc((size_t)T.c.layers * T.c.experts, sizeof(uint32_t)); T.pred_ok = calloc((size_t)T.c.layers, 1);
            if (!T.bcnt || !T.bofs || !T.bcls || !T.pred || !T.pred_ok) T.st_ok = 0;
        } else if (want_st)
            fprintf(stderr, "[VK] tier %s: no room for streaming slots beside the residents, cold experts stay on the CPU\n", eng);
    }
    T.max_resident = (int)fit;
    T.dmax[0] = (int)fit;
    T.budget = (size_t)want;
    /* Every expert fits in one block's worth: the pool's limit (and so its one block)
     * is what they take, not the budget, so a small model does not hold a 256 MB
     * block for a few experts. */
    size_t lim = T.budget - T.x_budget, need = (size_t)(all + 1 + (T.st_ok ? T.st_slots : 0)) * T.exp_bytes;
    if (fit + (T.st_ok ? T.st_slots : 0) >= all && fit_raw >= all + (T.st_ok ? T.st_slots : 0) &&
        need < lim && need <= coli_vk_block_bytes((size_t)256 << 20)) lim = need;
    coli_vk_tier_pool_limit(lim);
    if (T.xmax) coli_vk_tier_extra_pool_limit(T.x_budget);
    const char *ex = getenv("COLI_VK_TIER_EXCLUSIVE");
    T.excl = !(ex && *ex == '0');
    if (T.excl) {
        T.routed_at = calloc((size_t)T.c.layers * T.c.experts, sizeof(uint32_t));
        T.layer_seq = calloc((size_t)T.c.layers, sizeof(uint32_t));
        if (!T.routed_at || !T.layer_seq) { free(T.routed_at); free(T.layer_seq); T.routed_at = T.layer_seq = NULL; T.excl = 0; }
    }
    const char *bal = getenv("COLI_VK_TIER_BALANCE");
    T.balance = T.c.in_ram != NULL && !(bal && *bal == '0');
    T.share = 1.f;
    const char *r = getenv("COLI_VK_TIER_RATE");
    T.rate = r && *r ? atoi(r) : 16;
    if (T.rate < 0) T.rate = 0;
    T.s = calloc((size_t)T.c.layers * T.c.experts, sizeof(VSlot));
    T.grp = malloc((size_t)T.c.experts * sizeof(int));
    if (!T.s || !T.grp) { free(T.s); free(T.grp); return 0; }
    for (int e = 0; e < T.c.experts; e++) T.grp[e] = -1;
    T.max_dev_rows = T.c.max_rows > 0 ? T.c.max_rows : 1 << 20;
    T.last_layer = 1 << 30; T.first_layer = -1; T.decay_at = VKT_DECAY_TOKENS;
    /* The history, scaled: the hottest expert of a layer starts at 32 and the rest in
     * proportion, so a few minutes of a new workload can displace it (raw counts
     * from a long history would hold the tier for hours). */
    g_heat_hist = heat;
    if (heat)
        for (int l = 0; l < T.main_layers; l++) {   /* the history has the main layers */
            if (!heat[l]) continue;
            uint32_t mx = 0;
            for (int e = 0; e < T.c.experts; e++) if (heat[l][e] > mx) mx = heat[l][e];
            if (mx) for (int e = 0; e < T.c.experts; e++)
                if (heat[l][e]) T.s[(size_t)l * T.c.experts + e].heat = 1 + (uint32_t)(31.0 * heat[l][e] / mx);
        }
    if (pthread_mutex_init(&T.mx, NULL) || pthread_cond_init(&T.cv, NULL) || pthread_cond_init(&T.cv_room, NULL) ||
        pthread_cond_init(&T.cv_done, NULL)) return 0;
    { const char *sy = getenv("COLI_VK_TIER_SYNC"); T.sync = sy && *sy == '1'; }
    if (pthread_create(&T.th, NULL, uploader, NULL)) return 0;
    T.th_on = 1;
    T.on = 1;
    char hb[32], he[32];
    fprintf(stderr, "[VK] tier %s: on, %s, budget %s = %d experts of %s (fmt %d", eng,
            coli_vk_device_name(), human((double)T.budget, hb, sizeof hb), T.dmax[0],
            human((double)T.exp_bytes, he, sizeof he), T.form[0].gu_fmt);
    if (T.form[0].gu_gs) fprintf(stderr, " gs %d", T.form[0].gu_gs);
    fprintf(stderr, ", down fmt %d", T.form[0].dn_fmt);
    if (T.form[0].dn_gs) fprintf(stderr, " gs %d", T.form[0].dn_gs);
    fprintf(stderr, "), %s, %s queue, up to %d promotions per token%s%s",
            T.uma ? (cap && *cap ? "shared RAM (COLI_VK_TIER_GB)" : "shared RAM: a quarter of what the expert cache leaves")
                  : "device memory", coli_vk_xb_queue_shared() ? "shared" : "own", T.rate,
            T.balance ? ", balanced against the CPU" : "", T.sync ? ", uploads awaited (COLI_VK_TIER_SYNC)" : "");
    if (T.st_ok) fprintf(stderr, ", streaming cold experts of prompt steps through %d slots", T.st_slots);
    if (T.xmax) {
        char hx[32], hp[32];
        fprintf(stderr, "; the extra layers' experts (%d, fmt %d): %d of %s in a pool of %s", T.c.extra_layers,
                T.form[1].gu_fmt, T.xmax, human((double)T.x_bytes, hx, sizeof hx), human((double)T.x_budget, hp, sizeof hp));
    }
    fprintf(stderr, "\n");
    d2_init(fit, all, act);   /* COLI_VK_DEV2: the experts after these on a second device */
    return 1;
}

void vkt_report(const char *scope, unsigned long long ram_hits, unsigned long long disk_loads) {
    if (!T.s) return;
    ColiVkPoolStats ps; coli_vk_pool_stats(1, &ps);
    unsigned long long r = T.routed - T.rep_routed, sv = T.served - T.rep_served;
    T.rep_routed = T.routed; T.rep_served = T.served;
    double hidden = T.dev_ms > T.wait_ms ? T.dev_ms - T.wait_ms : 0;
    char hu[32], hb[32], hl[32];
    fprintf(stderr, "[VK] tier %s %s: device %llu of %llu routed experts (%.1f%%; this %s %llu of %llu) | "
            "CPU RAM hits %llu, disk loads %llu | resident %d (budget %d, %s of %s, %d blocks, frag %.2f) | "
            "uploads %llu (%s, %llu warm), evictions %llu, skipped %llu queue + %llu rate, failed %llu | "
            "device %.1f ms, CPU share %.1f ms, waited %.1f ms (%.0f%% of device time hidden)",
            T.engine, scope, T.served, T.routed, T.routed ? 100.0 * T.served / T.routed : 0.0, scope, sv, r,
            ram_hits, disk_loads, T.dres[0], T.dmax[0], human((double)ps.used, hu, sizeof hu),
            human((double)T.budget, hb, sizeof hb), ps.blocks, ps.frag, T.uploads,
            human((double)T.upload_bytes, hl, sizeof hl), T.warm, T.evictions, T.qfull, T.rated, T.failed,
            T.dev_ms, T.cpu_ms, T.wait_ms, T.dev_ms > 0 ? 100.0 * hidden / T.dev_ms : 0.0);
    if (T.balance) fprintf(stderr, " | balance: device share %.2f, %llu rows handed to the CPU", T.share, T.handed);
    if (T.c.extra_layers)
        fprintf(stderr, " | extra layers (%d): %llu of %llu routed experts on the device, resident %d (budget %d)",
                T.c.extra_layers, T.extra_served, T.extra_routed, T.xres, T.xmax);
    if (T.excl) fprintf(stderr, " | exclusive: %llu RAM copies of device experts given up first",
                        __atomic_load_n(&T.ram_gave, __ATOMIC_RELAXED));
    if (T.st_ok && T.st_steps) {
        char hs[32];
        fprintf(stderr, " | stream: %llu steps, %llu cold experts (%s, %.2f GB/s) and %llu rows streamed in %llu sub-batches, "
                "prefetched %llu (%llu used), %llu cold experts (%llu rows) kept on the CPU",
                T.st_steps, T.st_experts, human((double)T.st_bytes, hs, sizeof hs),
                T.st_up_ms > 0 ? (double)T.st_bytes / T.st_up_ms / 1e6 : 0.0, T.st_rows, T.st_subs, T.pf_n, T.pf_used,
                T.st_kept, T.st_kept_rows);
        if (T.st_sums) fprintf(stderr, ", %llu steps summed on the device", T.st_sums);
    }
    if (T.d2 || T.d2_lost) {
        ColiVkPoolStats p2; coli_vk_pool_stats(3, &p2);
        char h2[32], b2[32];
        fprintf(stderr, " | second device %s: resident %d (budget %d, %s of %s), %llu rows in %llu batches (%llu beside prompt steps), "
                "uploads %llu, %.1f ms%s", T.d2_name, T.dres[1], T.dmax[1], human((double)p2.used, h2, sizeof h2),
                human((double)T.d2_budget, b2, sizeof b2), T.d2_served, T.d2_steps, T.d2_big, T.d2_uploads, T.d2_ms,
                T.d2 ? "" : " (stopped)");
    }
    ColiVkXbStats xs; coli_vk_xb_stats(&xs);
    if (xs.cooperative_matmuls)
        fprintf(stderr, " | cooperative matmuls: %llu", xs.cooperative_matmuls);
    fprintf(stderr, "\n");
}

int vkt_devices(void) { return !T.on ? 0 : T.d2 ? 2 : 1; }
int vkt_layers(void) { return T.on ? T.c.layers : 0; }
static VSlot *g_old_slots;   /* the slot table of the tier shut down last (vkt_shutdown) */
static uint32_t *g_old_routed, *g_old_seq;

/* The engine's prefetch workers ask too, while the engine thread changes the slot table:
 * the state byte is read atomically (a stale answer costs one eviction order or one
 * read, never a wrong expert: the engine's own cache decides what its slots hold). */
int vkt_ram_first(int layer, int eid) {
    VSlot *s = __atomic_load_n(&T.s, __ATOMIC_ACQUIRE);
    uint32_t *routed = __atomic_load_n(&T.routed_at, __ATOMIC_ACQUIRE), *seq = __atomic_load_n(&T.layer_seq, __ATOMIC_ACQUIRE);
    int L = T.c.layers, E = T.c.experts;
    if (!__atomic_load_n(&T.on, __ATOMIC_RELAXED) || !T.excl || !s || !routed || !seq ||
        layer < 0 || layer >= L || eid < 0 || eid >= E) return 0;
    size_t i = (size_t)layer * E + eid;
    if (__atomic_load_n(&routed[i], __ATOMIC_RELAXED) == __atomic_load_n(&seq[layer], __ATOMIC_ACQUIRE)) return 0;
    return __atomic_load_n(&s[i].state, __ATOMIC_RELAXED) == VS_RESIDENT;
}
void vkt_ram_gave(void) { __atomic_add_fetch(&T.ram_gave, 1, __ATOMIC_RELAXED); }

void vkt_shutdown(void) {
    if (!T.s) return;
    if (T.th_on) {
        pthread_mutex_lock(&T.mx); T.stop = 1; pthread_cond_broadcast(&T.cv); pthread_cond_broadcast(&T.cv_room); pthread_cond_broadcast(&T.cv_done); pthread_mutex_unlock(&T.mx);
        pthread_join(T.th, NULL);
        T.th_on = 0;
    }
    int big = T.inflight && T.big;
    if (big) { st_join_half(0); st_join_half(1); }
    else if (T.inflight && T.d0_inflight) coli_vk_xb_join(T.by, NULL);
    if (T.d2_inflight) coli_vk_xb_join_dev(1, big ? T.d2_y : T.by + T.d2_rows0, NULL);   /* the second device's batch */
    T.inflight = T.big = T.d0_inflight = T.d2_inflight = 0;
    for (int i = 0; i < T.qn; i++) free(T.q[(T.qh + i) % VKT_QCAP].buf);
    T.qn = 0;
    if (coli_vk_available()) {
        quiesce();
        size_t n = (size_t)T.c.layers * T.c.experts;
        for (size_t i = 0; i < n; i++) if (T.s[i].ex) { coli_vk_xb_expert_free(T.s[i].ex); T.s[i].ex = NULL; }
        for (int i = 0; T.st && i < T.st_n; i++) if (T.st[i].ex) coli_vk_xb_expert_free(T.st[i].ex);
    }
    pthread_mutex_destroy(&T.mx); pthread_cond_destroy(&T.cv); pthread_cond_destroy(&T.cv_room); pthread_cond_destroy(&T.cv_done);
    /* the slot table outlives the tier by one shutdown: a prefetch worker may be in
     * vkt_ram_first with the pointer it read before this */
    free(g_old_slots); g_old_slots = T.s;
    free(g_old_routed); g_old_routed = T.routed_at;   /* the same for the routing marks */
    free(g_old_seq); g_old_seq = T.layer_seq;
    free(T.grp); free(T.map); free(T.touched); free(T.bex); free(T.brows); free(T.bfirst);
    free(T.bx); free(T.by); free(T.bw); free(T.evict); free(T.done);
    free(T.st); free(T.sy); free(T.bcnt); free(T.bofs); free(T.bcls); free(T.blist); free(T.pred); free(T.pred_ok);
    for (int h = 0; h < 2; h++) { free(T.hf[h].slot); free(T.hf[h].yout); }
    free(T.d2_idx); free(T.d2_y); free(T.d2_bex); free(T.d2_brows); free(T.d2_bx); free(T.d2_bw);
    memset(&T, 0, sizeof T);
}
#endif /* COLI_VULKAN */