Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
80.1 KB
Raw Blame
/* Indicizzazione e lettura on-demand di tensori da piu' file safetensors.
 * Equivale a Shards in engine.py, ma:
 *   - legge con pread (niente mmap) + posix_fadvise(DONTNEED) -> le pagine NON
 *     restano residenti nel processo. E' la correzione del bug di RSS: cosi' la
 *     RAM di picco resta densa+cache, non l'intero modello. (vedi memoria mmap-rss-bug)
 *   - converte sempre in float32 in uscita (BF16/F16/F32 supportati). */
#ifndef ST_H
#define ST_H
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>   /* ldexpf per ue8m0_to_f32 */
#include <stdint.h>
#include <fcntl.h>
#include <errno.h>
#include <unistd.h>
#include <dirent.h>
#include <sys/stat.h>
#include "json.h"
#include "compat.h"
#include "load_fail.h"

/* tetto sulla dimensione dell'header safetensors: gli header reali sono piccoli
 * (KB..pochi MB). Un file crafted che dichiara un hlen enorme causerebbe una
 * malloc gigante prima ancora di leggere: lo respingiamo. */
#define ST_MAX_HEADER (512ll << 20)
#define ST_MAX_RANK 8

typedef struct {
    char   *name;
    int     fd;
    int64_t off;       /* offset assoluto del dato dentro al file */
    int64_t nbytes;
    int     dtype;     /* 0=BF16 1=F16 2=F32 3=U8/I8 4=F8_E4M3 5=F8_E8M0 6=I64 7=U32/I32 */
    int64_t numel;
    int     rank;
    int64_t shape[ST_MAX_RANK];
} st_tensor;

typedef struct {
    st_tensor *t;
    int        n, cap;
    int        fds[512];
    int        dfds[512];  /* gemelli O_DIRECT (aperti pigramente): -2 = non ancora provato */
    char      *paths[512];
    int64_t    sizes[512];  /* indexed primary shard sizes, parallel to fds/paths */
    int        nfd;
#define ST_MAX_MIR 4       /* extra read replicas beyond the primary (multi-SSD) */
    int        mfds[ST_MAX_MIR][512];  /* MIRROR: fds of replica copy r+1 (multi-SSD), -1 = absent */
    int        mdfds[ST_MAX_MIR][512]; /* O_DIRECT twins of the replica copies, -1 = absent */
    int        nmirror[ST_MAX_MIR];    /* files accepted into replica r+1 */
    int        nrep;       /* registered replica copies (0 = mirror inactive) */
    int       *hidx;      /* hash map nome->indice (open addressing): con ~120k tensori
                           * (GLM: 256 expert x 78 layer x 3 x 2) la scansione lineare
                           * costava decine di secondi/token (misurato sul primo run reale) */
    int        hcap;
    /* FORMAT METADATA STAMP (reference impl, see colibri.c's qt_verify_fmt_stamp):
     * per-tensor {name -> format NAME string} pairs collected from every shard's
     * __metadata__["colibri.fmt"] JSON blob (safetensors __metadata__ values are
     * always strings, so colibri.fmt's value is itself JSON text, parsed a
     * second time -- see st_init_multi below). Small in practice (only the
     * tensors a stamping tool selected, a subset of S->t), so a flat array +
     * linear st_fmt_stamp() lookup is fine; this is a reference implementation,
     * not a framework -- no hash map for a handful-to-low-thousands of entries.
     * Both arrays own strdup'd strings, intentionally leaked like the rest of
     * st_init_multi's one-time startup parsing (see the json_parse callers
     * below). */
    char     **fmt_name;   /* stamped tensor name */
    char     **fmt_val;    /* stamped format NAME string */
    int        fmt_n, fmt_cap;
} shards;
#define ST_MAX_SHARDS 512

static uint64_t st_hash(const char *s){
    uint64_t h=1469598103934665603ULL;
    while(*s){ h^=(unsigned char)*s++; h*=1099511628211ULL; }
    return h;
}

static int st_dtype_code(const char *s) {
    if (!strcmp(s, "BF16")) return 0;
    if (!strcmp(s, "F16"))  return 1;
    if (!strcmp(s, "F32"))  return 2;
    if (!strcmp(s, "U8"))   return 3;   /* dati quantizzati (int4 packed / int8) */
    if (!strcmp(s, "I8"))   return 3;
    /* --- tipi dei checkpoint nativi fp8 (DeepSeek-V4, GLM-5.2-FP8 non ripacchettati) ---
     * PRIMA di questi, st_init faceva exit(1) su un checkpoint DeepSeek-V4 al primo
     * tensore I64, senza arrivare ai pesi. Sono INDICIZZATI qui e letti dal percorso
     * dei byte grezzi (st_read_raw); i lettori float li RIFIUTANO PER NOME invece di
     * caderci dentro -- vedi il commento in st_read_f32. Il loro codice numerico e'
     * nuovo e nessun codice esistente cambia: 0/1/2/3 restano quelli di prima. */
    if (!strcmp(s, "F8_E4M3") || !strcmp(s, "F8_E4M3FN") ||
        !strcmp(s, "float8_e4m3fn")) return 4;
    if (!strcmp(s, "F8_E8M0") || !strcmp(s, "F8_E8M0FNU")) return 5;
    if (!strcmp(s, "I64") || !strcmp(s, "U64")) return 6;
    /* I32/U32: token ids in reference dumps (tools/make_qwenimage_tiny.py), and
     * the packed U32 words of MLX affine-quantized checkpoints (Swiftlet qpack
     * containers included; qwen36's dense affine loader reads them through
     * st_read_raw after validating the packed geometry). Same rule as the fp8
     * codes above: a new number, read raw, refused by the float readers. */
    if (!strcmp(s, "I32") || !strcmp(s, "U32")) return 7;
    coli_load_fail(COLI_LOAD_FAIL_UNSUPPORTED, "unsupported dtype: %s", s);
}

/* Byte per elemento. UNICO posto che lo sa: prima la formula era ripetuta in tre
 * punti come `dtype==2 ? 4 : 2`, che con soli 0/1/2/3 era corretta e con i tipi
 * nuovi avrebbe silenziosamente detto "2 byte" per un I64 da 8. */
static inline int st_dtype_esz(int dtype) {
    switch (dtype) {
        case 2: return 4;                 /* F32 */
        case 3: case 4: case 5: return 1; /* U8/I8, F8_E4M3, F8_E8M0 */
        case 6: return 8;                 /* I64/U64 */
        case 7: return 4;                 /* I32/U32 */
        default: return 2;                /* BF16, F16 */
    }
}

/* Nome leggibile, per i messaggi di rifiuto. */
static inline const char *st_dtype_name(int dtype) {
    switch (dtype) {
        case 0: return "BF16"; case 1: return "F16"; case 2: return "F32";
        case 3: return "U8/I8"; case 4: return "F8_E4M3"; case 5: return "F8_E8M0";
        case 6: return "I64"; case 7: return "I32"; default: return "?";
    }
}

static inline float bf16_to_f32(uint16_t h) {
    uint32_t u = (uint32_t)h << 16; float f; memcpy(&f, &u, 4); return f;
}
static inline float f16_to_f32(uint16_t h) {
    uint32_t sign = (uint32_t)(h & 0x8000) << 16;
    uint32_t exp  = (h >> 10) & 0x1F;
    uint32_t man  = h & 0x3FF;
    uint32_t u;
    if (exp == 0) {            /* subnormale o zero */
        if (man == 0) u = sign;
        else { exp = 127 - 15 + 1; while (!(man & 0x400)) { man <<= 1; exp--; } man &= 0x3FF; u = sign | (exp << 23) | (man << 13); }
    } else if (exp == 0x1F) {  /* inf/nan */
        u = sign | 0x7F800000 | (man << 13);
    } else {
        u = sign | ((exp - 15 + 127) << 23) | (man << 13);
    }
    float f; memcpy(&f, &u, 4); return f;
}

/* ---- bulk BF16/F16 -> F32, AVX2/SSE4.1/scalar tiers --------------------
 * st_read_f32/st_read_slice_f32 convert whole tensors (up to the embed/
 * lm_head matrix, vocab*hidden elements) through bf16_to_f32/f16_to_f32 one
 * halfword at a time; that loop is pure overhead once the read syscall is
 * off the critical path. The tiers below batch it, same idea and layering
 * as gsgemv.h's AVX2/SSE4.1 kernels (immintrin.h only under the ISA guard,
 * sse41_kernels.h not needed here -- no FMA, no shared load helper to reuse).
 *
 * BF16 -> F32 is an exact zero-pad widening for every bit pattern (BF16
 * shares F32's 8-bit exponent field, so there is no special-casing --
 * zero, normal, subnormal, inf, NaN all take the same `<<16`), so the
 * vectorized tiers cannot disagree with the scalar reference.
 *
 * F16 -> F32's zero/normal/inf-NaN classes are each the same closed-form
 * bit algebra as the scalar reference above, just run on several lanes at
 * once -- no reassociation, nothing to round, so still bit-exact. True
 * subnormals (exp==0, man!=0) need the scalar reference's shift-to-normalize
 * loop, which does not vectorize; those (rare in real model weights) fall
 * back to f16_to_f32 per element. Exhaustive verification (all 65536
 * patterns per format) lives in tests/test_st_f16_bf16_simd.c. */
#if defined(__AVX2__) || defined(__SSE4_1__)
#include <immintrin.h>
#endif

#if defined(__AVX2__)
static void bf16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
    int64_t i = 0;
    for (; i + 8 <= n; i += 8) {
        __m128i h = _mm_loadu_si128((const __m128i*)(src + i));
        __m256i w = _mm256_slli_epi32(_mm256_cvtepu16_epi32(h), 16);
        _mm256_storeu_ps(dst + i, _mm256_castsi256_ps(w));
    }
    for (; i < n; i++) dst[i] = bf16_to_f32(src[i]);
}
static void f16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
    int64_t i = 0;
    const __m256i vsign_mask = _mm256_set1_epi32(0x8000);
    const __m256i vexp_mask  = _mm256_set1_epi32(0x1F);
    const __m256i vman_mask  = _mm256_set1_epi32(0x3FF);
    const __m256i v112       = _mm256_set1_epi32(112);
    const __m256i vinfnan_e  = _mm256_set1_epi32(0x7F800000);
    const __m256i vzero      = _mm256_setzero_si256();
    const __m256i v31        = _mm256_set1_epi32(31);
    for (; i + 8 <= n; i += 8) {
        __m128i h16 = _mm_loadu_si128((const __m128i*)(src + i));
        __m256i h    = _mm256_cvtepu16_epi32(h16);
        __m256i sign = _mm256_slli_epi32(_mm256_and_si256(h, vsign_mask), 16);
        __m256i exp  = _mm256_and_si256(_mm256_srli_epi32(h, 10), vexp_mask);
        __m256i man  = _mm256_and_si256(h, vman_mask);
        __m256i normal_u = _mm256_or_si256(sign, _mm256_or_si256(
            _mm256_slli_epi32(_mm256_add_epi32(exp, v112), 23), _mm256_slli_epi32(man, 13)));
        __m256i infnan_u = _mm256_or_si256(sign, _mm256_or_si256(vinfnan_e, _mm256_slli_epi32(man, 13)));
        __m256i exp_is_zero = _mm256_cmpeq_epi32(exp, vzero);
        __m256i man_is_zero = _mm256_cmpeq_epi32(man, vzero);
        __m256i is_zero    = _mm256_and_si256(exp_is_zero, man_is_zero);
        __m256i is_subnorm = _mm256_andnot_si256(man_is_zero, exp_is_zero);
        __m256i is_infnan  = _mm256_cmpeq_epi32(exp, v31);
        __m256i result = _mm256_blendv_epi8(normal_u, sign, is_zero);
        result = _mm256_blendv_epi8(result, infnan_u, is_infnan);
        _mm256_storeu_ps(dst + i, _mm256_castsi256_ps(result));
        int m = _mm256_movemask_ps(_mm256_castsi256_ps(is_subnorm));
        if (m) for (int k = 0; k < 8; k++) if ((m >> k) & 1) dst[i+k] = f16_to_f32(src[i+k]);
    }
    for (; i < n; i++) dst[i] = f16_to_f32(src[i]);
}
#elif defined(__SSE4_1__)
static void bf16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
    int64_t i = 0;
    for (; i + 4 <= n; i += 4) {
        __m128i h = _mm_loadl_epi64((const __m128i*)(src + i));
        __m128i w = _mm_slli_epi32(_mm_cvtepu16_epi32(h), 16);
        _mm_storeu_ps(dst + i, _mm_castsi128_ps(w));
    }
    for (; i < n; i++) dst[i] = bf16_to_f32(src[i]);
}
static void f16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
    int64_t i = 0;
    const __m128i vsign_mask = _mm_set1_epi32(0x8000);
    const __m128i vexp_mask  = _mm_set1_epi32(0x1F);
    const __m128i vman_mask  = _mm_set1_epi32(0x3FF);
    const __m128i v112       = _mm_set1_epi32(112);
    const __m128i vinfnan_e  = _mm_set1_epi32(0x7F800000);
    const __m128i vzero      = _mm_setzero_si128();
    const __m128i v31        = _mm_set1_epi32(31);
    for (; i + 4 <= n; i += 4) {
        __m128i h16 = _mm_loadl_epi64((const __m128i*)(src + i));
        __m128i h    = _mm_cvtepu16_epi32(h16);
        __m128i sign = _mm_slli_epi32(_mm_and_si128(h, vsign_mask), 16);
        __m128i exp  = _mm_and_si128(_mm_srli_epi32(h, 10), vexp_mask);
        __m128i man  = _mm_and_si128(h, vman_mask);
        __m128i normal_u = _mm_or_si128(sign, _mm_or_si128(
            _mm_slli_epi32(_mm_add_epi32(exp, v112), 23), _mm_slli_epi32(man, 13)));
        __m128i infnan_u = _mm_or_si128(sign, _mm_or_si128(vinfnan_e, _mm_slli_epi32(man, 13)));
        __m128i exp_is_zero = _mm_cmpeq_epi32(exp, vzero);
        __m128i man_is_zero = _mm_cmpeq_epi32(man, vzero);
        __m128i is_zero    = _mm_and_si128(exp_is_zero, man_is_zero);
        __m128i is_subnorm = _mm_andnot_si128(man_is_zero, exp_is_zero);
        __m128i is_infnan  = _mm_cmpeq_epi32(exp, v31);
        __m128i result = _mm_blendv_epi8(normal_u, sign, is_zero);
        result = _mm_blendv_epi8(result, infnan_u, is_infnan);
        _mm_storeu_ps(dst + i, _mm_castsi128_ps(result));
        int m = _mm_movemask_ps(_mm_castsi128_ps(is_subnorm));
        if (m) for (int k = 0; k < 4; k++) if ((m >> k) & 1) dst[i+k] = f16_to_f32(src[i+k]);
    }
    for (; i < n; i++) dst[i] = f16_to_f32(src[i]);
}
#else
static void bf16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
    for (int64_t i = 0; i < n; i++) dst[i] = bf16_to_f32(src[i]);
}
static void f16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
    for (int64_t i = 0; i < n; i++) dst[i] = f16_to_f32(src[i]);
}
#endif

static int st_open_fd(shards *S, const char *path) {
    for (int i = 0; i < S->nfd; i++) if (!strcmp(S->paths[i], path)) return S->fds[i];
    int fd = open(path, COMPAT_O_RDONLY);
    if (fd < 0) coli_load_fail(coli_load_fail_kind_from_errno(errno), "%s: %s", path, strerror(errno));
    struct stat sb;
    if (fstat(fd, &sb) != 0) { int saved = errno; close(fd);
        coli_load_fail(coli_load_fail_kind_from_errno(saved), "fstat shard: %s", strerror(saved)); }
    S->paths[S->nfd] = strdup(path); S->fds[S->nfd] = fd;
    S->sizes[S->nfd] = (int64_t)sb.st_size;
#ifdef O_DIRECT
    S->dfds[S->nfd] = open(path, COMPAT_O_RDONLY | O_DIRECT);   /* eager: lookup poi thread-safe */
#elif defined(__APPLE__) || defined(_WIN32)
    S->dfds[S->nfd] = compat_open_direct(path);          /* macOS: F_NOCACHE; Windows: NO_BUFFERING */
#else
    S->dfds[S->nfd] = -1;                                /* niente equivalente: solo buffered */
#endif
    S->nfd++;
    return fd;
}

/* fd gemello O_DIRECT dello stesso file (bypassa la page cache: il buffered read su
 * ext4-in-VHDX si strozza a ~0.8 GB/s, O_DIRECT arriva a 2.3+; misurato). -1 se non disponibile. */
static int st_fidx(shards *S, int fd) {
    for (int i = 0; i < S->nfd; i++) if (S->fds[i] == fd) return i;
    return -1;
}
static int st_direct_fd(shards *S, int fd) {
    int i = st_fidx(S, fd); return i < 0 ? -1 : S->dfds[i];
}

/* ---- MIRROR (multi-SSD): read-only copies of the model on other drives ----
 * st_fd_rep/st_direct_fd_rep: fd of replica `rep` (0 = primary, 1..nrep =
 * mirrors) for the SAME file identified by its primary fd. -1 if absent. */
static int st_fd_rep(shards *S, int fd, int rep) {
    if (!rep) return fd;
    if (rep > S->nrep) return -1;
    int i = st_fidx(S, fd); return i < 0 ? -1 : S->mfds[rep-1][i];
}
static int st_direct_fd_rep(shards *S, int fd, int rep) {
    if (!rep) return st_direct_fd(S, fd);
    if (rep > S->nrep) return -1;
    int i = st_fidx(S, fd); return i < 0 ? -1 : S->mdfds[rep-1][i];
}

/* Registers <dir>/<basename> as read replica S->nrep+1 of every already-indexed
 * shard. A file is accepted ONLY if its size and safetensors header are
 * byte-identical to the primary: the data_offsets then match by construction,
 * so every pread is valid on any copy. Missing or divergent files simply stay
 * on the primary (a mirror may be partial, e.g. a smaller SSD holding only the
 * expert shards). Returns the number of accepted files; a dir contributing 0
 * files claims no replica slot. Mirrors are NEVER written to: .coli_usage /
 * .coli_kv keep deriving from the primary alone. */
static void st_mirror_reset(shards *S) {           /* re-init: drop every replica */
    for (int r = 0; r < S->nrep; r++) for (int i = 0; i < S->nfd; i++) {
        if (S->mfds[r][i] >= 0) close(S->mfds[r][i]);
        if (S->mdfds[r][i] >= 0) close(S->mdfds[r][i]);
    }
    memset(S->nmirror, 0, sizeof(S->nmirror));
    S->nrep = 0;
}
static int st_mirror_add(shards *S, const char *dir) {
    if (S->nrep >= ST_MAX_MIR) return 0;
    int r = S->nrep;
    for (int i = 0; i < ST_MAX_SHARDS; i++) { S->mfds[r][i] = -1; S->mdfds[r][i] = -1; }
    S->nmirror[r] = 0;
    for (int i = 0; i < S->nfd; i++) {
        const char *base = strrchr(S->paths[i], '/');
#ifdef _WIN32
        const char *b2 = strrchr(S->paths[i], '\\');
        if (b2 && (!base || b2 > base)) base = b2;
#endif
        base = base ? base + 1 : S->paths[i];
        char mp[2048]; snprintf(mp, sizeof(mp), "%s/%s", dir, base);
        int mfd = open(mp, COMPAT_O_RDONLY);
        if (mfd < 0) continue;               /* partial mirror: this shard stays on the primary */
        int64_t sza = lseek(S->fds[i], 0, SEEK_END), szb = lseek(mfd, 0, SEEK_END);
        if (sza != szb) {
            fprintf(stderr, "[MIRROR] %s: size differs from the primary copy β€” file skipped\n", mp);
            close(mfd); continue;
        }
        uint64_t ha = 0, hb = 0; int ok = 1;   /* identical header => identical data_offsets */
        if (pread(S->fds[i], &ha, 8, 0) != 8 || pread(mfd, &hb, 8, 0) != 8 ||
            ha != hb || ha == 0 || ha > (uint64_t)256 << 20 || (int64_t)(8 + ha) > sza) ok = 0;
        if (ok) {
            char *ba = malloc(ha), *bb = malloc(ha);
            if (!ba || !bb || pread(S->fds[i], ba, ha, 8) != (ssize_t)ha ||
                pread(mfd, bb, ha, 8) != (ssize_t)ha || memcmp(ba, bb, ha)) ok = 0;
            free(ba); free(bb);
        }
        if (!ok) {
            fprintf(stderr, "[MIRROR] %s: header differs from the primary copy β€” file skipped\n", mp);
            close(mfd); continue;
        }
        S->mfds[r][i] = mfd;
#ifdef O_DIRECT
        S->mdfds[r][i] = open(mp, COMPAT_O_RDONLY | O_DIRECT);
#elif defined(__APPLE__) || defined(_WIN32)
        S->mdfds[r][i] = compat_open_direct(mp);
#endif
        S->nmirror[r]++;
    }
    if (S->nmirror[r] > 0) { S->nrep++; return S->nmirror[r]; }
    return 0;
}
/* backward-compatible single-mirror entry point */
static int st_mirror_init(shards *S, const char *dir) {
    st_mirror_reset(S);
    return st_mirror_add(S, dir);
}

/* indicizza tutti i model-*.safetensors in snap_dir */
/* pread completo: chunk-loop (una singola pread si ferma a ~2^31 byte su Linux
 * β€” i tensori bf16 grandi la superano), riprova su EINTR e riporta un errore
 * ONESTO: perror stampava "Success" su una short-read (errno resta 0), lo
 * stesso sintomo corretto in glm.c per #236. ST_PREAD_CHUNK e' sovrascrivibile
 * per i test. EN: full pread β€” chunk loop (one pread caps at ~2^31 bytes and
 * big bf16 tensors exceed it), EINTR retry, honest short-read errors.
 * Exits on failure, like every st.h reader. */
#ifndef ST_PREAD_CHUNK
#define ST_PREAD_CHUNK (1u << 30)
#endif
static void st_pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag) {
    char *p = (char *)buf;
    int64_t got = 0;
    while (got < n) {
        int64_t want = n - got;
        if (want > (int64_t)ST_PREAD_CHUNK) want = ST_PREAD_CHUNK;
        ssize_t r = pread(fd, p + got, (size_t)want, off + got);
        if (r < 0) {
            if (errno == EINTR) continue;
            int saved = errno;
            coli_load_fail(coli_load_fail_kind_from_errno(saved),
                           "%s: %s (off %lld, %lld/%lld bytes)", tag, strerror(saved),
                           (long long)off, (long long)got, (long long)n);
        }
        if (r == 0) {
            coli_load_fail(COLI_LOAD_FAIL_FORMAT,
                           "%s: short read at EOF (off %lld, %lld/%lld bytes) β€” truncated file?",
                           tag, (long long)off, (long long)got, (long long)n);
        }
        got += r;
    }
}

/* Stamps are a resident-tensor convention (see docs/FORMATS.md's "Stamp-map
 * scan bound"): a handful to a few hundred entries per model
 * (q_a/q_b/kv_a/kv_b_proj, o_proj, shared-expert and dense-MLP gate/up/down),
 * NEVER the tens of thousands of routed-expert tensors a large MoE model
 * carries (tools/repack_fp8_passthrough.py never stamps routed experts). A
 * container whose combined __metadata__["colibri.fmt"] entries exceed this
 * cap is not using the convention as designed -- CAP, not a switch to a hash
 * table. Precisely what this bounds: the colibri.fmt blob is json_parse'd in
 * FULL before the per-entry check below fires, so the parse allocation
 * itself is bounded by ST_MAX_HEADER (the shard-header size cap), not by
 * this constant -- what the cap bounds is the PERSISTENT fmt_name/fmt_val
 * strdup arrays on `shards` (and every later st_fmt_stamp linear scan over
 * them), which would otherwise grow with an adversarial map. Refuse loudly
 * rather than carry an absurd stamp map forward. */
#define ST_FMT_STAMP_MAX 4096

/* Parses one shard's __metadata__["colibri.fmt"] value (a safetensors metadata
 * value is always a plain string, so colibri.fmt's VALUE is itself JSON text --
 * a flat {tensor_name: format_name} object -- parsed a second time here) and
 * appends every entry to S->fmt_name/fmt_val. Absent __metadata__, or a
 * __metadata__ without a colibri.fmt key, is NOT an error: that's simply an
 * unstamped container, and byte-arithmetic inference alone decides, exactly as
 * before this feature existed (see qt_verify_fmt_stamp in colibri.c). A
 * colibri.fmt key that IS present but doesn't parse into that shape is refused
 * loudly -- same "untrusted container" discipline qt_resolve_fmt applies
 * elsewhere: a stamp the engine cannot make sense of must not be silently
 * ignored (that would be indistinguishable from a real mismatch going
 * unnoticed).
 *
 * DISCOVERY-TIME ABORT SURFACE: every exit(1) below (malformed stamp value,
 * malformed entry, or the ST_FMT_STAMP_MAX cap) fires from inside
 * st_init_multi's shard-header-parse loop -- i.e. at CONTAINER DISCOVERY
 * time, while the engine is still building its tensor index, before it has
 * resolved a single tensor against the model's architecture or read one byte
 * of weight data. This is coarser-grained and EARLIER than
 * qt_resolve_fmt/qt_verify_fmt_stamp's own per-tensor refusals in colibri.c
 * (which fire much later, during weight load, once a specific tensor's
 * [O,I] shape and stamp are both known): a malformed stamp anywhere in any
 * shard aborts the ENTIRE model load immediately, before the user ever sees
 * which layer or tensor was implicated -- these messages name a shard FILE,
 * never a tensor, which is how to tell this abort surface apart from the
 * later per-tensor one at a glance. See docs/FORMATS.md's own section on
 * this. */
static void st_fmt_stamp_ingest(shards *S, jval *root, const char *shard_path) {
    jval *meta = json_get(root, "__metadata__");
    if (!meta || meta->t != J_OBJ) return;                /* no metadata object: unstamped, fine */
    jval *stamp = json_get(meta, "colibri.fmt");
    if (!stamp) return;                                    /* no stamp key: unstamped, fine */
    if (stamp->t != J_STR) {
        fprintf(stderr, "%s: __metadata__[\"colibri.fmt\"] is not a JSON string -- malformed stamp, refusing (untrusted container)\n",
                shard_path); exit(1); }
    char *arena2 = NULL;
    jval *inner = json_parse_checked(stamp->str);
    if (!inner || inner->t != J_OBJ) {
        fprintf(stderr, "%s: __metadata__[\"colibri.fmt\"] does not parse as a JSON object -- malformed stamp, refusing (untrusted container)\n",
                shard_path); exit(1); }
    for (int i = 0; i < inner->len; i++) {
        jval *v = inner->kids[i];
        if (v->t != J_STR) {
            fprintf(stderr, "%s: colibri.fmt entry '%s' is not a string -- malformed stamp, refusing (untrusted container)\n",
                    shard_path, inner->keys[i]); exit(1); }
        /* DUPLICATE CLAIMS (user-ratified design, register D8): at most one
         * DISTINCT format claim per tensor name, container-wide. An entry
         * that repeats an already-ingested claim verbatim is tolerated and
         * collapsed to one entry (idempotent -- a centralized-manifest
         * writer may legally stamp the same map into every shard, and a
         * shard may stamp tensors it does not itself contain; there is no
         * locality constraint). An entry that CONTRADICTS an earlier claim
         * refuses by name: a container that disagrees with itself about a
         * tensor's format is corrupted or hostile, and the previous
         * first-wins behavior made the outcome depend on shard enumeration
         * order (st_scan_dir is raw readdir order, not sorted) while
         * mis-diagnosing the real problem downstream as a stamp/inference
         * mismatch -- or hiding it entirely when the enumeration happened
         * to favor the agreeing claim. The refusal names the tensor and
         * both format names; the EARLIER claim's shard file is not named
         * (per-entry shard provenance isn't stored, and adding it just for
         * this message would be new plumbing -- only the current shard's
         * path is in scope here). Linear rescan per entry is O(n^2) worst
         * case, bounded by ST_FMT_STAMP_MAX at one-time startup. */
        int dup = -1;
        for (int k = 0; k < S->fmt_n; k++)
            if (!strcmp(S->fmt_name[k], inner->keys[i])) { dup = k; break; }
        if (dup >= 0) {
            if (!strcmp(S->fmt_val[dup], v->str)) continue;   /* agreeing duplicate: keep one */
            fprintf(stderr, "%s: __metadata__[\"colibri.fmt\"] stamps tensor '%s' as '%s', but an "
                    "earlier shard's map already stamped it '%s' -- conflicting format claims, "
                    "refusing (untrusted container)\n",
                    shard_path, inner->keys[i], v->str, S->fmt_val[dup]); exit(1); }
        if (S->fmt_n >= ST_FMT_STAMP_MAX) {
            fprintf(stderr, "%s: __metadata__[\"colibri.fmt\"] stamps more than %d tensor names across "
                    "this container's shards -- stamps are a resident-tensor convention (docs/FORMATS.md), "
                    "not a bulk migration path; a container stamping this many names is malformed, "
                    "refusing (untrusted container)\n",
                    shard_path, ST_FMT_STAMP_MAX); exit(1); }
        if (S->fmt_n == S->fmt_cap) {
            S->fmt_cap = S->fmt_cap ? S->fmt_cap * 2 : 16;
            char **nn = (char**)realloc(S->fmt_name, S->fmt_cap * sizeof(char*));
            if (!nn) { fprintf(stderr, "%s: OOM reallocating format-stamp table (fmt_name, %d entries)\n",
                        shard_path, S->fmt_cap); exit(1); }
            S->fmt_name = nn;
            char **nv = (char**)realloc(S->fmt_val, S->fmt_cap * sizeof(char*));
            if (!nv) { fprintf(stderr, "%s: OOM reallocating format-stamp table (fmt_val, %d entries)\n",
                        shard_path, S->fmt_cap); exit(1); }
            S->fmt_val = nv;
        }
        /* Neither strdup here is benign to drop: a NULL fmt_name would be dereferenced by
         * the duplicate-scan strcmp above on the very next entry, and a NULL fmt_val is
         * worse -- the process would exit 0 with no diagnostic, st_fmt_stamp() would then
         * return NULL for a tensor that IS stamped, and the container would silently read
         * as unstamped: qt_verify_fmt_stamp's TRUST-VERIFY-REFUSE check (colibri.c) is a
         * no-op on an unstamped tensor, so one failed allocation would quietly disable it
         * on an untrusted-container path. Refuse instead. */
        char *sn = strdup(inner->keys[i]), *sv = strdup(v->str);
        if (!sn || !sv) { fprintf(stderr, "%s: OOM duplicating __metadata__[\"colibri.fmt\"] stamp "
                    "for tensor '%s' -- refusing (a dropped stamp would silently disable format "
                    "verification for this tensor)\n", shard_path, inner->keys[i]); exit(1); }
        S->fmt_name[S->fmt_n] = sn;
        S->fmt_val[S->fmt_n]  = sv;
        S->fmt_n++;
    }
    json_free(inner);
    free(arena2);  /* always NULL (json_parse never populates it -- see j_dup). */
}

/* Stamped format NAME for `name`, or NULL if this tensor carries no stamp
 * (either because no shard stamped it, or the container predates this
 * feature). Linear scan: S->fmt_n is a subset of S->n (only stamped tensors),
 * small in practice -- see the shards struct comment.
 *
 * SCOPE: .qs-BACKED TENSORS ONLY. This function itself will happily look up
 * ANY name that got stamped -- st_fmt_stamp_ingest doesn't know or check
 * whether a stamped name belongs to a quantized (.qs-backed) tensor -- but
 * qt_from_disk (colibri.c) only ever CALLS this inside its `st_has(name+
 * ".qs")` branch, i.e. only for a tensor that already carries a quantized
 * scale sidecar. A colibri.fmt entry naming a raw f32/bf16 weight, a norm, a
 * router, or embed/lm_head is stored here like any other entry but never
 * looked up: it is silently ignored BY DESIGN, not an oversight -- the
 * convention exists to disambiguate a byte-count collision among quantized
 * formats, and only a .qs-backed tensor can have one. See docs/FORMATS.md's
 * "Scope: .qs-backed tensors only". */
static const char *st_fmt_stamp(shards *S, const char *name) {
    for (int i = 0; i < S->fmt_n; i++)
        if (!strcmp(S->fmt_name[i], name)) return S->fmt_val[i];
    return NULL;
}

/* model.safetensors.index.json, consulted only when two indexed shards carry the
 * same tensor name. Checkpoints distributed with an "overlay" shard (a community
 * abliteration, say) keep the superseded tensors physically in the base shards
 * and let the index say which copy is authoritative; HF transformers honours
 * the index, and refusing the directory outright (#1479) made such checkpoints
 * unloadable without rewriting their shards. Loaded lazily: a clean container
 * never pays the parse (the Qwen3.8 index alone is 17 MB), and a container
 * without an index keeps the old refusal, because then nothing says which
 * bytes a name should resolve to. */
typedef struct { jval *root, *map; char *arena; int *h; int hcap; int tried; } st_index;

static const char *st_basename(const char *p) {
    const char *b = strrchr(p, '/');
#ifdef _WIN32
    const char *b2 = strrchr(p, '\\');
    if (b2 && (!b || b2 > b)) b = b2;
#endif
    return b ? b + 1 : p;
}
static void st_index_load(st_index *ix, const char *dir) {
    if (ix->tried) return;
    ix->tried = 1;
    char path[1200];
    int written = snprintf(path, sizeof(path), "%s/model.safetensors.index.json", dir);
    if (written < 0 || (size_t)written >= sizeof(path)) return;
    FILE *f = fopen(path, "rb");
    if (!f) return;
    fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET);
    if (size <= 0 || size > (256l << 20)) { fclose(f); return; }
    char *text = malloc((size_t)size + 1);
    if (!text || fread(text, 1, (size_t)size, f) != (size_t)size) { free(text); fclose(f); return; }
    text[size] = 0; fclose(f);
    ix->root = memchr(text, 0, (size_t)size) ? NULL : json_parse_checked(text);
    free(text);
    jval *map = ix->root ? json_get(ix->root, "weight_map") : NULL;
    if (!map || map->t != J_OBJ) { json_free(ix->root); ix->root = NULL; free(ix->arena); ix->arena = NULL; return; }
    ix->map = map;
    ix->hcap = 1; while (ix->hcap < map->len * 2) ix->hcap <<= 1;
    ix->h = malloc((size_t)ix->hcap * sizeof(int));
    if (!ix->h) coli_load_fail(COLI_LOAD_FAIL_NOMEM, "OOM indexing model.safetensors.index.json");
    for (int i = 0; i < ix->hcap; i++) ix->h[i] = -1;
    for (int i = 0; i < map->len; i++) {
        uint64_t hh = st_hash(map->keys[i]) & (ix->hcap - 1);
        while (ix->h[hh] >= 0) hh = (hh + 1) & (ix->hcap - 1);
        ix->h[hh] = i;
    }
}
/* the shard basename the index declares for `name`, or NULL (no index, or name absent) */
static const char *st_index_shard(st_index *ix, const char *name) {
    if (!ix->map) return NULL;
    uint64_t hh = st_hash(name) & (ix->hcap - 1);
    while (ix->h[hh] >= 0) {
        int i = ix->h[hh];
        if (!strcmp(ix->map->keys[i], name))
            return ix->map->kids[i]->t == J_STR ? ix->map->kids[i]->str : NULL;
        hh = (hh + 1) & (ix->hcap - 1);
    }
    return NULL;
}
static void st_index_free(st_index *ix) {
    json_free(ix->root); free(ix->arena); free(ix->h);
    memset(ix, 0, sizeof(*ix));
}

/* Scan one directory for *.safetensors shards, appending to files[] (dedup by

 * basename, so a list of directories acts as a SEARCH PATH: the same shard
 * present on two drives is taken from the first-listed one only). *added
 * returns how many shards this dir contributed. */
static void st_scan_dir(const char *dir, char files[][1024], int *nf, int *added) {
    DIR *d = opendir(dir); struct dirent *e;
    if (!d) coli_load_fail(coli_load_fail_kind_from_errno(errno), "%s: %s", dir, strerror(errno));
    int base_n = *nf;
    while ((e = readdir(d))) {
        const char *dot = strrchr(e->d_name, '.');
        if (dot && !strcmp(dot, ".safetensors")) {  /* model.safetensors o model-0000N-of-... */
            int dup = 0;
            for (int i = 0; i < *nf; i++) {
                const char *b = strrchr(files[i], '/');
#ifdef _WIN32
                const char *b2 = strrchr(files[i], '\\'); if (b2 && (!b || b2 > b)) b = b2;
#endif
                b = b ? b + 1 : files[i];
                if (!strcmp(b, e->d_name)) { dup = 1; break; }  /* already taken from a higher-priority drive */
            }
            if (dup) continue;
            if (*nf >= ST_MAX_SHARDS) { fprintf(stderr, "too many shards (>%d): raise ST_MAX_SHARDS\n", ST_MAX_SHARDS); exit(1); }
            snprintf(files[(*nf)++], 1024, "%s/%s", dir, e->d_name);
        }
    }
    closedir(d);
    if (added) *added = *nf - base_n;
}

/* Index shards from snap_dir, optionally SPLIT across extra drives listed in
 * extra_dirs (';' or ',' separated). Each shard lives on exactly ONE drive
 * (no duplication β€” unlike the dual-SSD mirror); a demand pread hits whichever
 * drive holds that shard, so concurrent expert loads parallelise across drives
 * and combined capacity is used. Scales to N drives. Metadata (config /
 * tokenizer / .coli_usage / .coli_kv) is read from snap_dir only. */
static void st_init_multi(shards *S, const char *snap_dir, const char *extra_dirs) {
    memset(S, 0, sizeof(*S));
    S->cap = 4096; S->t = calloc(S->cap, sizeof(st_tensor));
    /* Duplicate-tensor-name detector, indexed-shard scope.
     * st_find()/S->hidx (below) aren't built until every shard has been read, so a
     * duplicate name across two shards would otherwise go unnoticed until AFTER both are
     * already appended to S->t -- by then the collision has already happened silently:
     * st_find() would return whichever shard was appended first, while a container-wide
     * __metadata__["colibri.fmt"] stamp for that name (st_fmt_stamp_ingest above) would
     * still apply uniformly. A directory blending two containers under the same tensor
     * names (e.g. an FP8 copy and an int8 copy of the same o_proj) would then silently read
     * one shard's bytes labeled with a format that describes the other's -- no error, no
     * warning, plausible numbers. This is a DIFFERENT concern from st_fmt_stamp_ingest's
     * own duplicate check a few lines above: that dedups format CLAIMS keyed by name (and
     * already refuses on a disagreeing claim); this dedups the tensor DATA entries
     * themselves, the actual byte ranges a read resolves to. Kept as its own temporary
     * open-addressing table (same probe scheme st_hash()/S->hidx use below) rather than a
     * linear rescan, because a real container indexes on the order of 1e5 tensors and an
     * O(n^2) scan at that scale is not a rounding error. Grown in lockstep with S->cap
     * (below) and freed at the end of this function, just before S->hidx is built. */
    int dup_cap = 1; while (dup_cap < S->cap * 2) dup_cap <<= 1;
    int *dup_idx = (int*)malloc(dup_cap * sizeof(int));
    if (!dup_idx) { fprintf(stderr, "OOM allocating duplicate-tensor-name detector (%d entries)\n",
                dup_cap); exit(1); }
    for (int di = 0; di < dup_cap; di++) dup_idx[di] = -1;
    /* raccoglie ordinatamente i nomi dei file shard */
    static char files[ST_MAX_SHARDS][1024]; int nf = 0;
    int c0 = 0; st_scan_dir(snap_dir, files, &nf, &c0);
    int ndir = 1;
    if (extra_dirs && *extra_dirs) {
        char buf[4096]; snprintf(buf, sizeof(buf), "%s", extra_dirs);
        char *p = buf;
        while (p && *p) {
            char *sep = p; while (*sep && *sep != ';' && *sep != ',') sep++;
            int last = (*sep == 0); *sep = 0;
            while (*p == ' ') p++;
            size_t plen = strlen(p); while (plen > 0 && p[plen-1] == ' ') p[--plen] = 0;
            if (*p) {
                int cN = 0; st_scan_dir(p, files, &nf, &cN);
                fprintf(stderr, "[SPLIT] +%s -> %d shard(s)\n", p, cN);
                ndir++;
            }
            p = last ? NULL : sep + 1;
        }
        fprintf(stderr, "[SPLIT] model across %d dir(s): %d shard(s) total (primary %s -> %d shard(s)), no duplication\n",
                ndir, nf, snap_dir, c0);
    }
    for (int a = 0; a < nf; a++) for (int b = a+1; b < nf; b++)
        if (strcmp(files[a], files[b]) > 0) { char tmp[1024]; memcpy(tmp, files[a], 1024); memcpy(files[a], files[b], 1024); memcpy(files[b], tmp, 1024); }

    st_index ix; memset(&ix, 0, sizeof(ix));
    int n_overlay = 0;
    for (int fi = 0; fi < nf; fi++) {
        int fd = st_open_fd(S, files[fi]);
        int fidx = st_fidx(S, fd);
        if (fidx < 0) { fprintf(stderr, "%s: indexed shard fd is missing\n", files[fi]); exit(1); }
        int64_t fsz = S->sizes[fidx];
        uint64_t hlen;
        st_pread_full(fd, &hlen, 8, 0, "pread hlen");
        /* file malevolo/troncato: hlen deve stare nel file dopo gli 8 byte di
         * prefisso e sotto il tetto. Senza questo bound hlen+1 puo' andare in
         * overflow (malloc(0) e poi hdr[hlen]=0 fuori limiti) o forzare una
         * malloc gigante. */
        if (fsz < 8 || hlen > (uint64_t)(fsz - 8) || hlen > (uint64_t)ST_MAX_HEADER) {
            coli_load_fail(COLI_LOAD_FAIL_FORMAT, "%s: bad safetensors header length %llu (file %lld bytes)",
                           files[fi], (unsigned long long)hlen, (long long)fsz); }
        char *hdr = malloc(hlen + 1);
        if (!hdr) coli_load_fail(COLI_LOAD_FAIL_NOMEM, "malloc safetensors header: %s", strerror(errno));
        st_pread_full(fd, hdr, (int64_t)hlen, 8, "pread hdr");
        hdr[hlen] = 0;
        int64_t data_start = 8 + (int64_t)hlen;
        char *arena = NULL;
        /* A partial object must not index weights or authorize an overlay.
         * Header padding is JSON whitespace; embedded NUL is not padding. */
        jval *root = memchr(hdr, 0, (size_t)hlen) ? NULL : json_parse_checked(hdr);
        if (!root || root->t != J_OBJ) {
            coli_load_fail(COLI_LOAD_FAIL_FORMAT, "%s: safetensors header is not a JSON object", files[fi]); }
        st_fmt_stamp_ingest(S, root, files[fi]);
        for (int i = 0; i < root->len; i++) {
            const char *name = root->keys[i];
            if (!strcmp(name, "__metadata__")) continue;
            jval *m = root->kids[i];
            jval *dt = json_get(m, "dtype");
            jval *off = json_get(m, "data_offsets");
            jval *shp = json_get(m, "shape");
            /* un header crafted puo' omettere i campi o dare tipi sbagliati:
             * senza questi guard si dereferenzia NULL (json_get) o si legge
             * off->kids[0/1] oltre i limiti dell'array. */
            if (!dt || dt->t != J_STR || !off || off->t != J_ARR || off->len < 2 ||
                !shp || shp->t != J_ARR || shp->len > ST_MAX_RANK) {
                fprintf(stderr, "%s: tensor '%s' has malformed dtype/data_offsets/shape\n",
                        files[fi], name); exit(1); }
            jval *off_a = off->kids[0], *off_b = off->kids[1];
            if (!off_a || !off_b || off_a->t != J_NUM || off_b->t != J_NUM ||
                !isfinite(off_a->num) || !isfinite(off_b->num) ||
                off_a->num < 0.0 || off_b->num < 0.0 ||
                off_a->num >= ldexp(1.0, 63) || off_b->num >= ldexp(1.0, 63) ||
                floor(off_a->num) != off_a->num || floor(off_b->num) != off_b->num) {
                fprintf(stderr, "%s: tensor '%s' has invalid data_offsets\n",
                        files[fi], name); exit(1); }
            int64_t a0 = (int64_t)off_a->num, b0 = (int64_t)off_b->num;
            /* offset dichiarati dal file: non-negativi, ordinati e dentro al
             * file. Altrimenti nbytes=b0-a0 diventa negativo -> malloc((size_t))
             * gigante e la memcpy in st_read_f32 sfora il buffer del chiamante;
             * oppure off punta fuori dal file. */
            if (b0 < a0 || b0 > INT64_MAX - data_start || data_start + b0 > fsz) {
                fprintf(stderr, "%s: tensor '%s' data_offsets [%lld,%lld] out of file bounds (%lld)\n",
                        files[fi], name, (long long)a0, (long long)b0, (long long)fsz); exit(1); }
            /* SEC: lo shape viene da un file non fidato (mirror). Senza il guard
             * di overflow, uno shape tipo [65535,65535,65535,...] fa avvolgere
             * numel a un valore piccolo/negativo che poi passerebbe il cross-check
             * numel*esz==nbytes in st_read_f32, riaprendo l'OOB. */
            int64_t numel = 1; int bad_shape = 0;
            for (int k = 0; k < shp->len; k++) {
                jval *dim = shp->kids[k];
                if (!dim || dim->t != J_NUM || !isfinite(dim->num) ||
                    dim->num < 0.0 || dim->num >= ldexp(1.0, 63) ||
                    floor(dim->num) != dim->num) { bad_shape = 1; break; }
                int64_t d = (int64_t)dim->num;
                if (d != 0 && numel > INT64_MAX / d) { bad_shape = 1; break; }
                numel *= d;
            }
            if (bad_shape) {
                fprintf(stderr, "%s: tensor '%s' shape overflows int64 β€” refusing (hostile or corrupt file)\n",
                        files[fi], name); exit(1); }
            /* `name` already indexed from an earlier shard? The checkpoint's own index
             * decides: the copy the index maps the name to is read, the other one is
             * ignored (an overlay shard superseding base-shard tensors, #1479). No index,
             * or an index that names neither shard, keeps the refusal: a directory
             * holding both would silently read one shard's bytes for this name while
             * any __metadata__["colibri.fmt"] stamp for it still applies uniformly. This
             * is intentionally NOT the same check as st_fmt_stamp_ingest's -- that one
             * only ever sees the small subset of names a stamping tool chose to
             * annotate; this one sees every tensor. */
            int replace_idx = -1, skip = 0;
            {
                uint64_t dh = st_hash(name) & (dup_cap - 1);
                while (dup_idx[dh] >= 0) {
                    if (!strcmp(S->t[dup_idx[dh]].name, name)) {
                        int prev = dup_idx[dh];
                        int prev_fidx = st_fidx(S, S->t[prev].fd);
                        const char *prev_path = prev_fidx >= 0 ? S->paths[prev_fidx] : "<unknown>";
                        st_index_load(&ix, snap_dir);
                        const char *auth = st_index_shard(&ix, name);
                        if (auth && !strcmp(auth, st_basename(files[fi]))) { replace_idx = prev; n_overlay++; }
                        else if (auth && !strcmp(auth, st_basename(prev_path))) { skip = 1; n_overlay++; }
                        else {
                            fprintf(stderr, "%s: tensor '%s' is also indexed from shard '%s' -- duplicate "
                                    "tensor name across indexed shards, refusing (a directory holding both "
                                    "would silently read one shard's bytes for this name while any "
                                    "__metadata__[\"colibri.fmt\"] stamp for it still applies uniformly).\n"
                                    "  %s\n",
                                    files[fi], name, prev_path,
                                    ix.map ? (auth ? "model.safetensors.index.json maps it to a third shard; the index and the files disagree"
                                                   : "model.safetensors.index.json does not list this tensor, so it cannot say which copy is authoritative")
                                           : "no model.safetensors.index.json here to say which copy is authoritative; the checkpoint's index would resolve this");
                            exit(1);
                        }
                        break;
                    }
                    dh = (dh + 1) & (dup_cap - 1);
                }
            }
            if (skip) continue;
            if (S->n == S->cap) {
                S->cap *= 2;
                st_tensor *nt = (st_tensor*)realloc(S->t, S->cap * sizeof(st_tensor));
                if (!nt) { fprintf(stderr, "OOM reallocating shard tensor array\n"); exit(1); }
                S->t = nt;
                /* keep the duplicate-name detector's load factor low as S->t grows;
                 * rehash into a table twice the new capacity. */
                int new_dup_cap = dup_cap * 2;
                int *ndup = (int*)malloc(new_dup_cap * sizeof(int));
                if (!ndup) { fprintf(stderr, "OOM growing duplicate-tensor-name detector (%d entries)\n",
                            new_dup_cap); exit(1); }
                for (int di = 0; di < new_dup_cap; di++) ndup[di] = -1;
                for (int di = 0; di < dup_cap; di++) {
                    if (dup_idx[di] < 0) continue;
                    uint64_t h = st_hash(S->t[dup_idx[di]].name) & (new_dup_cap - 1);
                    while (ndup[h] >= 0) h = (h + 1) & (new_dup_cap - 1);
                    ndup[h] = dup_idx[di];
                }
                free(dup_idx); dup_idx = ndup; dup_cap = new_dup_cap;
            }
            st_tensor *t;
            if (replace_idx >= 0) { t = &S->t[replace_idx]; free(t->name); }   /* overlay wins: same slot, new bytes */
            else t = &S->t[S->n++];
            memset(t, 0, sizeof(*t));
            t->name = strdup(name); t->fd = fd; t->off = data_start + a0;
            t->nbytes = b0 - a0; t->dtype = st_dtype_code(dt->str); t->numel = numel;
            t->rank = shp->len;
            /* Register this name in the duplicate detector now that t->name is
             * stable, so the next shard's scan (above) sees it. A replaced entry
             * is already registered at the same index. */
            if (replace_idx < 0) {
              uint64_t dh = st_hash(t->name) & (dup_cap - 1);
              while (dup_idx[dh] >= 0) dh = (dh + 1) & (dup_cap - 1);
              dup_idx[dh] = S->n - 1; }
            for (int k = 0; k < t->rank; k++) t->shape[k] = (int64_t)shp->kids[k]->num;
            /* cross-check the declared element count against the byte span for FLOAT
             * dtypes: st_read_f32 writes `numel` floats (BF16/F16 loop or F32 memcpy)
             * into a caller-sized buffer, so a header with numel != nbytes/esz is an
             * OOB write primitive. U8/I8 (raw quant bytes) are read by byte count, so
             * their numel is unused by the read path and legitimately may differ. */
            { int esz = st_dtype_esz(t->dtype);
              if (t->dtype != 3 &&
                  (numel > INT64_MAX / esz || t->nbytes != numel * (int64_t)esz)) {
                  fprintf(stderr, "%s: tensor '%s' numel %lld disagrees with byte span %lld (esz %d)\n",
                          files[fi], name, (long long)numel, (long long)t->nbytes, esz); exit(1); } }
        }
        json_free(root);
        free(arena);
        free(hdr);
    }
    if (n_overlay)
        fprintf(stderr, "[st] %d tensor name(s) present in two shards, resolved by model.safetensors.index.json "
                        "(the copy the index maps each name to is read, the other ignored)\n", n_overlay);
    st_index_free(&ix);
    free(dup_idx);  /* duplicate detector: one-time-startup scratch, superseded by S->hidx below */
    /* indice hash costruito a fine indicizzazione (gli indici restano validi dopo i realloc) */
    S->hcap = 1; while (S->hcap < S->n * 2) S->hcap <<= 1;
    S->hidx = malloc(S->hcap * sizeof(int));
    for (int i = 0; i < S->hcap; i++) S->hidx[i] = -1;
    for (int i = 0; i < S->n; i++) {
        uint64_t h = st_hash(S->t[i].name) & (S->hcap - 1);
        while (S->hidx[h] >= 0) h = (h + 1) & (S->hcap - 1);
        S->hidx[h] = i;
    }
}

/* backward-compatible single-directory entry point */
static void st_init(shards *S, const char *snap_dir) { st_init_multi(S, snap_dir, NULL); }

/* Long-lived embedding consumers (the Segment runtime is the first one) can
 * open and close several model ranges in one process.  The historical CLI
 * exits after one model and intentionally relied on process teardown; expose a
 * real cleanup path without changing that CLI lifecycle. */
static void st_destroy(shards *S) {
    if (!S) return;
    st_mirror_reset(S);
    for (int index = 0; index < S->nfd; index++) {
        if (S->fds[index] >= 0) close(S->fds[index]);
        if (S->dfds[index] >= 0 && S->dfds[index] != S->fds[index])
            close(S->dfds[index]);
        free(S->paths[index]);
    }
    for (int index = 0; index < S->n; index++) free(S->t[index].name);
    for (int index = 0; index < S->fmt_n; index++) {
        free(S->fmt_name[index]);
        free(S->fmt_val[index]);
    }
    free(S->fmt_name);
    free(S->fmt_val);
    free(S->hidx);
    free(S->t);
    memset(S, 0, sizeof(*S));
}

static st_tensor *st_find(shards *S, const char *name) {
    if (S->hidx) {
        uint64_t h = st_hash(name) & (S->hcap - 1);
        while (S->hidx[h] >= 0) {
            st_tensor *t = &S->t[S->hidx[h]];
            if (!strcmp(t->name, name)) return t;
            h = (h + 1) & (S->hcap - 1);
        }
        return NULL;
    }
    for (int i = 0; i < S->n; i++) if (!strcmp(S->t[i].name, name)) return &S->t[i];
    return NULL;
}
static int st_has(shards *S, const char *name) { return st_find(S, name) != NULL; }

/* A missing CORE tensor is almost never an engine bug: the converter writes the final
 * norm and lm_head into the LAST shards, so a transfer that stopped early indexes
 * nearly everything and then dies on the first tensor from the gap. Bare "missing
 * model.norm.weight" gave the user no way to tell that apart from a wrong --model or a
 * real bug (#586, #583). Report what was actually found, and let the filenames say
 * whether shards are absent: the HF layout declares the total in `-of-NNNNN`, the
 * converter layout at least reveals a truncated tail. */
static void st_die_missing(shards *S, const char *name) {
    fprintf(stderr, "missing %s\n\n", name);
    if (S->nfd == 0 || S->n == 0) {
        fprintf(stderr, "  No safetensors tensors were indexed at all. --model must point AT the\n"
                        "  container directory -- the one holding the *.safetensors shards.\n");
        exit(1);
    }
    int lo = -1, hi = -1, declared = 0, numbered = 0;
    for (int i = 0; i < S->nfd; i++) {
        const char *b = strrchr(S->paths[i], '/');
#ifdef _WIN32
        const char *b2 = strrchr(S->paths[i], '\\');
        if (b2 && (!b || b2 > b)) b = b2;
#endif
        b = b ? b + 1 : S->paths[i];
        int idx, tot;
        if (sscanf(b, "model-%d-of-%d", &idx, &tot) == 2) declared = tot;
        else if (sscanf(b, "out-%d", &idx) != 1) continue;   /* out-mtp-* etc: not numbered */
        numbered++;
        if (lo < 0 || idx < lo) lo = idx;
        if (idx > hi) hi = idx;
    }
    int complete = declared > 0 && numbered >= declared;
    fprintf(stderr, "  indexed %d tensors from %d shard file(s)\n", S->n, S->nfd);
    if (declared > 0 && numbered < declared)
        fprintf(stderr, "  shard filenames declare %d shards, but only %d are here -- %d MISSING\n",
                declared, numbered, declared - numbered);
    else if (declared > 0)
        fprintf(stderr, "  shard filenames declare %d shards and %d are here\n", declared, numbered);
    else if (numbered > 0)
        fprintf(stderr, "  shards numbered %05d..%05d, %d file(s)%s\n", lo, hi, numbered,
                numbered == hi - lo + 1 ? " (contiguous: a gap would be at the tail)" : " (GAPS in the numbering)");
    /* The checkpoint's own index says more than a file count can: which shard
     * should hold this name, whether that shard is here, and whether the name
     * exists in this checkpoint at all. The last case is the one a file count
     * gets exactly wrong: every shard present, and the tensor never existed,
     * because the engine running here is not this checkpoint's engine (a
     * directory copied without config.json, and the GLM default looking for
     * model.embed_tokens.weight in a Qwen3.8 that stores it under
     * model.language_model). */
    {
        char dir[1200]; snprintf(dir, sizeof dir, "%s", S->paths[0]);
        char *slash = strrchr(dir, '/');
#ifdef _WIN32
        char *bs = strrchr(dir, '\\'); if (bs && (!slash || bs > slash)) slash = bs;
#endif
        if (slash) *slash = 0; else snprintf(dir, sizeof dir, ".");
        st_index ix; memset(&ix, 0, sizeof ix);
        st_index_load(&ix, dir);
        if (ix.map) {
            const char *auth = st_index_shard(&ix, name);
            if (!auth) {
                fprintf(stderr,
                    "\n  model.safetensors.index.json does not list '%s': this checkpoint never had a\n"
                    "  tensor by that name. The engine running here expects one, so it is not this\n"
                    "  checkpoint's engine. coli picks the engine from config.json -- run\n"
                    "  `coli info --model <dir>` and check the family it names; without a config.json\n"
                    "  there is no family, and the files to copy next to the shards are config.json,\n"
                    "  tokenizer.json and model.safetensors.index.json from the model repo.\n", name);
            } else {
                int present = 0;
                for (int i = 0; i < S->nfd && !present; i++) present = !strcmp(st_basename(S->paths[i]), auth);
                if (!present)
                    fprintf(stderr,
                        "\n  model.safetensors.index.json maps '%s' to %s, which is not in this\n"
                        "  directory: that shard is MISSING. Fetch just that file:\n"
                        "      hf download <repo> %s --local-dir <model-dir>\n", name, auth, auth);
                else
                    fprintf(stderr,
                        "\n  model.safetensors.index.json maps '%s' to %s, which is here but whose\n"
                        "  header does not declare it: the file is not the one the index describes (an\n"
                        "  interrupted transfer that kept the name, or a shard from another revision of\n"
                        "  the repo). Re-download that one shard and compare its size with the repo's.\n", name, auth);
            }
            st_index_free(&ix);
            exit(1);
        }
    }
    /* Follow the evidence: telling someone who already has every declared shard to
     * re-download sends them round a loop that cannot help them. */
    if (complete) {
        fprintf(stderr,
            "\n  '%s' is a core tensor the engine cannot run without, and every shard the\n"
            "  filenames declare is present -- so this is NOT the usual truncated download.\n"
            "  Either the container was built without this tensor, or the engine is looking\n"
            "  for the wrong name. Please report it with this output.\n", name);
        exit(1);
    }
    /* The final norm and lm_head sit at the END of the model, so an interrupted transfer
     * loses them first -- worth saying, but only where it's true. */
    if (strstr(name, "model.norm") || strstr(name, "lm_head"))
        fprintf(stderr, "\n  '%s' is written into one of the LAST shards, so a container whose tail\n"
                        "  never arrived indexes almost everything and then fails exactly here.\n", name);
    else
        fprintf(stderr, "\n  '%s' is a core tensor: the engine cannot run without it.\n", name);
    fprintf(stderr,
        "  An incomplete transfer is far more likely than a corrupt engine. Re-running the\n"
        "  download resumes only the missing shards; the HF xet backend is known to stall\n"
        "  mid-transfer (#452), so disable it:\n"
        "      HF_HUB_DISABLE_XET=1 hf download <repo> --local-dir <model-dir>\n"
        "  If your shard count is already complete, that IS a bug worth reporting.\n");
    exit(1);
}

/* prefetch ASINCRONO: dice al kernel di iniziare a leggere le pagine del tensore in
 * background (readahead). Serve a sovrapporre l'I/O degli expert col calcolo: si
 * prefetcha tutto il set di expert di un layer, poi le pread sincrone trovano la cache
 * gia' calda. No-op se il tensore non esiste (es. il primo .qs prima della lettura). */
static void st_prefetch(shards *S, const char *name) {
    st_tensor *t = st_find(S, name);
    if (t) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_WILLNEED);
}

/* like st_prefetch, but on replica `rep`'s drive: the WILLNEED must warm the
 * page cache of the SAME fd the later demand pread will hit. */
static void st_prefetch_rep(shards *S, const char *name, int rep) {
    st_tensor *t = st_find(S, name);
    if (!t) return;
    int fd = st_fd_rep(S, t->fd, rep);
    if (fd < 0) fd = t->fd;
    posix_fadvise(fd, t->off, t->nbytes, POSIX_FADV_WILLNEED);
}

/* legge un tensore in un buffer float32 fornito dal chiamante (numel float).
 * drop=1 -> consiglia al kernel di scartare le pagine (per gli expert in streaming). */
static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) {
    st_tensor *t = st_find(S, name);
    if (!t) st_die_missing(S, name);   /* #1317: la riga nuda esisteva gia' spiegata */
    /* SEC: numel viene dallo shape, nbytes dagli offset β€” due campi indipendenti
     * del file. Se non concordano, la memcpy F32 (nbytes) o i loop BF16/F16
     * (numel elementi da un raw di soli nbytes) sforano il buffer del chiamante,
     * che e' dimensionato sul config, non sul file. Il chiamante che alloca su
     * st_numel resta coerente; questo blocca l'ingresso ostile a monte. */
    /* I tipi non-float si leggono con st_read_raw, non qui. Senza questo rifiuto
     * cadrebbero nel ramo `else` in fondo, che assume F16: un tensore F8_E4M3 o
     * I64 verrebbe letto come mezze-precisioni e produrrebbe numeri plausibili e
     * sbagliati, in silenzio. Con soli i dtype 0/1/2/3 il fallthrough era corretto;
     * dal momento in cui ne esistono altri, non lo e' piu'. */
    if (t->dtype >= 3) {
        fprintf(stderr, "%s: tensor '%s' is %s β€” not a float tensor; read it with st_read_raw\n",
                name, name, st_dtype_name(t->dtype)); exit(1); }
    int esz = st_dtype_esz(t->dtype);
    if (t->numel < 0 || t->numel > t->nbytes / esz || t->numel * (int64_t)esz != t->nbytes) {
        fprintf(stderr, "%s: tensor '%s' shape/bytes mismatch (numel %lld, %lld bytes, dtype %d) β€” refusing (hostile or corrupt file)\n",
                name, name, (long long)t->numel, (long long)t->nbytes, t->dtype); exit(1); }
    void *raw = malloc(t->nbytes);
    if (!raw) coli_load_fail(COLI_LOAD_FAIL_NOMEM, "malloc %lld bytes for tensor %s failed", (long long)t->nbytes, name);
    st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data");
    if (t->dtype == 2) {
        memcpy(out, raw, t->nbytes);
    } else if (t->dtype == 0) {
        bf16_to_f32_bulk((uint16_t *)raw, out, t->numel);
    } else {
        f16_to_f32_bulk((uint16_t *)raw, out, t->numel);
    }
    free(raw);
    if (drop) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_DONTNEED);
    return t->numel;
}

/* like st_read_f32 but refuses to write more than `cap` floats into `out`.
 * Callers that size `out` from CONFIG dims (qt_alloc's O*I, per-row scales) must
 * use this: a crafted header whose tensor holds more elements than the config
 * shape would otherwise overrun `out`. Callers that size `out` from st_numel are
 * self-consistent and may keep using st_read_f32. */
static int64_t st_read_f32_cap(shards *S, const char *name, float *out, int64_t cap, int drop) {
    st_tensor *t = st_find(S, name);
    if (!t) st_die_missing(S, name);   /* #1317: la riga nuda esisteva gia' spiegata */
    if (t->numel < 0 || t->numel > cap) {
        fprintf(stderr, "tensor %s: numel %lld exceeds destination capacity %lld\n",
                name, (long long)t->numel, (long long)cap); exit(1); }
    return st_read_f32(S, name, out, drop);
}

static int64_t st_numel(shards *S, const char *name) {
    st_tensor *t = st_find(S, name); return t ? t->numel : -1;
}
static int64_t st_nbytes(shards *S, const char *name) {
    st_tensor *t = st_find(S, name); return t ? t->nbytes : -1;
}

/* --- ue8m0_to_f32 / st_read_scale_f32: sidecar di scale a 1 byte ------------
 *
 * UE8M0 e' un esponente potenza-di-due senza segno e senza mantissa: il valore
 * e' 2^(v-127), e 0xff e' NaN. Un byte per scala invece di quattro.
 *
 * Serve perche' un checkpoint fp8 nativo (DeepSeek-V4, e in generale
 * quantization_config.scale_fmt == "ue8m0") scrive le scale di blocco cosi',
 * mentre i container ripacchettati da noi le scrivono in f32. La GEOMETRIA e'
 * identica -- stessa forma, stesso significato, stessa moltiplicazione -- cambia
 * solo la codifica del numero.
 *
 * Si espande a f32 UNA VOLTA al caricamento invece di decodificare nel kernel:
 * le scale sono ~1/16384 dei byte dei pesi (mezzo MB per gli 8,4 GB densi di
 * DeepSeek-V4), quindi il costo in memoria e' trascurabile e matmul_fp8 resta
 * UNA sola implementazione, senza un ramo dentro il ciclo caldo. E' la stessa
 * scelta gia' fatta in kimi_k3.c per le scale ue8m0 di MXFP4 (`mx4_scale`).
 *
 * NaN: 0xff decodifica a un NaN IEEE reale e lo si lascia propagare, coerente
 * con la politica gia' documentata per i pesi fp8 in quant.h -- la rete di
 * sicurezza sta a valle, sul sampler (test_logit_nan.c), non qui. */
static inline float ue8m0_to_f32(uint8_t v) {
    if (v == 0xff) { uint32_t n = 0x7fc00000u; float f; memcpy(&f, &n, 4); return f; }
    /* ldexpf e NON il trucco `(uint32_t)v << 23`: quel trucco e' esatto per
     * v in [1,254], ma a v==0 produce il pattern di bit 0x00000000, che in IEEE
     * 754 e' ZERO ESATTO e non 2^-127. Un blocco di pesi con quella scala
     * verrebbe azzerato invece che reso quasi-zero -- differenza piccola in
     * ampiezza, ma e' comunque un valore sbagliato, e 2^-127 e' rappresentabile
     * come subnormale. Costa solo al caricamento (una volta per scala), quindi
     * si paga la chiamata e si tiene la correttezza. */
    return ldexpf(1.0f, (int)v - 127);
}

/* Legge un sidecar di scale in `out` come f32, accettando SIA F32 SIA F8_E8M0.
 * Rifiuta qualunque altro dtype per nome. `cap` e' il numero massimo di float
 * che il chiamante ha allocato, come in st_read_f32_cap. */
static int64_t st_read_scale_f32(shards *S, const char *name, float *out, int64_t cap, int drop) {
    st_tensor *t = st_find(S, name);
    if (!t) st_die_missing(S, name);   /* #1317: la riga nuda esisteva gia' spiegata */
    if (t->numel < 0 || t->numel > cap) {
        fprintf(stderr, "scale %s: numel %lld exceeds destination capacity %lld\n",
                name, (long long)t->numel, (long long)cap); exit(1); }
    if (t->dtype == 2 || t->dtype == 0 || t->dtype == 1) return st_read_f32(S, name, out, drop);
    if (t->dtype != 5) {
        fprintf(stderr, "scale %s: dtype %s is neither F32 nor F8_E8M0\n",
                name, st_dtype_name(t->dtype)); exit(1); }
    /* stessa validazione byte-vs-numel dei percorsi float: 1 byte per scala */
    if (t->nbytes != t->numel) {
        fprintf(stderr, "scale %s: F8_E8M0 numel %lld disagrees with %lld bytes\n",
                name, (long long)t->numel, (long long)t->nbytes); exit(1); }
    uint8_t *raw = (uint8_t*)malloc((size_t)t->nbytes);
    if (!raw) coli_load_fail(COLI_LOAD_FAIL_NOMEM, "malloc %lld bytes for scale %s failed", (long long)t->nbytes, name);
    st_pread_full(t->fd, raw, t->nbytes, t->off, "pread ue8m0 scale");
    for (int64_t i = 0; i < t->numel; i++) out[i] = ue8m0_to_f32(raw[i]);
    free(raw);
    if (drop) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_DONTNEED);
    return t->numel;
}

/* legge i byte GREZZI di un tensore (nessuna conversione di dtype): per i pesi gia'
 * quantizzati int4/int8 del nostro container (dtype U8). drop=1 -> fadvise DONTNEED.
 *
 * CALLER CONTRACT: this reads `t->nbytes` -- a length declared by the file header --
 * into `out`, and has no bound of its own. st_init cannot supply one: it deliberately
 * skips its numel*esz==nbytes cross-check for dtype 3, because packed quant bytes
 * legitimately have numel != nbytes. So the caller MUST establish that its destination
 * is at least t->nbytes before calling. Today all callers do, by three routes:
 *   - colibri.c sizes the buffer from st_nbytes() itself, and qt_resolve_fmt validates
 *     both byte counts against [O,I];
 *   - kimi_k3.c makes the byte count the branch predicate (`if(t->nbytes==O*I ...)`),
 *     so identifying the format and validating it are the same act;
 *   - olmoe.c compares against a config-derived want_w and refuses by name.
 * A new caller with none of those wants st_read_raw_cap below. */
static void st_read_raw(shards *S, const char *name, void *out, int drop) {
    st_tensor *t = st_find(S, name);
    if (!t) st_die_missing(S, name);   /* #1317: la riga nuda esisteva gia' spiegata */
    st_pread_full(t->fd, out, t->nbytes, t->off, "pread raw");
    if (drop) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_DONTNEED);
}

/* st_read_raw with the bound made explicit: `cap` is the byte capacity of `out`, in the
 * same position and spirit as st_read_f32_cap's element cap. Refuses rather than writing
 * past the destination, so a caller that has an expected size need not invent its own
 * check -- and one that has none cannot silently do the wrong thing. */
static void st_read_raw_cap(shards *S, const char *name, void *out, int64_t cap, int drop) {
    st_tensor *t = st_find(S, name);
    if (!t) st_die_missing(S, name);   /* #1317: la riga nuda esisteva gia' spiegata */
    if (t->nbytes < 0 || t->nbytes > cap) {
        fprintf(stderr, "%s: tensor declares %lld bytes, destination holds %lld β€” refusing "
                "(untrusted container)\n", name, (long long)t->nbytes, (long long)cap); exit(1); }
    st_read_raw(S, name, out, drop);
}

/* Read an exact physical range from one indexed shard.  This is intentionally
 * separate from tensor slices: callers may use it only after validating a
 * checkpoint-specific packing invariant (for example, two adjacent tensors).
 * The shard membership, file bounds and destination capacity remain enforced
 * here so a malformed header cannot turn that optimization into an unchecked
 * pread. */
static void st_read_range_raw_cap(shards *S, int fd, int64_t off,
                                  int64_t nbytes, void *out, int64_t cap,
                                  int drop, const char *tag) {
    int fidx=S?st_fidx(S,fd):-1;
    if(fidx<0){fprintf(stderr,"physical range uses an unindexed shard fd\n");exit(1);}
    if(off<0||nbytes<0||cap<0||nbytes>cap||
       off>S->sizes[fidx]||nbytes>S->sizes[fidx]-off||
       (uint64_t)nbytes>SIZE_MAX){
        fprintf(stderr,"physical shard range [%lld,+%lld) exceeds file/destination bounds "
                       "(file %lld, cap %lld) β€” refusing (untrusted container)\n",
                (long long)off,(long long)nbytes,(long long)S->sizes[fidx],
                (long long)cap);exit(1);
    }
    if(nbytes>0&&!out){fprintf(stderr,"physical range has a NULL destination\n");exit(1);}
    if(nbytes>0)st_pread_full(fd,out,nbytes,off,tag&&*tag?tag:"pread physical range");
    if(drop&&nbytes>0)posix_fadvise(fd,off,nbytes,POSIX_FADV_DONTNEED);
}

/* ---- replica-aware range read (multi-SSD streaming) ---------------------
 * st_read_range_raw_cap serves the range from the primary copy and leaves the
 * bytes in the page cache on the way through. An engine that streams large
 * expert tensors wants two more things, and the mirror machinery above already
 * knows where to find them:
 *
 *   - the range served by replica `rep` (0 = primary, 1..nrep = mirrors), so
 *     independent drives can answer together instead of one behind another;
 *   - that replica's O_DIRECT twin, because a large transient read that is
 *     copied into the page cache and out again pays for the same bytes twice,
 *     and an engine that reads them once does not want them cached at all.
 *
 * O_DIRECT constrains buffer address, file offset and length to the logical
 * block size, and a safetensors range starts wherever the writer put it. The
 * obvious fix -- bounce the range through an aligned scratch buffer -- adds one
 * full copy of every expert byte to a workload where the disk is the point, so
 * this instead expects the caller to reserve `off % ST_DIRECT_ALIGN` bytes of
 * scratch BEHIND the destination and hand over the payload pointer. The window
 * starting at the enclosing block boundary is then already aligned: one
 * O_DIRECT pread carries the block-aligned bulk, one short buffered pread
 * carries the sub-block tail, and the payload lands exactly where the caller
 * asked for it with nothing to copy afterwards.
 *
 * ST_DIRECT_ALIGN is the slack a caller must add to each such buffer. The
 * alignment is verified rather than assumed, and a caller that did not reserve
 * the prefix -- or a platform with no O_DIRECT, or a replica without a twin --
 * silently gets the buffered path. `direct` is the caller's own switch: 0 forces
 * that path everywhere, which is what an engine exposes so the choice can be
 * A/B'd on the hardware in front of it rather than argued about. */
#define ST_DIRECT_ALIGN 4096

/* One aligned pread, EINTR retried. Returns 0 only on a full transfer: O_DIRECT
 * wants a block-aligned length, so a short read cannot be finished by simply
 * asking for the rest -- the caller falls back to the buffered path instead. */
static int st_pread_aligned_try(int fd, void *buf, int64_t n, int64_t off) {
    for (;;) {
        ssize_t r = pread(fd, buf, (size_t)n, (off_t)off);
        if (r == n) return 0;
        if (r < 0 && errno == EINTR) continue;
        return -1;
    }
}

static void st_read_range_rep(shards *S, int fd, int rep, int64_t off, int64_t nbytes,
                              void *out, int64_t cap, int drop, int direct,
                              const char *tag) {
    int fidx = S ? st_fidx(S, fd) : -1;
    if (fidx < 0) { fprintf(stderr, "replica range uses an unindexed shard fd\n"); exit(1); }
    if (off < 0 || nbytes < 0 || cap < 0 || nbytes > cap ||
        off > S->sizes[fidx] || nbytes > S->sizes[fidx] - off ||
        (uint64_t)nbytes > SIZE_MAX) {
        fprintf(stderr, "replica range [%lld,+%lld) exceeds file/destination bounds "
                        "(file %lld, cap %lld) β€” refusing (untrusted container)\n",
                (long long)off, (long long)nbytes, (long long)S->sizes[fidx], (long long)cap);
        exit(1);
    }
    if (nbytes == 0) return;
    if (!out) { fprintf(stderr, "replica range has a NULL destination\n"); exit(1); }
    const char *label = tag && *tag ? tag : "pread replica range";
    /* A partial mirror leaves this shard on the primary; so does a replica index
     * that no longer exists. Either way the read is valid, just not split. */
    int rfd = st_fd_rep(S, fd, rep);
    if (rfd < 0) { rfd = fd; rep = 0; }
    int64_t pad = off % ST_DIRECT_ALIGN;
    uint8_t *window = (uint8_t *)out - pad;      /* the caller's reserved scratch */
    int dfd = direct ? st_direct_fd_rep(S, fd, rep) : -1;
    if (dfd >= 0 && ((uintptr_t)window & (uintptr_t)(ST_DIRECT_ALIGN - 1)) == 0 &&
        pad + nbytes <= (int64_t)ST_PREAD_CHUNK) {
        int64_t want = pad + nbytes;              /* bytes to fill, window-relative */
        int64_t left = S->sizes[fidx] - (off - pad);
        int64_t bulk = want & ~(int64_t)(ST_DIRECT_ALIGN - 1);
        if (bulk > left) bulk = left & ~(int64_t)(ST_DIRECT_ALIGN - 1);
        /* A device that refuses O_DIRECT costs the call some speed and never its
         * correctness: the buffered path below reads exactly the same bytes. */
        if (bulk > 0 && st_pread_aligned_try(dfd, window, bulk, off - pad) == 0) {
            int64_t tail = want - bulk;
            if (tail > 0) st_pread_full(rfd, window + bulk, tail, off - pad + bulk, label);
            return;
        }
    }
    st_pread_full(rfd, out, nbytes, off, label);
    if (drop) posix_fadvise(rfd, off, nbytes, POSIX_FADV_DONTNEED);
}

/* Read-only view of one tensor's exact stored bytes.  Unlike st_read_raw this
 * performs no allocation or copy; unlike a naked mmap pointer it carries the
 * aligned OS view/handle required for cleanup.  Callers must still validate
 * dtype and shape for their own format before consuming `data`. */
typedef struct {
    compat_ro_map map;
    const void *data;
    int64_t nbytes;
} st_mapped_raw;

static int st_map_raw(shards *S, const char *name, st_mapped_raw *out) {
    st_tensor *t = st_find(S, name);
    if (!t) { fprintf(stderr, "missing tensor: %s\n", name); return -1; }
    if (!out || t->nbytes <= 0 || (uint64_t)t->nbytes > SIZE_MAX) {
        errno = EINVAL; return -1;
    }
    memset(out, 0, sizeof(*out));
    if (compat_map_readonly(t->fd, t->off, (size_t)t->nbytes,
                            &out->map, &out->data) != 0) return -1;
    out->nbytes = t->nbytes;
    return 0;
}

static void st_unmap_raw(st_mapped_raw *mapped) {
    if (!mapped) return;
    compat_unmap_readonly(&mapped->map);
    mapped->data = NULL;
    mapped->nbytes = 0;
}

/* ---- per-shard mapping cache (opt-in: COLI_MAP_EXPERTS=1) --------------
 * One mapping per FILE, kept for the process lifetime (shards stay open for
 * the whole run, so there is nothing shorter-lived to tie the mapping to).
 * A failed mapping is remembered per fd so it is not retried on every call;
 * callers must treat a NULL return as "use pread", not as an error. */
#define ST_MAX_MAPPED_FD 4096
static uint8_t *g_st_shard_base[ST_MAX_MAPPED_FD];
static int64_t g_st_shard_len[ST_MAX_MAPPED_FD];
static signed char g_st_shard_tried[ST_MAX_MAPPED_FD];

static int st_map_experts_enabled(void) {
    static int on = -1;
    if (on < 0) { const char *e = getenv("COLI_MAP_EXPERTS"); on = (e && atoi(e)) ? 1 : 0; }
    return on;
}

/* Reached from parallel regions: glm53.c's `#pragma omp parallel for` over the
 * batch calls expert_read -> st_map_shard_range -> here from every worker at
 * once, and qwen38_core.h does the same. Two threads racing the plain version
 * of this both saw g_st_shard_tried[fd] == 0, both mapped the file, and one
 * leaked its mapping while a third could read g_st_shard_len[fd] as 0 through
 * an already-published base -- st_map_shard_range then rejects every range for
 * that shard. Latent only because COLI_MAP_EXPERTS is opt-in; #1350 proposes
 * flipping that default.
 *
 * Lock-free rather than the mutex qwen38_core.h uses for its lazy HITS table,
 * because st.h is also compiled into c/tools/check_glm53_container.c, whose
 * documented build line is plain `gcc -O2 -std=gnu11 -Ic ... -lm` with neither
 * -pthread nor -fopenmp. */
static const uint8_t *st_shard_mapped(int fd) {
    if (fd < 0 || fd >= ST_MAX_MAPPED_FD || !st_map_experts_enabled()) return NULL;
    /* Acquire: a base published below carries g_st_shard_len[fd] with it. */
    const uint8_t *base = __atomic_load_n(&g_st_shard_base[fd], __ATOMIC_ACQUIRE);
    if (base) return base;
    /* Claim the fd exactly once. The losing thread does not block: NULL is the
     * documented "use pread" answer, so it takes the pread path for this one
     * call rather than parking an OpenMP worker behind an mmap. */
    signed char unclaimed = 0;
    if (!__atomic_compare_exchange_n(&g_st_shard_tried[fd], &unclaimed, (signed char)1,
                                     0, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE))
        return __atomic_load_n(&g_st_shard_base[fd], __ATOMIC_ACQUIRE);
    int64_t len = (int64_t)lseek(fd, 0, SEEK_END);
    if (len <= 0) return NULL;
    compat_ro_map map; const void *data;
    if (compat_map_readonly(fd, 0, (size_t)len, &map, &data) != 0) return NULL;
    /* The compat_ro_map itself is intentionally not tracked for unmap: shard
     * mappings live exactly as long as the fd they wrap, i.e. the process. */
    /* Length first, then release-store the base: whoever sees the base through
     * the acquire load above necessarily sees the length too. */
    g_st_shard_len[fd] = len;
    __atomic_store_n(&g_st_shard_base[fd], (uint8_t *)data, __ATOMIC_RELEASE);
    return (const uint8_t *)data;
}

/* Serves [off, off+nbytes) of file `fd` directly out of its persistent
 * mapping -- no allocation, no copy. Returns NULL if mapping is disabled,
 * unavailable for this fd, or the range doesn't fit; the caller's existing
 * pread path is the correct fallback in every NULL case. */
static const void *st_map_shard_range(int fd, int64_t off, int64_t nbytes) {
    if (off < 0 || nbytes <= 0) return NULL;
    const uint8_t *base = st_shard_mapped(fd);
    if (!base) return NULL;
    if (off > g_st_shard_len[fd] - nbytes) return NULL;
    return base + off;
}

/* Read an exact byte slice from a tensor into a caller-owned buffer.  This is
 * the raw counterpart of st_read_slice_f32: byte_off/nbytes are relative to
 * the tensor (not the shard), and cap is the actual byte capacity of `out`.
 * It is deliberately usable for one-byte F8 rows and eight-byte I64 rows;
 * there is no dtype conversion or element-size assumption here. */
static void st_read_slice_raw_cap(shards *S, const char *name, int64_t byte_off,
                                  int64_t nbytes, void *out, int64_t cap, int drop) {
    st_tensor *t = st_find(S, name);
    if (!t) st_die_missing(S, name);   /* #1317: la riga nuda esisteva gia' spiegata */
    if (byte_off < 0 || nbytes < 0) {
        fprintf(stderr, "slice %s [%lld,+%lld) has negative offset/length\n",
                name, (long long)byte_off, (long long)nbytes); exit(1);
    }
    if (cap < 0 || nbytes > cap) {
        fprintf(stderr, "slice %s requests %lld bytes, destination holds %lld β€” refusing\n",
                name, (long long)nbytes, (long long)cap); exit(1);
    }
    if (t->nbytes < 0 || byte_off > t->nbytes || nbytes > t->nbytes - byte_off) {
        fprintf(stderr, "slice %s [%lld,+%lld) out of tensor byte bounds (%lld)\n",
                name, (long long)byte_off, (long long)nbytes, (long long)t->nbytes); exit(1);
    }
    if ((uint64_t)nbytes > SIZE_MAX) {
        fprintf(stderr, "slice %s requests %lld bytes, exceeds host size_t\n",
                name, (long long)nbytes); exit(1);
    }
    if (nbytes > 0 && !out) {
        fprintf(stderr, "slice %s has a NULL destination for %lld bytes\n",
                name, (long long)nbytes); exit(1);
    }
    if (t->off < 0 || byte_off > INT64_MAX - t->off) {
        fprintf(stderr, "slice %s file offset overflows int64\n", name); exit(1);
    }
    int64_t boff = t->off + byte_off;
    st_pread_full(t->fd, out, nbytes, boff, "pread raw slice");
    /* A zero length to posix_fadvise means "through EOF" on POSIX, which is
     * broader than this request.  Do not issue it for an empty slice. */
    if (drop && nbytes > 0) posix_fadvise(t->fd, boff, nbytes, POSIX_FADV_DONTNEED);
}

/* legge una FETTA di un tensore: n_elems a partire dall'elemento elem_off.
 * Serve per gli expert fusi di GLM (un tensore = blocco [E, ...]): si legge il
 * solo expert richiesto via pread del sotto-range, niente lettura dell'intero blocco. */
static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int64_t n_elems, float *out, int drop) {
    st_tensor *t = st_find(S, name);
    if (!t) st_die_missing(S, name);   /* #1317: la riga nuda esisteva gia' spiegata */
    if (t->dtype >= 3) {   /* stesso motivo di st_read_f32 sopra */
        fprintf(stderr, "slice %s: tensor is %s β€” not a float tensor\n",
                name, st_dtype_name(t->dtype)); exit(1); }
    int esz = st_dtype_esz(t->dtype);
    if (elem_off < 0 || n_elems < 0 || elem_off > t->numel || n_elems > t->numel - elem_off) {   /* keep the slice inside the tensor; subtraction avoids overflow (#1) */
        fprintf(stderr, "slice %s [%lld,+%lld) out of tensor bounds (numel %lld)\n",
                name, (long long)elem_off, (long long)n_elems, (long long)t->numel); exit(1); }
    if (elem_off > INT64_MAX / esz || n_elems > INT64_MAX / esz ||
        t->off < 0 || elem_off * esz > INT64_MAX - t->off ||
        (uint64_t)(n_elems * esz) > SIZE_MAX || (n_elems > 0 && !out)) {
        fprintf(stderr, "slice %s byte arithmetic/destination is invalid β€” refusing\n", name); exit(1); }
    int64_t boff = t->off + elem_off * esz, nb = n_elems * esz;
    void *raw = nb ? malloc((size_t)nb) : NULL;
    if (nb && !raw) coli_load_fail(COLI_LOAD_FAIL_NOMEM, "malloc %lld bytes for slice %s failed", (long long)nb, name);
    if (nb) st_pread_full(t->fd, raw, nb, boff, "pread slice");   /* dev #331: chunked + EINTR + honest short-read */
    if (nb) {
        if (t->dtype == 2) memcpy(out, raw, (size_t)nb);
        else if (t->dtype == 0) bf16_to_f32_bulk((uint16_t *)raw, out, n_elems);
        else f16_to_f32_bulk((uint16_t *)raw, out, n_elems);
    }
    free(raw);
    if (drop && nb) posix_fadvise(t->fd, boff, nb, POSIX_FADV_DONTNEED);
}

#endif