Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
24.6 KB
Raw Blame
// include/strata/core/expert_nibble.hpp - STRATA_EXPERT_NIBBLE=1 (opt-in, default off): the gate/up experts of
// IQ2_S / IQ3_XXS / IQ3_S layers kept as 4-bit codes in their VRAM slots, for Pascal GP100 (sm_60).
//
// **WHY.**  On a GP100 the decode window's gate/up expert kernels are not held by memory bandwidth: they use a small
// part of it and wait on each block's serial chain (the weights, the codebook lookups, the emulated dp4a, the
// reduction).  Re-encoding each 32-weight item of these three formats as sixteen bytes of 4-bit codes (one aligned
// 16-byte load) and keeping the integer scale and the fp16 d as they are makes the slot 1.25-1.68x the GGUF bytes and
// the decode of four weights 1-4 integer instructions instead of two codebook lookups.
//
// **WHY IT IS LOSSLESS.**  Their grids have only 3 (IQ2_S) or 8 (IQ3_XXS, IQ3_S) magnitudes, so each int8 weight the
// production `Split<TG>::load` decodes is one of at most 16 signed values.  At every start the transcoder decodes every
// item of every converted slot with the production load, encodes it, decodes the written code again and compares; and
// the GGUF blocks are rebuilt from every slot and compared with the blob byte for byte (`check_gguf`).  The decode
// kernels run the production integer chain and float tail in the production order: every gate / up value is the
// production kernel's, bit for bit.  The prompt path's MMQ reads the GGUF blocks rebuilt from the slot (`gather_gguf`,
// in place of the copy it does today): the same bytes, so the same bits.
//
// **OFF IS TODAY.**  Unset or 0: no layer is converted, `slot_bytes(l)` is `expert_layout().blob_bytes(l)`, nothing is
// transcoded, sealed or hashed, and none of the new branches is entered.  Only a CUDA build with
// STRATA_EXPERIMENTAL_SM60 has the kernels (STRATA_EXPERT_NIBBLE_BUILT); every other build refuses the switch at start.
#pragma once

#include "strata/kernels/cpu/expert_layout.hpp"

#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>
#include <vector>

#if defined(STRATA_EXPERIMENTAL_SM60) && !defined(__HIPCC__) && !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && \
    !defined(STRATA_GPU_METAL)
#define STRATA_EXPERT_NIBBLE_BUILT 1
#else
#define STRATA_EXPERT_NIBBLE_BUILT 0
#endif

#if defined(__CUDACC__) || defined(__HIPCC__)
#define STRATA_NIB_HD __host__ __device__
#else
#define STRATA_NIB_HD
#endif

namespace strata::core::nibble {

// ------------------------------------------------------------------------------------------------ the slot layout
//
// A converted layer's expert slot, in order (the host arena and the pack keep the GGUF blocks):
//
//   codes   kRows x kItems x 16 B   row r, item k at (r * kItems + k) * 16; rows 0..kNFf-1 gate, kNFf..kRows-1 up
//   scales  kRows x kItems x 1 B    row r, item k at kScOff + r * kItems + k (the production scale step's input)
//   d       kRows x kSuper x 2 B    row r, superblock s at kDOff + (r * kSuper + s) * 2: the block's fp16 d, bit for bit
//   ------  kGuBytes = 1,766,400 B (6900 x 256: the down part starts 256-aligned)
//   down    the GGUF down rows, copied verbatim from the raw blob's [down_off, bytes)
//   ------  slot_bytes = round_up(kGuBytes + down bytes, kSlotAlign)
//
// One item = 32 weights of a row, one 16-byte code (four 32-bit words, see `encode_item`).  Only the expert shape
// n_embd 2560, n_ff 640 is converted: every other layer keeps its GGUF blob and its kernels.

inline constexpr int64_t kNEmbd = 2560;                 ///< the gate/up rows' width (the activation)
inline constexpr int64_t kNFf = 640;                    ///< gate rows = up rows = n_ff
inline constexpr int64_t kRows = 2 * kNFf;              ///< 1280: gate, then up
inline constexpr int64_t kItems = kNEmbd / 32;          ///< 80 items of 32 weights per row
inline constexpr int64_t kSuper = kNEmbd / 256;         ///< 10 superblocks (one fp16 d each) per row
inline constexpr uint64_t kItemBytes = 16;              ///< one item's code: four words, one 16-byte load

inline constexpr uint64_t kCodesOff = 0;
inline constexpr uint64_t kUpCodesOff = (uint64_t) kNFf * kItems * kItemBytes;          ///<   819,200: row kNFf
inline constexpr uint64_t kScOff = (uint64_t) kRows * kItems * kItemBytes;              ///< 1,638,400
inline constexpr uint64_t kDOff = kScOff + (uint64_t) kRows * kItems;                   ///< 1,740,800
inline constexpr uint64_t kGuBytes = kDOff + (uint64_t) kRows * kSuper * 2;             ///< 1,766,400
inline constexpr uint64_t kDownOff = kGuBytes;                                          ///< the down rows
inline constexpr uint64_t kSlotAlign = 256;                                             ///< every slot's size

static_assert(kGuBytes == 1766400, "the slot layout above");
static_assert(kGuBytes % kSlotAlign == 0, "the down part starts 256-aligned");
static_assert(kScOff % 16 == 0 && kDOff % 16 == 0, "each region starts 16-aligned");

/// Byte offset of row `row` (0..kRows-1; up rows are kNFf + r) item `k`'s code inside the slot.
STRATA_NIB_HD constexpr uint64_t code_off(int64_t row, int64_t k) {
    return kCodesOff + ((uint64_t) row * kItems + (uint64_t) k) * kItemBytes;
}
/// Byte offset of row `row` item `k`'s scale byte.
STRATA_NIB_HD constexpr uint64_t scale_off(int64_t row, int64_t k) { return kScOff + (uint64_t) row * kItems + (uint64_t) k; }
/// Byte offset of row `row` superblock `s`'s fp16 d (s = k / 8 for item k).
STRATA_NIB_HD constexpr uint64_t d_off(int64_t row, int64_t s) { return kDOff + ((uint64_t) row * kSuper + (uint64_t) s) * 2; }
/// A converted slot's size for a raw blob whose down rows are `down_bytes` long.
STRATA_NIB_HD constexpr uint64_t slot_bytes_for(uint64_t down_bytes) {
    return (kGuBytes + down_bytes + kSlotAlign - 1) / kSlotAlign * kSlotAlign;
}

// ------------------------------------------------------------------------------------------------ the formats

inline constexpr int kIQ3_XXS = 18;
inline constexpr int kIQ3_S = 21;
inline constexpr int kIQ2_S = 22;

/// Whether `gu_type` is converted (the three formats whose weights take at most 16 values).
STRATA_NIB_HD constexpr bool fmt_built(int gu_type) { return gu_type == kIQ2_S || gu_type == kIQ3_XXS || gu_type == kIQ3_S; }
/// The GGUF gate/up row bytes of `gu_type` at kNEmbd (a converted layer's raw blob must agree; 0 = not a format here).
STRATA_NIB_HD constexpr uint64_t raw_gu_row(int gu_type) {
    return gu_type == kIQ2_S ? 820 : gu_type == kIQ3_XXS ? 980 : gu_type == kIQ3_S ? 1100 : 0;
}
inline const char* fmt_name(int gu_type) {
    switch (gu_type) {
        case kIQ3_XXS: return "IQ3_XXS";
        case kIQ3_S: return "IQ3_S";
        case kIQ2_S: return "IQ2_S";
        default: return "?";
    }
}

// ------------------------------------------------------------------------------------------------ the item code
//
// `g[32]` below is one item as the production `Split<TG>::load` leaves it: g[4 * j + b] is byte b of W.g[j]
// (j = 0..7, the dp4a order against the q8_1 block's qs words).  Word m (0..3) of the 16-byte code holds W.g[2m]
// and W.g[2m + 1]:
//
//   IQ2_S    bits 4b..4b+3 = c(g[8m + b]), bits 16+4b.. = c(g[8m + 4 + b]), b = 0..3
//            c = magnitude index of {8, 25, 43} | sign << 2: the PRMT selector into {8,25,43,0} {-8,-25,-43,0}
//   IQ3_XXS  same nibble positions; c = index of {4, 12, 20, 28, 36, 44, 52, 62} | sign << 3
//   IQ3_S    byte b = u(g[8m + b]) | u(g[8m + 4 + b]) << 4, u = (v + 15) / 2 (v odd, -15..15); v = 2u - 15
//
// The scale byte keeps the production integer scale step's input:
//   IQ2_S    the block's scales byte for the item: ls0 | ls1 << 4 (W.ls0, W.ls1)
//   IQ3_XXS  aux32 >> 28 (0..15, W.ls)
//   IQ3_S    1 + 2 * nibble (1..31, W.ls)

/// Encodes one item; false when a weight is outside the format's code set (the conversion then refuses to start).
inline bool encode_item(int gu_type, const int8_t g[32], uint32_t q[4]) {
    bool ok = true;
    auto code = [&](int8_t v) -> uint32_t {
        const int a = v < 0 ? -(int) v : (int) v;
        if (gu_type == kIQ3_S) {
            if ((v & 1) == 0 || v < -15 || v > 15) { ok = false; return 0; }
            return (uint32_t) (((int) v + 15) / 2);
        }
        if (gu_type == kIQ3_XXS) {
            static constexpr int mag[8] = { 4, 12, 20, 28, 36, 44, 52, 62 };
            for (uint32_t i = 0; i < 8; ++i)
                if (a == mag[i]) return i | (v < 0 ? 8u : 0u);
            ok = false;
            return 0;
        }
        const uint32_t m = a == 8 ? 0u : a == 25 ? 1u : a == 43 ? 2u : 3u;   // IQ2_S
        if (m == 3u) { ok = false; return 0; }
        return m | (v < 0 ? 4u : 0u);
    };
    for (int m = 0; m < 4; ++m) {
        uint32_t w = 0;
        for (int b = 0; b < 4; ++b) {
            const uint32_t c0 = code(g[8 * m + b]), c1 = code(g[8 * m + 4 + b]);
            if (gu_type == kIQ3_S) w |= (c0 | c1 << 4) << (8 * b);
            else w |= c0 << (4 * b) | c1 << (16 + 4 * b);
        }
        q[m] = w;
    }
    return ok;
}

/// Decodes one item's code back to the production W.g bytes (the host reference the parity program compares with;
/// the device decoders give the same bytes).  Codes outside the set decode as the PRMT does: IQ2_S index 3 -> 0.
inline void decode_item(int gu_type, const uint32_t q[4], int8_t g[32]) {
    static constexpr int8_t iq2[8] = { 8, 25, 43, 0, -8, -25, -43, 0 };
    static constexpr int8_t iq3[8] = { 4, 12, 20, 28, 36, 44, 52, 62 };
    for (int m = 0; m < 4; ++m)
        for (int b = 0; b < 4; ++b) {
            uint32_t c0, c1;
            if (gu_type == kIQ3_S) {
                c0 = q[m] >> (8 * b) & 15u;
                c1 = q[m] >> (8 * b + 4) & 15u;
            } else {
                c0 = q[m] >> (4 * b) & 15u;
                c1 = q[m] >> (16 + 4 * b) & 15u;
            }
            auto val = [&](uint32_t c) -> int8_t {
                if (gu_type == kIQ3_S) return (int8_t) (2 * (int) c - 15);
                if (gu_type == kIQ3_XXS) return (int8_t) ((c & 8u) ? -iq3[c & 7u] : iq3[c & 7u]);
                return iq2[c & 7u];
            };
            g[8 * m + b] = val(c0);
            g[8 * m + 4 + b] = val(c1);
        }
}

// ------------------------------------------------------------------------------------------------ the slot hash
//
// STRATA_EXPERT_NIBBLE_RECHECK's fingerprint of a slot: over the slot's first `hashed` bytes (kGuBytes plus the down
// rows; the alignment tail is not written and not hashed) as little-endian 32-bit words w_i, the last one
// zero-extended,  H = sum_i (uint64) (w_i ^ i) * kHashMul  mod 2^64.  A sum, so the device's order of adding does not
// change it; the index term makes a moved word count.

inline constexpr uint64_t kHashMul = 0x9E3779B97F4A7C15ull;

/// The host reference of the slot hash (the device hash gives the same value for the same bytes).
inline uint64_t hash_host(const void* p, uint64_t bytes) {
    const uint8_t* s = (const uint8_t*) p;
    uint64_t h = 0;
    const uint64_t n = (bytes + 3) / 4;
    for (uint64_t i = 0; i < n; ++i) {
        uint32_t w = 0;
        const uint64_t at = 4 * i, take = bytes - at < 4 ? bytes - at : 4;
        std::memcpy(&w, s + at, (size_t) take);   // little-endian hosts only
        h += (uint64_t) (w ^ (uint32_t) i) * kHashMul;
    }
    return h;
}

// ------------------------------------------------------------------------------------------------ the switches

struct Config {
    bool on = false;                       ///< STRATA_EXPERT_NIBBLE=1
    bool recheck = false;                  ///< STRATA_EXPERT_NIBBLE_RECHECK=1 (debug: every slot's hash per request)
    std::vector<int> fmts{ kIQ2_S, kIQ3_XXS, kIQ3_S };   ///< STRATA_EXPERT_NIBBLE_FMTS (default 22,18,21)
    std::vector<int64_t> skip;             ///< STRATA_EXPERT_NIBBLE_SKIP: layers kept in their GGUF blocks
    std::string error;                     ///< non-empty: a malformed value - the start is refused
};

namespace detail {
/// "a,b,c" -> integers; false on anything else (empty = an empty list).
inline bool parse_list(const char* s, std::vector<int64_t>& out) {
    out.clear();
    while (s != nullptr && *s != '\0') {
        char* end = nullptr;
        const long long v = std::strtoll(s, &end, 10);
        if (end == s) return false;
        out.push_back((int64_t) v);
        s = end;
        if (*s == ',') ++s;
        else if (*s != '\0') return false;
    }
    return true;
}
}  // namespace detail

/// The switches, read once (generate.cpp reads them before anything is allocated).
inline const Config& config() {
    static const Config c = [] {
        Config r;
        const char* e = std::getenv("STRATA_EXPERT_NIBBLE");
        if (e != nullptr && e[0] != '\0' && std::strcmp(e, "0") != 0) {
            if (std::strcmp(e, "1") == 0) r.on = true;
            else r.error = std::string("STRATA_EXPERT_NIBBLE=") + e + ": expected 0 or 1";
        }
        const char* rc = std::getenv("STRATA_EXPERT_NIBBLE_RECHECK");
        r.recheck = rc != nullptr && std::strcmp(rc, "1") == 0;
        std::vector<int64_t> v;
        if (const char* f = std::getenv("STRATA_EXPERT_NIBBLE_FMTS"); f != nullptr && f[0] != '\0') {
            if (!detail::parse_list(f, v)) {
                if (r.error.empty()) r.error = std::string("STRATA_EXPERT_NIBBLE_FMTS=") + f + ": expected ggml types, e.g. 22,18,21";
            } else {
                r.fmts.clear();
                for (const int64_t t : v) {
                    if (!fmt_built((int) t) && r.error.empty())
                        r.error = "STRATA_EXPERT_NIBBLE_FMTS: type " + std::to_string(t) +
                                  " is not converted (22 IQ2_S, 18 IQ3_XXS, 21 IQ3_S)";
                    r.fmts.push_back((int) t);
                }
            }
        }
        if (const char* k = std::getenv("STRATA_EXPERT_NIBBLE_SKIP"); k != nullptr && k[0] != '\0') {
            if (!detail::parse_list(k, r.skip) && r.error.empty())
                r.error = std::string("STRATA_EXPERT_NIBBLE_SKIP=") + k + ": expected layer numbers, e.g. 40,44";
        }
        return r;
    }();
    return c;
}

/// With the switch on, the first environment setting it refuses, as a message (empty = none): the paths that would
/// read a converted slot as GGUF blocks, write GGUF bytes into one, or run other kernels for the converted layers
/// than the ones the 4-bit kernels are bit for bit.  The command-line half (--peer-device, remote expert caches,
/// --kv-grow, --vram-elastic), the cc 6.0 check, full residency and no loan are generate.cpp's checks; the prompt
/// path's own (MMQ on every converted layer) is Prefill::nibble_prompt_check.
inline std::string env_conflicts() {
    auto set = [](const char* n) -> const char* {
        const char* v = std::getenv(n);
        return v != nullptr && v[0] != '\0' ? v : nullptr;
    };
    const char* v = set("STRATA_MMQ_DIRECT");
    if (v != nullptr && std::atoi(v) != 0)
        return "STRATA_MMQ_DIRECT (the prompt MMQ would read a converted slot in place as GGUF blocks)";
    v = set("STRATA_PREFILL_MMQ");
    if (v != nullptr && std::atoi(v) == 0) return "STRATA_PREFILL_MMQ=0 (the FP16 prompt path reads GGUF blocks)";
    v = set("STRATA_PF_FUSED");
    if (v != nullptr && v[0] == '1') return "STRATA_PF_FUSED=1 (the fused prompt kernels read GGUF blocks)";
    v = set("STRATA_FS_SLOTS");
    if (v != nullptr && std::atoi(v) > 0) return "STRATA_FS_SLOTS (Foresight swaps GGUF blobs into slots)";
    v = set("STRATA_EXPERT_V2");
    if (v != nullptr && v[0] == '1') return "STRATA_EXPERT_V2=1 (the S26 kernels, not the CUDA layout, on IQ3_S)";
    v = set("STRATA_KV_GROW");
    if (v != nullptr && v[0] != '0') return "STRATA_KV_GROW (the elastic K/V moves slots at their GGUF size)";
    v = set("STRATA_EXP_MODE");
    if (v != nullptr && std::atoi(v) != 0 && std::atoi(v) != 8)
        return "STRATA_EXP_MODE other than 0 or 8 (another layout's gate/up kernels)";
    return {};
}

// ------------------------------------------------------------------------------------------------ the VRAM table

/// One layer's slots.  `on` false: the GGUF blob, `slot_bytes == raw_bytes`, the production kernels.
struct LayerVram {
    bool on = false;
    int gu_type = -1, d_type = -1;
    uint64_t down_off = 0;     ///< in the slot: kDownOff when on, else the raw blob's down_off
    uint64_t down_bytes = 0;   ///< the down rows (raw_bytes - raw_down_off)
    uint64_t hashed = 0;       ///< bytes the slot hash covers: kGuBytes + down_bytes when on, else 0
    uint64_t slot_bytes = 0;   ///< what the expert cache allocates per slot (`open_sized`, every estimate)
    // the raw blob as the arena and the pack hold it (expert_layout().fmt[l]); the transcoder reads these
    uint64_t raw_bytes = 0, raw_gu_row = 0, raw_up_off = 0, raw_down_off = 0;
};

/// One layer's entry from its GGUF format and the switches (pure; `plan_layers` calls it per layer).
inline LayerVram plan_layer(int64_t l, const strata::kernels::cpu::NativeFmt& f, const Config& c) {
    LayerVram v;
    v.gu_type = f.gu_type;
    v.d_type = f.d_type;
    v.raw_bytes = f.bytes;
    v.raw_gu_row = f.gu_row;
    v.raw_up_off = f.up_off;
    v.raw_down_off = f.down_off;
    v.down_bytes = f.bytes - f.down_off;
    v.down_off = f.down_off;
    v.slot_bytes = f.bytes;
    bool listed = false, skipped = false;
    for (const int t : c.fmts) listed = listed || t == f.gu_type;
    for (const int64_t s : c.skip) skipped = skipped || s == l;
    // [gate | up | down] exactly, no NVFP4 tail, the down rows a whole number of 16-byte words (the gathers copy them so)
    const bool shape = f.n_embd == kNEmbd && f.n_ff == kNFf && f.gu_row == raw_gu_row(f.gu_type) &&
                       f.up_off == (uint64_t) kNFf * f.gu_row && f.down_off == 2 * f.up_off && f.bytes > f.down_off &&
                       f.tail_off == 0 && (f.bytes - f.down_off) % 16 == 0;
    if (c.on && c.error.empty() && listed && !skipped && fmt_built(f.gu_type) && shape) {
        v.on = true;
        v.down_off = kDownOff;
        v.hashed = kGuBytes + v.down_bytes;
        v.slot_bytes = slot_bytes_for(v.down_bytes);
    }
    return v;
}

/// Every layer's entry for this pack (all off for a Q2_0 pack or with the switch off).  `err`: a SKIP layer out of
/// range, or the switch on with a Q2_0 pack.
inline std::vector<LayerVram> plan_layers(const strata::kernels::cpu::ExpertLayout& lay, std::string& err) {
    const Config& c = config();
    std::vector<LayerVram> out((size_t) (lay.n_layers > 0 ? lay.n_layers : 0));
    for (int64_t l = 0; l < (int64_t) out.size(); ++l) {
        if (lay.native && (size_t) l < lay.fmt.size()) out[(size_t) l] = plan_layer(l, lay.fmt[(size_t) l], c);
        else out[(size_t) l].slot_bytes = out[(size_t) l].raw_bytes = lay.blob_bytes(l);
    }
    if (c.on && !lay.native) err = "STRATA_EXPERT_NIBBLE needs a native pack (native_experts.txt)";
    for (const int64_t s : c.skip)
        if (c.on && (s < 0 || s >= lay.n_layers) && err.empty())
            err = "STRATA_EXPERT_NIBBLE_SKIP: layer " + std::to_string(s) + " is not in the model";
    return out;
}

namespace detail {
inline std::vector<LayerVram>& table() {
    static std::vector<LayerVram> t;
    return t;
}
}  // namespace detail

/// generate.cpp only: the table, once, after `expert_layout_load` and BEFORE the first slot estimate (placement
/// search, can_start, the cache's auto size, every `open_sized`).  Not thread-safe: set before any other thread.
inline void set_layers(std::vector<LayerVram> v) { detail::table() = std::move(v); }

/// Layer `l`'s entry (an off entry when the table is not set or `l` is outside it).  Read-only after `set_layers`.
inline const LayerVram& layer(int64_t l) {
    static const LayerVram off;
    const auto& t = detail::table();
    return l >= 0 && (size_t) l < t.size() ? t[(size_t) l] : off;
}

/// Whether any layer is converted (the switch on and the table set).
inline bool any_on() {
    for (const LayerVram& v : detail::table())
        if (v.on) return true;
    return false;
}

/// Bytes of one of layer `l`'s VRAM slots: the converted size for a converted layer, else the layout's blob bytes
/// (exactly `expert_layout().blob_bytes(l)`: with the switch off every caller sizes what it sized before).
inline uint64_t slot_bytes(int64_t l) {
    const auto& t = detail::table();
    if (l >= 0 && (size_t) l < t.size()) return t[(size_t) l].slot_bytes;
    return strata::kernels::cpu::expert_layout().blob_bytes(l);
}

// ------------------------------------------------------------------------------------------------ fatal paths

/// A converted layer reached a path that would compute or copy it as GGUF blocks: say where and stop.  Never a
/// silent wrong answer.
[[noreturn]] inline void fatal(int64_t l, const char* where, const char* why) {
    std::fprintf(stderr, "strata: STRATA_EXPERT_NIBBLE: layer %lld (%s, 4-bit slots) reached %s: %s - stopping\n",
                 (long long) l, fmt_name(layer(l).gu_type), where != nullptr ? where : "?", why != nullptr ? why : "");
    std::fflush(stderr);
    std::abort();
}

/// Before each raw slot copy that bypasses the ExpertCache API (the adaptive tier's copies, the elastic K/V's slot
/// moves): fatal when `l` is a converted layer.  The startup refusals keep these paths off; this is the backstop
/// (with `ExpertCache::seal()` for the API's own fills).
inline void guard_raw(int64_t l, const char* where) {
    if (layer(l).on) fatal(l, where, "a raw slot copy at the GGUF size");
}

// ------------------------------------------------------------------------------------------------ the decode kernels
//
// Two skeletons in the shape of native_gu_multi_kernel (16 lanes a row, a group's entries GRP_NC at a time), both bit
// for bit the production gate/up: S stages the entries' q8_1 blocks in shared memory and decodes each item where it
// is used (IQ3_S: v = 2u - 15 restored per byte, then the production apply); P loads the lane's codes, scales and d
// before staging the qs words as 16-byte rows (IQ3_S: sum = 2 dot(u, x) - 15 sum(x), the same int32).  Per format the
// faster one on 4x GP100 at T = 1..6 (native_expert_bench --nibble): IQ2_S P, IQ3_XXS S, IQ3_S S.

inline constexpr int kSkelS = 0, kSkelP = 1;
STRATA_NIB_HD constexpr int gu_skeleton(int gu_type) { return gu_type == kIQ2_S ? kSkelP : kSkelS; }

// ------------------------------------------------------------------------------------------------ iq_kernels.cu
//
// Device addresses are `unsigned long long` (as native_expert_grouped's grp_ptr), streams `void*`; each returns the
// CUDA error code (0 = success) so the caller can name the layer.  A build without STRATA_EXPERT_NIBBLE_BUILT
// returns cudaErrorInvalidValue (1) from each.

/// The reverse codebooks (grid magnitudes -> GGUF grid index) `check_gguf` and `gather_gguf` use, built on the CURRENT
/// device (once per device; call it at load, before any capture).  Non-zero also when a grid entry is not unique.
int prepare(void* stream);

/// Converts `n` raw blobs of layer `L` (device `staging`, `L.raw_bytes` apart, as the host arena holds them) into
/// the converted slots at `slot_ptrs[0..n)` (a device array): codes, scales, d.  The down rows are not touched (the
/// caller copies them to `slot + kDownOff`).  Every item is decoded with the production `Split<TG>::load`, encoded,
/// decoded from the written code again and compared with W.g[0..7], the scale and d; each item that differs or holds
/// a weight outside the code set adds 1 to `*bad` (device).  L.on must be true.
int transcode(const LayerVram& L, const void* staging, int64_t n, const unsigned long long* slot_ptrs,
              unsigned long long* bad, void* stream);

/// The GGUF gate/up blocks rebuilt from each of the `n` (<= 64) converted slots at `slot_ptrs` and its down rows,
/// compared with the raw blobs in `staging` (`L.raw_bytes` apart) byte for byte: each 32-bit word that differs adds 1
/// to `*bad`.  `prepare` first.
int check_gguf(const LayerVram& L, const void* staging, int64_t n, const unsigned long long* slot_ptrs,
               unsigned long long* bad, void* stream);

/// The prompt path's gather of a converted layer's experts (prefill.cpp, MMQ): for q in [first, n) (n <= 16), the
/// GGUF [gate | up] blocks rebuilt from slot `slots[q]` (host array of device addresses) into gu_dst + q * gu_stride -
/// the bytes the raw blob holds, byte for byte - and its down rows copied to d_dst + q * d_stride; `zero_bytes` zeroed
/// after expert n - 1's gate/up and down (MMQ's tail).  Every address, stride and size 16-byte aligned.  `prepare`
/// on this device first.
int gather_gguf(const LayerVram& L, const uint8_t* const* slots, int first, int n, void* gu_dst, uint64_t gu_stride,
                void* d_dst, uint64_t d_stride, uint64_t zero_bytes, void* stream);

/// The slot hash (`hash_host`'s value) of the `L.hashed` bytes at each `slot_ptrs[i]`, into device `out[0..n)`.
int hash_slots(const LayerVram& L, const unsigned long long* slot_ptrs, int64_t n, unsigned long long* out,
               void* stream);

}  // namespace strata::core::nibble