Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
76.1 KB
Raw Blame
/* Optional AMD XDNA2 helper binding -- the C side, which never touches XRT.
 *
 * The split mirrors backend_loader.c: the host resolves an optional native DLL
 * at runtime and links none of its dependencies, so an ordinary Colibri build
 * has no XRT headers, no XRT import library and no XRT DLL import. Everything
 * that needs XRT lives in coli_xdna.dll, which this file loads and may refuse.
 *
 * Refusal is the normal case. A machine with no NPU, no XRT or no helper is
 * ordinary, and every rejection path here ends the same way: the lane is
 * unavailable and the caller keeps doing exactly what it does today.
 *
 * W2-N7-I1 implements the binding only. There is deliberately no device open,
 * no artifact, no weight preparation and no dispatch here yet; the entry-point
 * set is the smallest one that makes the boundary versioned and testable, so
 * later slices extend it rather than inheriting speculative signatures.
 */

#include "backend_xdna.h"

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>

/* posix_memalign / compat_aligned_free. compat.h already maps these onto
 * _aligned_malloc/_aligned_free on Windows and carries the warning that the
 * two MUST be paired -- reusing it avoids a second aligned-allocation
 * convention in the same tree. */
#include "compat.h"

#ifdef _WIN32
#include <windows.h>
#endif

/* Entry points required of a helper at ABI generation 2. Still minimal: exactly
 * the calls the first blocking operation needs, and nothing speculative.
 *
 * Everything crossing this boundary is a fixed-width integer, a size_t, a C
 * string, or a raw pointer with an explicit byte count. No engine type, no XRT
 * type and no C++ type appears in any signature, so the helper cannot be handed
 * -- and cannot acquire -- any knowledge of tensors, experts, layers, routing or
 * models. It executes what it is given. */
typedef unsigned int (*coli_xdna_fn_abi_version)(void);
/* device + runtime + artifact program, for one (M,K,N). */
typedef int          (*coli_xdna_fn_open)(const char *xclbin, const char *insts,
                                          uint32_t m, uint32_t k, uint32_t n);
/* wrap caller-owned prepared BF16 through the qualified userptr path. */
typedef int          (*coli_xdna_fn_wrap_weight)(void *bf16, uint64_t bytes);
/* blocking: submit, wait, copy the artifact-M output out. */
typedef int          (*coli_xdna_fn_execute)(const void *a_bf16, uint64_t a_bytes,
                                             void *c_f32, uint64_t c_bytes);
typedef int          (*coli_xdna_fn_release_weight)(void);
typedef void         (*coli_xdna_fn_shutdown)(void);
typedef const char  *(*coli_xdna_fn_last_error)(void);

static struct {
    ColiXdnaBinding state;
    int             load_attempts;
    char            path[1024];      /* test override; empty = default lookup */
#ifdef _WIN32
    HMODULE         dll;
#endif
    coli_xdna_fn_abi_version    abi_version;
    coli_xdna_fn_open           open;
    coli_xdna_fn_wrap_weight    wrap_weight;
    coli_xdna_fn_execute        execute;
    coli_xdna_fn_release_weight release_weight;
    coli_xdna_fn_shutdown       shutdown;
    coli_xdna_fn_last_error     last_error;
} g_xdna;

/* Drop every callable pointer. Always runs BEFORE the module is released, so a
 * rejected or shut-down helper can never leave a pointer into an unmapped
 * module behind -- the all-or-nothing half of the binding contract. */
static void coli_xdna_clear_entry_points(void){
    g_xdna.abi_version    = NULL;
    g_xdna.open           = NULL;
    g_xdna.wrap_weight    = NULL;
    g_xdna.execute        = NULL;
    g_xdna.release_weight = NULL;
    g_xdna.shutdown       = NULL;
    g_xdna.last_error     = NULL;
}

/* Every helper call goes through one of these. They are the only places a
 * helper pointer is dereferenced, and each refuses when unbound rather than
 * trusting a caller to have checked. */
static int coli_xdna_helper_open_call(const char *xb, const char *ib,
                                      unsigned m, unsigned k, unsigned n){
    if(!g_xdna.open) return -1;
    return g_xdna.open(xb, ib, (uint32_t)m, (uint32_t)k, (uint32_t)n);
}
static int coli_xdna_helper_wrap_call(void *p, uint64_t bytes){
    if(!g_xdna.wrap_weight) return -1;
    return g_xdna.wrap_weight(p, bytes);
}
static int coli_xdna_helper_execute_call(const void *a, uint64_t ab, void *c, uint64_t cb){
    if(!g_xdna.execute) return -1;
    return g_xdna.execute(a, ab, c, cb);
}
static int coli_xdna_helper_release_weight_call(void){
    if(!g_xdna.release_weight) return -1;
    return g_xdna.release_weight();
}
static void coli_xdna_helper_shutdown_call(void){
    if(g_xdna.shutdown) g_xdna.shutdown();
}

const char *coli_xdna_binding_text(ColiXdnaBinding state){
    switch(state){
        case COLI_XDNA_UNPROBED:          return "UNPROBED";
        case COLI_XDNA_AVAILABLE:         return "AVAILABLE";
        case COLI_XDNA_ABSENT:            return "ABSENT";
        case COLI_XDNA_LOAD_FAILED:       return "LOAD_FAILED";
        case COLI_XDNA_ABI_INCOMPATIBLE:  return "ABI_INCOMPATIBLE";
        case COLI_XDNA_SYMBOL_INCOMPLETE: return "SYMBOL_INCOMPLETE";
    }
    return "UNKNOWN";
}

#ifdef _WIN32

/* Where the helper is looked for when no test override is set: beside the
 * running executable, by absolute path. Deliberately NOT a search: no PATH, no
 * current directory, no application-directory fallback. W1 established that an
 * already-loaded module with the same basename wins over a later absolute-path
 * load, so a search order here would be both unsafe and unpredictable. A helper
 * that is not where we look is simply ABSENT. */
static int coli_xdna_default_helper_path(char *out, size_t cap){
    char exe[512];
    DWORD n = GetModuleFileNameA(NULL, exe, (DWORD)sizeof exe);
    if(n == 0 || n >= sizeof exe) return 0;
    char *slash = strrchr(exe, '\\');
    char *fwd   = strrchr(exe, '/');
    if(fwd && (!slash || fwd > slash)) slash = fwd;
    if(!slash) return 0;
    *slash = '\0';
    int written = snprintf(out, cap, "%s\\%s", exe, COLI_XDNA_HELPER_DLL);
    return written > 0 && (size_t)written < cap;
}

static void coli_xdna_probe(void){
    char resolved[512];
    const char *path = g_xdna.path[0] ? g_xdna.path : NULL;

    if(!path){
        if(!coli_xdna_default_helper_path(resolved, sizeof resolved)){
            g_xdna.state = COLI_XDNA_ABSENT;
            return;
        }
        path = resolved;
    }

    /* Absence is not a load failure, and the difference is worth keeping: one
     * means "this build simply has no optional lane here", the other means "a
     * helper is installed but something about it is wrong". */
    if(GetFileAttributesA(path) == INVALID_FILE_ATTRIBUTES){
        g_xdna.state = COLI_XDNA_ABSENT;
        return;
    }

    g_xdna.load_attempts++;
    /* LOAD_WITH_ALTERED_SEARCH_PATH lets the helper's own directory satisfy the
     * helper's dependencies (it will eventually need the XRT runtime beside it)
     * without widening the search for the helper itself, which we located by
     * absolute path above. */
    HMODULE dll = LoadLibraryExA(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
    if(!dll){
        g_xdna.state = COLI_XDNA_LOAD_FAILED;
        return;
    }

    /* Resolve into locals: nothing becomes callable until the whole set has
     * validated and the ABI generation matches. */
#if defined(__GNUC__)
    /* GetProcAddress returns FARPROC; casting to the exact exported signature
     * is the standard LoadLibrary idiom, same as backend_loader.c's RESOLVE. */
    #pragma GCC diagnostic push
    #pragma GCC diagnostic ignored "-Wcast-function-type"
#endif
    coli_xdna_fn_abi_version abi_version =
        (coli_xdna_fn_abi_version)GetProcAddress(dll, "coli_xdna_helper_abi_version");
    coli_xdna_fn_open open_fn =
        (coli_xdna_fn_open)GetProcAddress(dll, "coli_xdna_helper_open");
    coli_xdna_fn_wrap_weight wrap_fn =
        (coli_xdna_fn_wrap_weight)GetProcAddress(dll, "coli_xdna_helper_wrap_weight");
    coli_xdna_fn_execute exec_fn =
        (coli_xdna_fn_execute)GetProcAddress(dll, "coli_xdna_helper_execute");
    coli_xdna_fn_release_weight relw_fn =
        (coli_xdna_fn_release_weight)GetProcAddress(dll, "coli_xdna_helper_release_weight");
    coli_xdna_fn_shutdown shutdown =
        (coli_xdna_fn_shutdown)GetProcAddress(dll, "coli_xdna_helper_shutdown");
    coli_xdna_fn_last_error lasterr_fn =
        (coli_xdna_fn_last_error)GetProcAddress(dll, "coli_xdna_helper_last_error");
#if defined(__GNUC__)
    #pragma GCC diagnostic pop
#endif

    /* The version handshake comes first: a helper from a different generation
     * may well export a symbol of the same name with a different meaning, so
     * the missing-symbol check below is only meaningful once generations agree.
     * A helper that cannot even report its generation is incompatible, not
     * incomplete -- there is no version to compare. */
    if(!abi_version || abi_version() != COLI_XDNA_ABI_VERSION){
        FreeLibrary(dll);
        g_xdna.state = COLI_XDNA_ABI_INCOMPATIBLE;
        return;
    }
    /* All-or-nothing: one missing entry point rejects the whole helper. A
     * partially bound helper would report availability and then fail somewhere
     * further in, where the failure is far harder to attribute. */
    if(!open_fn || !wrap_fn || !exec_fn || !relw_fn || !shutdown || !lasterr_fn){
        FreeLibrary(dll);
        g_xdna.state = COLI_XDNA_SYMBOL_INCOMPLETE;
        return;
    }

    g_xdna.dll            = dll;
    g_xdna.abi_version    = abi_version;
    g_xdna.open           = open_fn;
    g_xdna.wrap_weight    = wrap_fn;
    g_xdna.execute        = exec_fn;
    g_xdna.release_weight = relw_fn;
    g_xdna.shutdown       = shutdown;
    g_xdna.last_error     = lasterr_fn;
    g_xdna.state          = COLI_XDNA_AVAILABLE;
}

void coli_xdna_shutdown(void){
    /* Release lane state (which itself calls the helper) BEFORE the module goes
     * away: no helper-owned object may outlive the module that created it, and
     * no host pointer into it may survive the FreeLibrary. */
    coli_xdna_execution_shutdown();
    coli_xdna_clear_entry_points();
    if(g_xdna.dll){
        FreeLibrary(g_xdna.dll);
        g_xdna.dll = NULL;      /* released exactly once; repeat calls are no-ops */
    }
    /* The verdict is deliberately NOT reset to UNPROBED: shutting the lane down
     * is not a reason to re-probe it later in the same process. */
}

#else  /* !_WIN32 */

/* The qualified XDNA lane is Windows/XDNA2-specific (see the N6 architecture
 * freeze). Elsewhere the loader compiles and answers ABSENT, so the rest of the
 * engine needs no platform branches and the contract stays uniform. */
static void coli_xdna_probe(void){
    g_xdna.state = COLI_XDNA_ABSENT;
}

void coli_xdna_shutdown(void){
    coli_xdna_execution_shutdown();
    coli_xdna_clear_entry_points();
}

#endif /* _WIN32 */

ColiXdnaBinding coli_xdna_binding(void){
    /* Sticky: probe at most once per path. Everything after the first call is a
     * branch, which is what keeps a machine with no helper from paying a module
     * load on every candidate operation. */
    if(g_xdna.state == COLI_XDNA_UNPROBED) coli_xdna_probe();
    return g_xdna.state;
}

void coli_xdna_test_set_helper_path(const char *path){
    coli_xdna_shutdown();
    g_xdna.state         = COLI_XDNA_UNPROBED;
    g_xdna.load_attempts = 0;
    if(path && *path){
        snprintf(g_xdna.path, sizeof g_xdna.path, "%s", path);
    } else {
        g_xdna.path[0] = '\0';
    }
}

ColiXdnaBinding coli_xdna_test_state(void){ return g_xdna.state; }
int coli_xdna_test_load_attempts(void){ return g_xdna.load_attempts; }
int coli_xdna_test_entry_points_bound(void){
    return g_xdna.abi_version != NULL && g_xdna.shutdown != NULL;
}

/* ======================================================================
 * SHA-256 (FIPS 180-4)
 *
 * Self-contained on purpose. BCrypt is available to the Windows toolchain, but
 * using it would add -lbcrypt to the host link line for a lane the host does
 * not even link yet, and would fork this file's otherwise portable registry
 * logic along a platform boundary it has no other reason to have. ~90 lines of
 * standard SHA-256, covered by the NIST vectors in the registry test, buys
 * portability and zero new dependencies.
 * ====================================================================== */

static uint32_t coli_sha_ror(uint32_t x, int n){ return (x >> n) | (x << (32 - n)); }

static void coli_sha256_block(uint32_t h[8], const unsigned char *p){
    static const uint32_t K[64] = {
        0x428a2f98u,0x71374491u,0xb5c0fbcfu,0xe9b5dba5u,0x3956c25bu,0x59f111f1u,
        0x923f82a4u,0xab1c5ed5u,0xd807aa98u,0x12835b01u,0x243185beu,0x550c7dc3u,
        0x72be5d74u,0x80deb1feu,0x9bdc06a7u,0xc19bf174u,0xe49b69c1u,0xefbe4786u,
        0x0fc19dc6u,0x240ca1ccu,0x2de92c6fu,0x4a7484aau,0x5cb0a9dcu,0x76f988dau,
        0x983e5152u,0xa831c66du,0xb00327c8u,0xbf597fc7u,0xc6e00bf3u,0xd5a79147u,
        0x06ca6351u,0x14292967u,0x27b70a85u,0x2e1b2138u,0x4d2c6dfcu,0x53380d13u,
        0x650a7354u,0x766a0abbu,0x81c2c92eu,0x92722c85u,0xa2bfe8a1u,0xa81a664bu,
        0xc24b8b70u,0xc76c51a3u,0xd192e819u,0xd6990624u,0xf40e3585u,0x106aa070u,
        0x19a4c116u,0x1e376c08u,0x2748774cu,0x34b0bcb5u,0x391c0cb3u,0x4ed8aa4au,
        0x5b9cca4fu,0x682e6ff3u,0x748f82eeu,0x78a5636fu,0x84c87814u,0x8cc70208u,
        0x90befffau,0xa4506cebu,0xbef9a3f7u,0xc67178f2u };
    uint32_t w[64], a,b,c,d,e,f,g,hh;
    for(int i = 0; i < 16; i++)
        w[i] = ((uint32_t)p[i*4] << 24) | ((uint32_t)p[i*4+1] << 16)
             | ((uint32_t)p[i*4+2] << 8) | (uint32_t)p[i*4+3];
    for(int i = 16; i < 64; i++){
        uint32_t s0 = coli_sha_ror(w[i-15],7) ^ coli_sha_ror(w[i-15],18) ^ (w[i-15] >> 3);
        uint32_t s1 = coli_sha_ror(w[i-2],17) ^ coli_sha_ror(w[i-2],19)  ^ (w[i-2] >> 10);
        w[i] = w[i-16] + s0 + w[i-7] + s1;
    }
    a=h[0]; b=h[1]; c=h[2]; d=h[3]; e=h[4]; f=h[5]; g=h[6]; hh=h[7];
    for(int i = 0; i < 64; i++){
        uint32_t S1 = coli_sha_ror(e,6) ^ coli_sha_ror(e,11) ^ coli_sha_ror(e,25);
        uint32_t ch = (e & f) ^ ((~e) & g);
        uint32_t t1 = hh + S1 + ch + K[i] + w[i];
        uint32_t S0 = coli_sha_ror(a,2) ^ coli_sha_ror(a,13) ^ coli_sha_ror(a,22);
        uint32_t mj = (a & b) ^ (a & c) ^ (b & c);
        uint32_t t2 = S0 + mj;
        hh=g; g=f; f=e; e=d+t1; d=c; c=b; b=a; a=t1+t2;
    }
    h[0]+=a; h[1]+=b; h[2]+=c; h[3]+=d; h[4]+=e; h[5]+=f; h[6]+=g; h[7]+=hh;
}

typedef struct { uint32_t h[8]; unsigned char buf[64]; size_t n; uint64_t bits; } ColiSha;

static void coli_sha256_init(ColiSha *s){
    s->h[0]=0x6a09e667u; s->h[1]=0xbb67ae85u; s->h[2]=0x3c6ef372u; s->h[3]=0xa54ff53au;
    s->h[4]=0x510e527fu; s->h[5]=0x9b05688cu; s->h[6]=0x1f83d9abu; s->h[7]=0x5be0cd19u;
    s->n = 0; s->bits = 0;
}
static void coli_sha256_update(ColiSha *s, const unsigned char *p, size_t len){
    s->bits += (uint64_t)len * 8u;
    while(len){
        size_t take = 64 - s->n;
        if(take > len) take = len;
        memcpy(s->buf + s->n, p, take);
        s->n += take; p += take; len -= take;
        if(s->n == 64){ coli_sha256_block(s->h, s->buf); s->n = 0; }
    }
}
static void coli_sha256_final(ColiSha *s, unsigned char out[32]){
    uint64_t bits = s->bits;
    unsigned char pad = 0x80;
    coli_sha256_update(s, &pad, 1);
    s->bits = bits;                       /* padding is not message length */
    unsigned char zero = 0;
    while(s->n != 56){ coli_sha256_update(s, &zero, 1); s->bits = bits; }
    unsigned char len[8];
    for(int i = 0; i < 8; i++) len[i] = (unsigned char)(bits >> (56 - i*8));
    coli_sha256_update(s, len, 8);
    for(int i = 0; i < 8; i++){
        out[i*4]   = (unsigned char)(s->h[i] >> 24);
        out[i*4+1] = (unsigned char)(s->h[i] >> 16);
        out[i*4+2] = (unsigned char)(s->h[i] >> 8);
        out[i*4+3] = (unsigned char)(s->h[i]);
    }
}

void coli_xdna_sha256(const void *data, size_t len, unsigned char out[32]){
    ColiSha s; coli_sha256_init(&s);
    coli_sha256_update(&s, (const unsigned char *)data, len);
    coli_sha256_final(&s, out);
}

int coli_xdna_sha256_file(const char *path, unsigned char out[32]){
    FILE *f = fopen(path, "rb");
    if(!f) return 0;
    ColiSha s; coli_sha256_init(&s);
    unsigned char buf[65536];
    size_t got;
    while((got = fread(buf, 1, sizeof buf, f)) > 0) coli_sha256_update(&s, buf, got);
    int ok = !ferror(f);
    fclose(f);
    if(!ok) return 0;
    coli_sha256_final(&s, out);
    return 1;
}

/* ======================================================================
 * Artifact registry
 * ====================================================================== */

/* Production rows.
 *
 * Only the first qualified family is present: the MoE shared-expert gate/up
 * projection, K=6144 -> N=2048, at the two M buckets the research programme
 * actually qualified. Other shape families were qualified as research artifacts
 * but are deliberately absent -- V1 scope is one unconditional family.
 *
 * Every hash below was recovered from sealed research evidence and recomputed
 * from the artifact bytes; the qualification flags come from a per-artifact
 * eligibility table, not from "it compiled". Filenames are LOGICAL: they are
 * resolved under a caller-supplied root, and no build path appears here.
 *
 * The artifacts themselves are not shipped by this build. A row describing an
 * absent file is the intended state and yields ARTIFACT_UNAVAILABLE. */
static const ColiXdnaArtifact g_xdna_production_rows[] = {
    {
        COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, "F3",
        64, 6144, 2048,
        COLI_XDNA_DT_BF16, COLI_XDNA_DT_BF16, COLI_XDNA_DT_F32,
        COLI_XDNA_TARGET_XDNA2,
        "wa_F3_M64_K6144_N2048.xclbin",
        "59309a70af9fd45ec5ed9467661332e08b9440276cf41afccecf439d34f0a4de",
        "wa_F3_M64_K6144_N2048_insts.bin",
        "bde093ac7bcf2df39159376bb11f6f631be3f5fdf6dac6219ce177bd3c80fc97",
        1, 1, 1, 1
    },
    {
        COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP, "F3",
        256, 6144, 2048,
        COLI_XDNA_DT_BF16, COLI_XDNA_DT_BF16, COLI_XDNA_DT_F32,
        COLI_XDNA_TARGET_XDNA2,
        "wa_F3_M256_K6144_N2048.xclbin",
        "e49c34388efea654eb63d006c4024ffe0e0ff763d62c3f769c0362ac3bb17263",
        "wa_F3_M256_K6144_N2048_insts.bin",
        "1a17753ce2516d8ccd7b992b61e1aa7c6b7a9fb655c4c0c0179eff075130d4b1",
        1, 1, 1, 1
    }
};
static const int g_xdna_production_count =
    (int)(sizeof g_xdna_production_rows / sizeof g_xdna_production_rows[0]);

static const ColiXdnaArtifact *g_rows  = g_xdna_production_rows;
/* Initialised to the production count, NOT to 0. Zero here would leave the
 * production table installed but empty, so every production lookup would answer
 * UNKNOWN_FAMILY until a test happened to install a registry -- invisible while
 * the only callers were tests that always install one, and wrong the moment a
 * production caller arrived. I5 is that first production caller. */
static int                     g_nrows =
    (int)(sizeof g_xdna_production_rows / sizeof g_xdna_production_rows[0]);

const char *coli_xdna_static_text(ColiXdnaStatic s){
    switch(s){
        case COLI_XDNA_STATIC_QUALIFIED:                 return "STATIC_ARTIFACT_QUALIFIED";
        case COLI_XDNA_STATIC_HELPER_UNAVAILABLE:        return "HELPER_UNAVAILABLE";
        case COLI_XDNA_STATIC_UNKNOWN_FAMILY:            return "UNKNOWN_FAMILY";
        case COLI_XDNA_STATIC_UNSUPPORTED_SHAPE_OR_BUCKET:return "UNSUPPORTED_SHAPE_OR_BUCKET";
        case COLI_XDNA_STATIC_UNSUPPORTED_FORMAT:        return "UNSUPPORTED_FORMAT";
        case COLI_XDNA_STATIC_ARTIFACT_UNAVAILABLE:      return "ARTIFACT_UNAVAILABLE";
        case COLI_XDNA_STATIC_ARTIFACT_INTEGRITY_FAILED: return "ARTIFACT_INTEGRITY_FAILED";
        case COLI_XDNA_STATIC_ARTIFACT_UNQUALIFIED:      return "ARTIFACT_UNQUALIFIED";
        case COLI_XDNA_STATIC_REGISTRY_INVALID:          return "REGISTRY_INVALID";
    }
    return "UNKNOWN";
}

static int coli_xdna_dtype_ok(ColiXdnaDtype d){
    return d == COLI_XDNA_DT_BF16 || d == COLI_XDNA_DT_F32;
}

static int coli_xdna_sha_text_ok(const char *s){
    if(!s) return 0;
    int i = 0;
    for(; s[i]; i++){
        char c = s[i];
        if(!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) return 0;
    }
    return i == 64;
}

/* A logical artifact name must stay inside the supplied root. Rejecting the
 * traversal at VALIDATION rather than at open() means a malformed registry can
 * never reach the filesystem at all. */
static int coli_xdna_name_ok(const char *s){
    if(!s || !*s) return 0;
    if(s[0] == '/' || s[0] == '\\') return 0;                 /* rooted */
    if(s[0] && s[1] == ':') return 0;                          /* drive-qualified */
    for(const char *p = s; *p; p++){
        if(p[0] == '.' && p[1] == '.') return 0;               /* any .. segment */
    }
    return 1;
}

static int coli_xdna_same_key(const ColiXdnaArtifact *a, const ColiXdnaArtifact *b){
    return a->family == b->family && a->artifact_m == b->artifact_m
        && a->k == b->k && a->n == b->n
        && a->in_dtype == b->in_dtype && a->weight_dtype == b->weight_dtype
        && a->out_dtype == b->out_dtype;
}

int coli_xdna_registry_validate(void){
    for(int i = 0; i < g_nrows; i++){
        const ColiXdnaArtifact *r = &g_rows[i];
        if(r->family == COLI_XDNA_FAMILY_NONE) return 0;
        if(!r->research_family || !*r->research_family) return 0;
        if(r->artifact_m == 0 || r->k == 0 || r->n == 0) return 0;
        if(!coli_xdna_dtype_ok(r->in_dtype))     return 0;
        if(!coli_xdna_dtype_ok(r->weight_dtype)) return 0;
        if(!coli_xdna_dtype_ok(r->out_dtype))    return 0;
        if(r->target != COLI_XDNA_TARGET_XDNA2)  return 0;
        if(!coli_xdna_name_ok(r->xclbin_name))   return 0;
        if(!coli_xdna_name_ok(r->insts_name))    return 0;
        if(!coli_xdna_sha_text_ok(r->xclbin_sha256)) return 0;
        if(!coli_xdna_sha_text_ok(r->insts_sha256))  return 0;
        /* No silent first-match ambiguity: two rows may never answer one key. */
        for(int j = 0; j < i; j++)
            if(coli_xdna_same_key(r, &g_rows[j])) return 0;
    }
    return 1;
}

const ColiXdnaArtifact *coli_xdna_registry_lookup(const ColiXdnaRequest *q){
    if(!q || q->family == COLI_XDNA_FAMILY_NONE) return NULL;
    for(int i = 0; i < g_nrows; i++){
        const ColiXdnaArtifact *r = &g_rows[i];
        /* The full key. Shape alone is never enough: a different operation with
         * the same M/K/N must not inherit this artifact's qualification. The
         * bucket is matched exactly -- research qualified specific M values, and
         * nothing here extrapolates between them. */
        if(r->family == q->family && r->artifact_m == q->m
           && r->k == q->k && r->n == q->n
           && r->in_dtype == q->in_dtype && r->weight_dtype == q->weight_dtype
           && r->out_dtype == q->out_dtype)
            return r;
    }
    return NULL;
}

static int coli_xdna_join(char *out, size_t cap, const char *root, const char *name){
    int written = snprintf(out, cap, "%s/%s", root, name);
    return written > 0 && (size_t)written < cap;
}

/* Both components must exist and hash exactly. Presence and integrity are
 * separate verdicts because they mean different things operationally: one is
 * "this build does not ship that artifact", the other is "the bytes are not the
 * bytes that were qualified". */
static ColiXdnaStatic coli_xdna_check_file(const char *root, const char *name,
                                           const char *expect_hex){
    char path[2048];
    if(!coli_xdna_join(path, sizeof path, root, name))
        return COLI_XDNA_STATIC_ARTIFACT_UNAVAILABLE;
    unsigned char got[32];
    if(!coli_xdna_sha256_file(path, got))
        return COLI_XDNA_STATIC_ARTIFACT_UNAVAILABLE;
    static const char *H = "0123456789abcdef";
    char hex[65];
    for(int i = 0; i < 32; i++){ hex[i*2] = H[got[i] >> 4]; hex[i*2+1] = H[got[i] & 15]; }
    hex[64] = '\0';
    if(strcmp(hex, expect_hex) != 0)
        return COLI_XDNA_STATIC_ARTIFACT_INTEGRITY_FAILED;
    return COLI_XDNA_STATIC_QUALIFIED;
}

ColiXdnaStatic coli_xdna_artifact_status(const ColiXdnaRequest *q, const char *root){
    if(!coli_xdna_registry_validate()) return COLI_XDNA_STATIC_REGISTRY_INVALID;

    if(!q) return COLI_XDNA_STATIC_UNKNOWN_FAMILY;
    if(q->family == COLI_XDNA_FAMILY_NONE) return COLI_XDNA_STATIC_UNKNOWN_FAMILY;
    if(!coli_xdna_dtype_ok(q->in_dtype) || !coli_xdna_dtype_ok(q->weight_dtype)
       || !coli_xdna_dtype_ok(q->out_dtype))
        return COLI_XDNA_STATIC_UNSUPPORTED_FORMAT;

    const ColiXdnaArtifact *a = coli_xdna_registry_lookup(q);
    if(!a){
        /* Distinguish "we have never qualified this operation" from "we have,
         * but not at this shape or bucket": the second is actionable, the first
         * is not. */
        for(int i = 0; i < g_nrows; i++)
            if(g_rows[i].family == q->family)
                return COLI_XDNA_STATIC_UNSUPPORTED_SHAPE_OR_BUCKET;
        return COLI_XDNA_STATIC_UNKNOWN_FAMILY;
    }

    /* Research qualification is checked BEFORE the filesystem: a row that was
     * never correctness-qualified must decline whether or not its bytes are
     * present and intact. Compiling and dispatching successfully is not
     * evidence of correctness -- N6 measured a design that did both and was
     * numerically wrong. */
    if(!a->runtime_weight_qualified || !a->correctness_qualified
       || !a->userptr_qualified || !a->structural_qualified)
        return COLI_XDNA_STATIC_ARTIFACT_UNQUALIFIED;

    if(!root || !*root) return COLI_XDNA_STATIC_ARTIFACT_UNAVAILABLE;

    ColiXdnaStatic s = coli_xdna_check_file(root, a->xclbin_name, a->xclbin_sha256);
    if(s != COLI_XDNA_STATIC_QUALIFIED) return s;
    return coli_xdna_check_file(root, a->insts_name, a->insts_sha256);
}

ColiXdnaStatic coli_xdna_static_eligibility(const ColiXdnaRequest *q, const char *root){
    if(coli_xdna_binding() != COLI_XDNA_AVAILABLE)
        return COLI_XDNA_STATIC_HELPER_UNAVAILABLE;
    return coli_xdna_artifact_status(q, root);
}

void coli_xdna_test_set_registry(const ColiXdnaArtifact *rows, int count){
    if(rows && count > 0){ g_rows = rows;                    g_nrows = count; }
    else                 { g_rows = g_xdna_production_rows;  g_nrows = g_xdna_production_count; }
}

/* ======================================================================
 * Prepared host state
 *
 * Engine-owned, derived from the authoritative fmt=4 tensor and never a
 * replacement for it. The helper neither allocates nor frees any of this.
 * ====================================================================== */

struct ColiXdnaPrepared {
    ColiXdnaPrepState state;
    void            *base;      /* 4096-aligned, from posix_memalign */
    size_t           bytes;     /* logical payload: k*n*sizeof(bf16) */
    size_t           capacity;  /* what base can actually hold */
    unsigned         k, n;
    unsigned long long generation;   /* bumped on every successful publication */
};

/* Engine-side host accounting. Deliberately counts LOGICAL payload bytes: the
 * allocator's own bookkeeping overhead is real but not knowable from here, and
 * reporting a number we cannot substantiate would be worse than reporting the
 * one we can. */
static size_t g_xdna_prepared_bytes;
static int    g_xdna_prepared_objects;

/* Defined with the execution lane below, declared here because the prepared
 * buffer owner must be able to tell the lane to let go of memory it is about
 * to free or replace. */
static void coli_xdna_lane_forget_pointer(const void *p);

const char *coli_xdna_prep_text(ColiXdnaPrepState s){
    switch(s){
        case COLI_XDNA_PREP_UNPREPARED: return "UNPREPARED";
        case COLI_XDNA_PREP_PREPARING:  return "PREPARING";
        case COLI_XDNA_PREP_VALID:      return "PREPARED_VALID";
        case COLI_XDNA_PREP_INVALID:    return "PREPARED_INVALID";
    }
    return "UNKNOWN";
}

int coli_xdna_pointer_alignment_ok(const void *p){
    if(!p) return 0;
    return ((uintptr_t)p % (uintptr_t)COLI_XDNA_PREPARED_ALIGNMENT) == 0;
}

ColiXdnaPrepared *coli_xdna_prepared_create(void){
    ColiXdnaPrepared *p = (ColiXdnaPrepared *)calloc(1, sizeof *p);
    if(!p) return NULL;
    p->state = COLI_XDNA_PREP_UNPREPARED;
    g_xdna_prepared_objects++;
    return p;
}

/* Drop capacity only. Validity and allocation lifetime are separate concerns,
 * so they get separate operations. */
void coli_xdna_prepared_free_buffer(ColiXdnaPrepared *p){
    if(!p) return;
    /* Tell the lane before the memory goes away: a helper-owned userptr wrapper
     * borrowing this allocation must never outlive it. */
    coli_xdna_lane_forget_pointer(p->base);
    if(p->base){
        compat_aligned_free(p->base);       /* MUST pair with posix_memalign */
        p->base = NULL;
    }
    if(g_xdna_prepared_bytes >= p->bytes) g_xdna_prepared_bytes -= p->bytes;
    else                                  g_xdna_prepared_bytes = 0;
    p->bytes = 0; p->capacity = 0; p->k = 0; p->n = 0;
    p->state = COLI_XDNA_PREP_UNPREPARED;
}

void coli_xdna_prepared_release(ColiXdnaPrepared **pp){
    if(!pp || !*pp) return;
    coli_xdna_prepared_free_buffer(*pp);
    free(*pp);
    *pp = NULL;                              /* repeated release is a no-op */
    if(g_xdna_prepared_objects > 0) g_xdna_prepared_objects--;
}

/* k*n*2 with checked arithmetic. An unchecked product would wrap and reach the
 * allocator as a small, plausible number, which is the worst possible outcome:
 * a successful allocation far too small for what the caller will write. */
static int coli_xdna_payload_bytes(unsigned k, unsigned n, ColiXdnaDtype dt, size_t *out){
    if(k == 0 || n == 0) return 0;
    if(dt != COLI_XDNA_DT_BF16) return 0;    /* only the qualified prepared dtype */
    size_t kk = (size_t)k, nn = (size_t)n;
    if(kk > SIZE_MAX / nn) return 0;
    size_t elems = kk * nn;
    if(elems > SIZE_MAX / 2u) return 0;
    *out = elems * 2u;                       /* sizeof(bf16) */
    return 1;
}

int coli_xdna_prepare_begin(ColiXdnaPrepared *p, unsigned k, unsigned n,
                            ColiXdnaDtype prepared_dtype){
    if(!p) return 0;
    size_t need = 0;
    if(!coli_xdna_payload_bytes(k, n, prepared_dtype, &need)) return 0;

    /* Reuse retained capacity when it is genuinely large enough. The old bytes
     * stay physically present and are NOT semantically valid: the state is
     * PREPARING until a producer publishes success over them. */
    if(!p->base || p->capacity < need){
        void *fresh = NULL;
        if(p->base){
            compat_aligned_free(p->base);
            p->base = NULL;
            if(g_xdna_prepared_bytes >= p->bytes) g_xdna_prepared_bytes -= p->bytes;
            else                                  g_xdna_prepared_bytes = 0;
            p->bytes = 0; p->capacity = 0;
        }
        if(posix_memalign(&fresh, COLI_XDNA_PREPARED_ALIGNMENT, need) != 0 || !fresh){
            p->state = COLI_XDNA_PREP_UNPREPARED;
            return 0;
        }
        /* The allocator is contracted to align, but the contract is re-checked:
         * an unaligned buffer must never travel further than this line. */
        if(!coli_xdna_pointer_alignment_ok(fresh)){
            compat_aligned_free(fresh);
            p->state = COLI_XDNA_PREP_UNPREPARED;
            return 0;
        }
        p->base = fresh;
        p->capacity = need;
    } else {
        if(g_xdna_prepared_bytes >= p->bytes) g_xdna_prepared_bytes -= p->bytes;
        else                                  g_xdna_prepared_bytes = 0;
    }

    p->bytes = need;                         /* logical payload, never rounded */
    p->k = k; p->n = n;
    g_xdna_prepared_bytes += need;
    p->state = COLI_XDNA_PREP_PREPARING;
    return 1;
}

void *coli_xdna_prepare_dest(ColiXdnaPrepared *p){
    /* Only a producer inside an open cycle may write. A published image is not
     * writable, so it cannot be corrupted while something believes it valid. */
    if(!p || p->state != COLI_XDNA_PREP_PREPARING) return NULL;
    return p->base;
}

int coli_xdna_prepare_publish_success(ColiXdnaPrepared *p){
    if(!p) return 0;
    if(p->state != COLI_XDNA_PREP_PREPARING) return 0;   /* no shortcut to VALID */
    if(!p->base || !coli_xdna_pointer_alignment_ok(p->base)) return 0;
    if(p->bytes == 0 || p->capacity < p->bytes) return 0;
    p->state = COLI_XDNA_PREP_VALID;
    /* A new publication is a new image even when it lands on the same address:
     * retained capacity is reused in place. Anything caching a view of these
     * bytes keys on (pointer, generation), never on the pointer alone. */
    p->generation++;
    return 1;
}

void coli_xdna_prepare_publish_failure(ColiXdnaPrepared *p){
    if(!p) return;
    if(p->state != COLI_XDNA_PREP_PREPARING) return;
    /* Capacity is retained and still accounted; only the contents are revoked. */
    p->state = COLI_XDNA_PREP_INVALID;
}

int coli_xdna_prepared_invalidate(ColiXdnaPrepared *p){
    if(!p) return 0;
    if(p->state != COLI_XDNA_PREP_VALID) return 0;
    p->state = COLI_XDNA_PREP_INVALID;
    /* The image is no longer authoritative, so no device-side view of it may
     * remain current either. */
    coli_xdna_lane_forget_pointer(p->base);
    return 1;
}

ColiXdnaPrepState coli_xdna_prepared_state(const ColiXdnaPrepared *p){
    return p ? p->state : COLI_XDNA_PREP_UNPREPARED;
}
size_t   coli_xdna_prepared_bytes(const ColiXdnaPrepared *p){ return p ? p->bytes : 0; }
unsigned coli_xdna_prepared_k(const ColiXdnaPrepared *p){ return p ? p->k : 0u; }
unsigned coli_xdna_prepared_n(const ColiXdnaPrepared *p){ return p ? p->n : 0u; }
unsigned long long coli_xdna_prepared_generation(const ColiXdnaPrepared *p){
    return p ? p->generation : 0ull;
}

size_t coli_xdna_prepared_total_bytes(void){ return g_xdna_prepared_bytes; }
int    coli_xdna_prepared_live_objects(void){ return g_xdna_prepared_objects; }

const void *coli_xdna_prepared_image(const ColiXdnaPrepared *p){
    /* Only a published image is readable. An unpublished or revoked one returns
     * NULL rather than a pointer to bytes with no authority. */
    if(!p || p->state != COLI_XDNA_PREP_VALID) return NULL;
    return p->base;
}
const void *coli_xdna_prepared_image_unchecked(const ColiXdnaPrepared *p){
    return p ? p->base : NULL;
}

/* ======================================================================
 * fmt=4 grouped int4  ->  BF16 B[K,N]
 *
 * Semantics are taken from the production kernel (quant.h matmul_i4_grouped),
 * not from a specification of it:
 *
 *   rb = (I+1)/2                 bytes per output row
 *   ng = (I+gs-1)/gs             groups per output row
 *   row   = q4    + o*rb         weights are stored [O][I]
 *   scl   = scale + o*ng
 *   byte  = row[i>>1]
 *   nib   = (i&1) ? byte>>4 : byte&0x0F      even i low, odd i high
 *   value = (nib - 8) * scl[i/gs]
 *
 * The destination is BF16 B[K,N] with K=I and N=O, so the write is
 * dst[i*O + o] -- the [O,I] -> [I,O] transform is inherent in the addressing
 * and needs no intermediate matrix. Values are converted one at a time through
 * a float scalar, so no full-sized FP32 image is ever allocated.
 * ====================================================================== */

/* Test-only. Production never sets this; there is no env var or flag for it. */
static int g_xdna_convert_fail_pct;
/* How many conversions have been performed. I1-I3 pinned this at zero because
 * no converter existed; from I4 it counts real work, which is what makes
 * "conversion happened" and "device work happened" separately assertable. */
static int g_xdna_conversions;
void coli_xdna_test_set_convert_fail_pct(int pct){
    g_xdna_convert_fail_pct = (pct > 0 && pct < 100) ? pct : 0;
}

const char *coli_xdna_prep_result_text(ColiXdnaPrepResult r){
    switch(r){
        case COLI_XDNA_PREP_OK:                    return "OK";
        case COLI_XDNA_PREP_ERR_UNSUPPORTED_FORMAT:return "UNSUPPORTED_SOURCE_FORMAT";
        case COLI_XDNA_PREP_ERR_INVALID_SOURCE:    return "INVALID_SOURCE";
        case COLI_XDNA_PREP_ERR_SIZE:              return "SIZE_OVERFLOW";
        case COLI_XDNA_PREP_ERR_ALLOC:             return "ALLOCATION_FAILED";
        case COLI_XDNA_PREP_ERR_FAILED:            return "PREPARATION_FAILED";
        case COLI_XDNA_PREP_ERR_STATE:             return "STATE_ERROR";
    }
    return "UNKNOWN";
}

/* float -> bfloat16, round to nearest even. Truncation would be a different
 * function and a different image; the qualified oracle rounds. */
static unsigned short coli_xdna_f2b(float f){
    unsigned int u;
    memcpy(&u, &f, sizeof u);
    u += 0x7FFFu + ((u >> 16) & 1u);
    return (unsigned short)(u >> 16);
}

/* Convert ONE fmt=4 output column into the transposed BF16 destination.
 *
 * Extracted verbatim from the loop body that used to live inline in
 * coli_xdna_prepare_from_fmt4, so the per-element arithmetic -- nibble order,
 * (nib-8)*scale, the group index i/gs, the destination index i*O+o and the
 * coli_xdna_f2b rounding -- is textually what the qualified version computed.
 * Extracting it is what lets the failure-injection path stay serial while the
 * ordinary path runs the same code across threads. */
static void coli_xdna_convert_column(unsigned short *dst,
                                     const unsigned char *q4,
                                     const float *scale,
                                     int o, int I, int O, int gs,
                                     size_t rb, size_t ng)
{
    (void)ng;
    const unsigned char *row = q4 + (size_t)o * rb;
    const float *scl = scale + (size_t)o * ng;
    for(int i = 0; i < I; i++){
        unsigned char byte = row[(size_t)i >> 1];
        int nib = (i & 1) ? (int)(byte >> 4) : (int)(byte & 0x0F);
        dst[(size_t)i * (size_t)O + (size_t)o] =
            coli_xdna_f2b((float)(nib - 8) * scl[(size_t)i / (size_t)gs]);
    }
}

ColiXdnaPrepResult coli_xdna_prepare_from_fmt4(ColiXdnaPrepared *p,
                                               int fmt,
                                               const unsigned char *q4,
                                               const float *scale,
                                               int I, int O, int gs){
    if(!p) return COLI_XDNA_PREP_ERR_STATE;

    /* Everything that can be rejected is rejected BEFORE the cycle opens, so a
     * bad source never leaves an object stranded in PREPARING or INVALID. */
    if(fmt != 4) return COLI_XDNA_PREP_ERR_UNSUPPORTED_FORMAT;
    if(!q4 || !scale) return COLI_XDNA_PREP_ERR_INVALID_SOURCE;
    if(I <= 0 || O <= 0 || gs <= 0) return COLI_XDNA_PREP_ERR_INVALID_SOURCE;

    size_t need = 0;
    if(!coli_xdna_payload_bytes((unsigned)I, (unsigned)O, COLI_XDNA_DT_BF16, &need))
        return COLI_XDNA_PREP_ERR_SIZE;

    if(!coli_xdna_prepare_begin(p, (unsigned)I, (unsigned)O, COLI_XDNA_DT_BF16))
        return COLI_XDNA_PREP_ERR_ALLOC;

    unsigned short *dst = (unsigned short *)coli_xdna_prepare_dest(p);
    if(!dst){                                    /* cannot happen; never leave PREPARING */
        coli_xdna_prepare_publish_failure(p);
        return COLI_XDNA_PREP_ERR_STATE;
    }

    const size_t rb = ((size_t)I + 1u) / 2u;
    const size_t ng = ((size_t)I + (size_t)gs - 1u) / (size_t)gs;
    const int fail_after = g_xdna_convert_fail_pct
                         ? (int)(((long)O * g_xdna_convert_fail_pct) / 100) : -1;

    /* One output column per iteration. Every element of column o is written by
     * exactly that iteration, so distinct o never touch the same destination
     * byte -- which is what makes this loop safe to run in parallel. The
     * arithmetic is unchanged; only which thread runs which o differs. */
    if(fail_after >= 0){
        /* Failure injection stays SERIAL and byte-for-byte as it was. The early
         * return crosses a loop boundary, which an OpenMP structured block may
         * not, and A4 qualified degraded behaviour against exactly this
         * ordering. This path only runs when a test has armed
         * g_xdna_convert_fail_pct, where speed is irrelevant. */
        for(int o = 0; o < O; o++){
            if(o == fail_after){
                /* Deterministic mid-conversion failure: rows before this one are
                 * genuinely converted, the rest keep whatever they held. */
                coli_xdna_prepare_publish_failure(p);
                return COLI_XDNA_PREP_ERR_FAILED;
            }
            coli_xdna_convert_column(dst, q4, scale, o, I, O, gs, rb, ng);
        }
    } else {
        /* This file is already compiled with -fopenmp and libgomp is linked
         * statically, so this adds no flag, no import and no runtime
         * dependency. N6-A2-Q1 measured this exact parallelisation and reported
         * BF16_BIT_MISMATCH_COUNT = 0 at 1/4/8/16/32 threads: the output bytes
         * depend on neither thread count nor completion order. */
#ifdef _OPENMP
        #pragma omp parallel for schedule(static)
#endif
        for(int o = 0; o < O; o++)
            coli_xdna_convert_column(dst, q4, scale, o, I, O, gs, rb, ng);
    }

    if(!coli_xdna_prepare_publish_success(p)){
        coli_xdna_prepare_publish_failure(p);
        return COLI_XDNA_PREP_ERR_STATE;
    }
    g_xdna_conversions++;
    return COLI_XDNA_PREP_OK;
}

/* ======================================================================
 * Full hard eligibility and native execution  (W2-N7-I5)
 *
 * The engine keeps every decision that requires knowing what the operation
 * MEANS: which family it is, which artifact answers it, whether the bytes are
 * trustworthy, how the weight is represented, whether the request is in range.
 * The helper is handed an already-selected, already-verified artifact and an
 * already-prepared buffer, and executes. It is told nothing about tensors,
 * experts, layers, routing or models, and it decides nothing.
 * ====================================================================== */

const char *coli_xdna_hard_text(ColiXdnaHard h){
    switch(h){
        case COLI_XDNA_HARD_ELIGIBLE:                    return "HARD_ELIGIBLE";
        case COLI_XDNA_HARD_FAMILY_UNSUPPORTED:          return "FAMILY_UNSUPPORTED";
        case COLI_XDNA_HARD_M_OUT_OF_RANGE:              return "M_OUT_OF_RANGE";
        case COLI_XDNA_HARD_SHAPE_UNSUPPORTED:           return "SHAPE_UNSUPPORTED";
        case COLI_XDNA_HARD_FORMAT_UNSUPPORTED:          return "FORMAT_UNSUPPORTED";
        case COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED:      return "GROUP_SIZE_UNSUPPORTED";
        case COLI_XDNA_HARD_LAYOUT_UNSUPPORTED:          return "LAYOUT_UNSUPPORTED";
        case COLI_XDNA_HARD_ARTIFACT_UNAVAILABLE:        return "ARTIFACT_UNAVAILABLE";
        case COLI_XDNA_HARD_ARTIFACT_INTEGRITY_FAILED:   return "ARTIFACT_INTEGRITY_FAILED";
        case COLI_XDNA_HARD_ARTIFACT_UNQUALIFIED:        return "ARTIFACT_UNQUALIFIED";
        case COLI_XDNA_HARD_REGISTRY_INVALID:            return "REGISTRY_INVALID";
        case COLI_XDNA_HARD_PREPARED_INVALID:            return "PREPARED_INVALID";
        case COLI_XDNA_HARD_ALIGNMENT_INVALID:           return "ALIGNMENT_INVALID";
        case COLI_XDNA_HARD_HELPER_UNAVAILABLE:          return "HELPER_UNAVAILABLE";
        case COLI_XDNA_HARD_HELPER_ABI_INCOMPATIBLE:     return "HELPER_ABI_INCOMPATIBLE";
        case COLI_XDNA_HARD_DEVICE_UNAVAILABLE:          return "DEVICE_UNAVAILABLE";
        case COLI_XDNA_HARD_ARTIFACT_RUNTIME_UNAVAILABLE:return "ARTIFACT_RUNTIME_UNAVAILABLE";
        case COLI_XDNA_HARD_WEIGHT_WRAP_UNAVAILABLE:     return "WEIGHT_WRAP_UNAVAILABLE";
    }
    return "UNKNOWN";
}

const char *coli_xdna_exec_text(ColiXdnaExec e){
    switch(e){
        case COLI_XDNA_EXEC_OK:                    return "OK";
        case COLI_XDNA_EXEC_DECLINED:              return "DECLINED";
        case COLI_XDNA_EXEC_DEVICE_INIT_FAILED:    return "DEVICE_INIT_FAILED";
        case COLI_XDNA_EXEC_ARTIFACT_OPEN_FAILED:  return "ARTIFACT_OPEN_FAILED";
        case COLI_XDNA_EXEC_WEIGHT_PREPARE_FAILED: return "WEIGHT_PREPARE_FAILED";
        case COLI_XDNA_EXEC_WEIGHT_WRAP_FAILED:    return "WEIGHT_WRAP_FAILED";
        case COLI_XDNA_EXEC_ACTIVATION_FAILED:     return "ACTIVATION_FAILED";
        case COLI_XDNA_EXEC_EXECUTE_FAILED:        return "EXECUTE_FAILED";
        case COLI_XDNA_EXEC_COMPLETION_FAILED:     return "COMPLETION_FAILED";
    }
    return "UNKNOWN";
}

/* ---- execution-lane state ------------------------------------------------
 *
 * Transient per-operation staging (activation, output) and the identity of the
 * artifact currently open in the helper. The prepared WEIGHT is not here: it
 * belongs to the tensor, which outlives any of this. */
static int          g_xdna_force;                 /* internal test control only */
/* Explicit PRODUCT policy (W2-N7-P1), deliberately distinct from g_xdna_force.
 * Set only from a parsed user request; never from discovery, never by default. */
static int          g_xdna_explicit;
static char         g_xdna_root[1024];            /* artifact root; empty = none */
/* Set only by the test seam. When 0, the root is the PRODUCT one derived from
 * the executable directory. Kept as a separate flag rather than inferred from
 * g_xdna_root being empty, because "a test deliberately set no root" and "no
 * test has spoken, use the product default" are different states and the first
 * must not silently acquire the second. */
static int          g_xdna_root_overridden;
static ColiXdnaHard g_xdna_last_hard = COLI_XDNA_HARD_ELIGIBLE;
static ColiXdnaExec g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED;
static int g_xdna_dispatches, g_xdna_completions, g_xdna_artifact_opens;
static int g_xdna_act_preps, g_xdna_padded_ops, g_xdna_fallbacks;

/* ---- bucket accounting ---------------------------------------------------
 *
 * Two buckets are live, so a single padded_ops total no longer identifies what
 * happened: it cannot say which artifact ran, and it weights a 24-row pad the
 * same as a 191-row one. Everything below is indexed by BUCKET SLOT, and the
 * slot mapping is the one place that knows the bucket set.
 *
 * Slots, not artifact_m values, because the accounting must stay O(buckets)
 * rather than sparse-keyed, and because an unknown artifact_m must be a loud
 * miss rather than a silently created bin. */
enum { COLI_XDNA_SLOT_M64 = 0, COLI_XDNA_SLOT_M256 = 1, COLI_XDNA_SLOTS = 2 };

static int coli_xdna_bucket_slot(unsigned artifact_m){
    if(artifact_m == COLI_XDNA_BUCKET_M_SMALL) return COLI_XDNA_SLOT_M64;
    if(artifact_m == COLI_XDNA_BUCKET_M_LARGE) return COLI_XDNA_SLOT_M256;
    return -1;                                  /* not a compiled bucket */
}

static int       g_xdna_b_hard_eligible[COLI_XDNA_SLOTS];
static int       g_xdna_b_dispatches[COLI_XDNA_SLOTS];
static int       g_xdna_b_completions[COLI_XDNA_SLOTS];
static int       g_xdna_b_padded_ops[COLI_XDNA_SLOTS];
static long long g_xdna_b_padded_rows[COLI_XDNA_SLOTS];
static int       g_xdna_b_artifact_opens[COLI_XDNA_SLOTS];
static int       g_xdna_b_switches[COLI_XDNA_SLOTS][COLI_XDNA_SLOTS];
static int       g_xdna_m_above_range_declines;

/* THE BUCKET SELECTOR.
 *
 * The smallest compiled bucket that holds `logical_m`, or 0 if none does.
 * This is SHAPE ELIGIBILITY, not an economic choice: it has no cost model, no
 * device-load input and no preference. A caller may not override it, and it may
 * not round a request up into a bucket that was never built -- there is no
 * M128 artifact, so an S=65 request runs on M256 with 191 padded rows rather
 * than on something that does not exist.
 *
 * Pure: no globals, no device, no registry. The registry lookup that follows
 * still has to find a row for the returned bucket, so a bucket named here but
 * absent from the registry declines at gate 6 exactly as an unknown shape
 * would. */
static unsigned coli_xdna_bucket_for_logical_m(int logical_m){
    if(logical_m < COLI_XDNA_LOGICAL_M_MIN) return 0u;
    if(logical_m <= (int)COLI_XDNA_BUCKET_M_SMALL) return COLI_XDNA_BUCKET_M_SMALL;
    if(logical_m <= (int)COLI_XDNA_BUCKET_M_LARGE) return COLI_XDNA_BUCKET_M_LARGE;
    return 0u;                                  /* above every bucket: decline */
}
static int g_xdna_device_opens, g_xdna_helper_calls, g_xdna_userptr_wraps;
static int g_xdna_wrapper_reuses, g_xdna_wrapper_releases, g_xdna_stale_rejects;
static int g_xdna_output_valid;
static ColiXdnaLaneHealth g_xdna_lane_health = COLI_XDNA_LANE_UNINITIALIZED;

/* Which artifact the helper currently holds open, so a run of operations on the
 * same shape does not reopen the device, the context or the program. This is a
 * RUNTIME object cache, not an identity cache: it can only ever hold what the
 * engine already selected, and a different selection closes it and reopens. */
static struct {
    int      open;
    unsigned m, k, n;
    /* The wrapped weight, identified by (pointer, generation). The pointer
     * alone is NOT an identity: retained capacity is reused in place, so a
     * different image can occupy the same address. */
    const void        *wrapped;
    unsigned long long wrapped_gen;
    unsigned short *act;   size_t act_cap;    /* [artifact_m, K] BF16 staging */
    float          *out;   size_t out_cap;    /* [artifact_m, N] F32 staging  */
} g_lane;
/* Release the helper-owned userptr wrapper and forget its identity.
 *
 * Called from three places, and each one matters: before wrapping a different
 * image, before the engine frees or replaces the memory the wrapper borrows,
 * and at shutdown. The wrapper borrows CALLER memory, so it must never outlive
 * the allocation it points into. */
static void coli_xdna_lane_drop_wrapper(void){
    if(!g_lane.wrapped) return;
    coli_xdna_helper_release_weight_call();
    g_lane.wrapped = NULL;
    g_lane.wrapped_gen = 0;
    g_xdna_wrapper_releases++;
}

/* The engine is about to free or replace prepared memory. If the helper holds a
 * wrapper over exactly that allocation, drop it FIRST. Without this the wrapper
 * would reference a freed region until the next wrap replaced it -- and a
 * subsequent allocation landing on the same address would make the stale
 * wrapper look current. */
static void coli_xdna_lane_forget_pointer(const void *p){
    if(p && g_lane.wrapped == p) coli_xdna_lane_drop_wrapper();
}

/* ---- helper entry points (ABI generation 2) ------------------------------
 *
 * Everything crossing this boundary is a fixed-width integer, a size_t, a raw
 * pointer with an explicit byte count, or a C string. No C++ type, no XRT type,
 * no engine type, and no exception: the helper catches everything and converts
 * it to one of these status codes. */
enum {
    COLI_XDNA_H_OK              =  0,
    COLI_XDNA_H_E_DEVICE        = -1,
    COLI_XDNA_H_E_ARTIFACT      = -2,
    COLI_XDNA_H_E_NOT_OPEN      = -3,
    COLI_XDNA_H_E_WRAP          = -4,
    COLI_XDNA_H_E_SIZE          = -5,
    COLI_XDNA_H_E_DISPATCH      = -6,
    COLI_XDNA_H_E_COMPLETION    = -7,
    COLI_XDNA_H_E_EXCEPTION     = -8
};

int coli_xdna_test_dispatches(void){ return g_xdna_dispatches; }
int coli_xdna_test_completions(void){ return g_xdna_completions; }
int coli_xdna_test_artifact_opens(void){ return g_xdna_artifact_opens; }
int coli_xdna_test_activation_preparations(void){ return g_xdna_act_preps; }
int coli_xdna_test_padded_operations(void){ return g_xdna_padded_ops; }
int coli_xdna_test_fallbacks(void){ return g_xdna_fallbacks; }

/* Per-bucket accessors. An unrecognised artifact M returns 0 -- it is not a
 * bucket this build compiled, so there is nothing to report and nothing may be
 * silently folded into a neighbouring bin. */
int coli_xdna_test_bucket_dispatches(unsigned artifact_m){
    int sl = coli_xdna_bucket_slot(artifact_m); return sl < 0 ? 0 : g_xdna_b_dispatches[sl];
}
int coli_xdna_test_bucket_completions(unsigned artifact_m){
    int sl = coli_xdna_bucket_slot(artifact_m); return sl < 0 ? 0 : g_xdna_b_completions[sl];
}
int coli_xdna_test_bucket_padded_operations(unsigned artifact_m){
    int sl = coli_xdna_bucket_slot(artifact_m); return sl < 0 ? 0 : g_xdna_b_padded_ops[sl];
}
long long coli_xdna_test_bucket_padded_rows(unsigned artifact_m){
    int sl = coli_xdna_bucket_slot(artifact_m); return sl < 0 ? 0 : g_xdna_b_padded_rows[sl];
}
int coli_xdna_test_bucket_artifact_opens(unsigned artifact_m){
    int sl = coli_xdna_bucket_slot(artifact_m); return sl < 0 ? 0 : g_xdna_b_artifact_opens[sl];
}
int coli_xdna_test_bucket_hard_eligible(unsigned artifact_m){
    int sl = coli_xdna_bucket_slot(artifact_m); return sl < 0 ? 0 : g_xdna_b_hard_eligible[sl];
}
int coli_xdna_test_m_above_range_declines(void){ return g_xdna_m_above_range_declines; }
int coli_xdna_test_bucket_switches(unsigned from_m, unsigned to_m){
    int a = coli_xdna_bucket_slot(from_m), b = coli_xdna_bucket_slot(to_m);
    return (a < 0 || b < 0) ? 0 : g_xdna_b_switches[a][b];
}
unsigned coli_xdna_test_bucket_for(int logical_m){
    return coli_xdna_bucket_for_logical_m(logical_m);
}
void coli_xdna_test_reset_bucket_counters(void){
    memset(g_xdna_b_hard_eligible,  0, sizeof g_xdna_b_hard_eligible);
    memset(g_xdna_b_dispatches,     0, sizeof g_xdna_b_dispatches);
    memset(g_xdna_b_completions,    0, sizeof g_xdna_b_completions);
    memset(g_xdna_b_padded_ops,     0, sizeof g_xdna_b_padded_ops);
    memset(g_xdna_b_padded_rows,    0, sizeof g_xdna_b_padded_rows);
    memset(g_xdna_b_artifact_opens, 0, sizeof g_xdna_b_artifact_opens);
    memset(g_xdna_b_switches,       0, sizeof g_xdna_b_switches);
    g_xdna_m_above_range_declines = 0;
}
int coli_xdna_test_wrapper_reuses(void){ return g_xdna_wrapper_reuses; }
int coli_xdna_test_wrapper_releases(void){ return g_xdna_wrapper_releases; }
int coli_xdna_test_stale_wrapper_rejects(void){ return g_xdna_stale_rejects; }
ColiXdnaHard coli_xdna_test_last_hard(void){ return g_xdna_last_hard; }
ColiXdnaExec coli_xdna_test_last_exec(void){ return g_xdna_last_exec; }

void coli_xdna_test_set_force_execution(int on){ g_xdna_force = on ? 1 : 0; }
/* The product artifact root: <exe-dir>\xdna. Same anchor as the helper, same
 * absolute form, same refusal to search. */
int coli_xdna_product_artifact_root(char *out, size_t cap){
#ifdef _WIN32
    char exe[512];
    DWORD n = GetModuleFileNameA(NULL, exe, (DWORD)sizeof exe);
    if(n == 0 || n >= sizeof exe) return 0;
    char *slash = strrchr(exe, '\\');
    char *fwd   = strrchr(exe, '/');
    if(fwd && (!slash || fwd > slash)) slash = fwd;
    if(!slash) return 0;
    *slash = '\0';
    int written = snprintf(out, cap, "%s\\%s", exe, COLI_XDNA_ARTIFACT_DIR);
    return written > 0 && (size_t)written < cap;
#else
    (void)out; (void)cap;
    return 0;
#endif
}

/* The root actually used by execution: a test override when one was set,
 * otherwise the product location. Returns NULL when neither is available,
 * which the artifact status treats as ARTIFACT_UNAVAILABLE -- never as a
 * licence to look somewhere else. */
static const char *coli_xdna_effective_root(char *scratch, size_t cap){
    if(g_xdna_root_overridden) return g_xdna_root[0] ? g_xdna_root : NULL;
    if(coli_xdna_product_artifact_root(scratch, cap)) return scratch;
    return NULL;
}

const char *coli_xdna_provision_text(ColiXdnaProvision p){
    switch(p){
    case COLI_XDNA_PROV_READY:              return "READY";
    case COLI_XDNA_PROV_NOT_REQUESTED:      return "NOT_REQUESTED";
    case COLI_XDNA_PROV_PACKAGE_MISSING:    return "PACKAGE_MISSING";
    case COLI_XDNA_PROV_PACKAGE_INCOMPLETE: return "PACKAGE_INCOMPLETE";
    case COLI_XDNA_PROV_INTEGRITY_FAILED:   return "INTEGRITY_FAILED";
    case COLI_XDNA_PROV_HELPER_UNAVAILABLE: return "HELPER_UNAVAILABLE";
    case COLI_XDNA_PROV_REGISTRY_INVALID:   return "REGISTRY_INVALID";
    }
    return "UNKNOWN";
}

ColiXdnaProvision coli_xdna_provision_status(const char **missing_name){
    if(missing_name) *missing_name = NULL;
    if(!coli_xdna_registry_validate()) return COLI_XDNA_PROV_REGISTRY_INVALID;

    char scratch[1024];
    const char *root = coli_xdna_effective_root(scratch, sizeof scratch);
    if(!root) return COLI_XDNA_PROV_PACKAGE_MISSING;

    /* Every row this build ships, not just the first. A package holding only
     * the small bucket would otherwise report READY and then fail at the first
     * long request -- surfacing at activation is the whole point. */
    int checked = 0;
    for(int i = 0; i < g_nrows; i++){
        const ColiXdnaArtifact *a = &g_rows[i];
        const char *names[2]  = { a->xclbin_name, a->insts_name };
        const char *hashes[2] = { a->xclbin_sha256, a->insts_sha256 };
        for(int k = 0; k < 2; k++){
            ColiXdnaStatic st = coli_xdna_check_file(root, names[k], hashes[k]);
            if(st == COLI_XDNA_STATIC_QUALIFIED){ checked++; continue; }
            if(missing_name) *missing_name = names[k];
            if(st == COLI_XDNA_STATIC_ARTIFACT_INTEGRITY_FAILED)
                return COLI_XDNA_PROV_INTEGRITY_FAILED;
            /* Absent file. If NOTHING was verified yet the package as a whole is
             * missing; if some files verified, the package is present but
             * incomplete. Different problems, different fixes. */
            return checked ? COLI_XDNA_PROV_PACKAGE_INCOMPLETE
                           : COLI_XDNA_PROV_PACKAGE_MISSING;
        }
    }

    /* Bytes are good. The helper is the remaining prerequisite, and it is
     * reported separately because "install the package" and "the helper will
     * not load" are not the same instruction. */
    if(coli_xdna_binding() != COLI_XDNA_AVAILABLE)
        return COLI_XDNA_PROV_HELPER_UNAVAILABLE;

    return COLI_XDNA_PROV_READY;
}

void coli_xdna_set_explicit_enabled(int on){ g_xdna_explicit = on ? 1 : 0; }
int  coli_xdna_explicit_enabled(void){ return g_xdna_explicit; }

void coli_xdna_test_set_artifact_root(const char *root){
    g_xdna_root_overridden = 1;
    if(root && *root) snprintf(g_xdna_root, sizeof g_xdna_root, "%s", root);
    else              g_xdna_root[0] = '\0';
}

int coli_xdna_test_device_opens(void){ return g_xdna_device_opens; }
int coli_xdna_test_helper_calls(void){ return g_xdna_helper_calls; }
int coli_xdna_test_userptr_wraps(void){ return g_xdna_userptr_wraps; }
int coli_xdna_test_conversions(void){ return g_xdna_conversions; }

/* ---- the candidate --------------------------------------------------------
 *
 * Gate order is the point of this function, not an accident of it. Everything
 * the engine can answer from its own state is answered first, cheapest and most
 * semantic first, so an operation that could never run costs a handful of
 * integer comparisons and never touches the filesystem, the helper or the
 * device. Nothing later can retroactively excuse an earlier refusal: every gate
 * returns, none of them merely records. */

static ColiXdnaHard coli_xdna_engine_gates(ColiXdnaFamily family,
                                           int fmt, int I, int O, int gs, int planar, int S,
                                           const ColiXdnaArtifact **row_out){
    /*  1  semantic family -- passed in by the call site, never inferred */
    if(family != COLI_XDNA_FAMILY_MOE_SHARED_GATE_UP)
        return COLI_XDNA_HARD_FAMILY_UNSUPPORTED;

    /*  2  logical M within the range some compiled bucket serves.
     *
     *     The upper bound is now the LARGEST bucket, not the only one. Below
     *     the minimum and above the largest are the same verdict to the caller
     *     -- M_OUT_OF_RANGE, decline to the current path -- but the
     *     above-the-range case is counted separately, because it is the new
     *     first-decline boundary and a later slice will want to know how often
     *     real work lands there. */
    if(S < COLI_XDNA_LOGICAL_M_MIN || S > COLI_XDNA_LOGICAL_M_MAX){
        if(S > COLI_XDNA_LOGICAL_M_MAX) g_xdna_m_above_range_declines++;
        return COLI_XDNA_HARD_M_OUT_OF_RANGE;
    }

    /*  3  K / N representable and positive */
    if(I <= 0 || O <= 0) return COLI_XDNA_HARD_SHAPE_UNSUPPORTED;

    /*  4  stored format */
    if(fmt != 4) return COLI_XDNA_HARD_FORMAT_UNSUPPORTED;

    /*  5  group size -- the converter is more permissive than the device */
    if(gs != COLI_XDNA_QUALIFIED_GROUP_SIZE)
        return COLI_XDNA_HARD_GROUP_SIZE_UNSUPPORTED;

    /*  5b in-memory byte layout.
     *
     *     fmt=4 has TWO layouts in current Colibri. The classic PAIR layout
     *     puts elements 2j and 2j+1 in byte j; the K1 PLANAR layout puts
     *     elements k and k+32 in byte k of each 64-element block, and
     *     qt_planarize() rewrites the tensor IN PLACE when the grouped planar
     *     IDOT path is opted into (IDOT_GS=1), for exactly the gs>=64 tensors
     *     this lane qualifies. The converter was qualified against the pair
     *     layout only, so planar bytes would decode to nonsense rather than
     *     fail. Decline instead. */
    if(planar)
        return COLI_XDNA_HARD_LAYOUT_UNSUPPORTED;

    /*  6  a qualified artifact for exactly this family/bucket/shape/dtypes.
     *     The bucket is the artifact's M, which is where logical M is mapped. */
    ColiXdnaRequest q;
    q.family = family;
    q.m = coli_xdna_bucket_for_logical_m(S);      /* smallest bucket holding S */
    if(q.m == 0u) return COLI_XDNA_HARD_M_OUT_OF_RANGE;   /* unreachable: gate 2 */
    q.k = (unsigned)I; q.n = (unsigned)O;
    q.in_dtype = COLI_XDNA_DT_BF16;
    q.weight_dtype = COLI_XDNA_DT_BF16;
    q.out_dtype = COLI_XDNA_DT_F32;

    const ColiXdnaArtifact *row = coli_xdna_registry_lookup(&q);
    if(!row) return COLI_XDNA_HARD_SHAPE_UNSUPPORTED;

    /*  7-9  qualification flags, artifact presence, artifact integrity. All
     *       three live behind coli_xdna_artifact_status, which fails closed.
     *       The sub-reason is carried through rather than collapsed: "this
     *       build does not ship that artifact", "the bytes are not the bytes
     *       that were qualified", and "we never qualified this one" call for
     *       different actions, and an operator who is told only
     *       ARTIFACT_NOT_QUALIFIED cannot tell a missing file from a tampered
     *       one. */
    char root_scratch[1024];
    const char *eff_root = coli_xdna_effective_root(root_scratch, sizeof root_scratch);
    switch(coli_xdna_artifact_status(&q, eff_root)){
        case COLI_XDNA_STATIC_QUALIFIED:
            break;
        case COLI_XDNA_STATIC_ARTIFACT_UNAVAILABLE:
            return COLI_XDNA_HARD_ARTIFACT_UNAVAILABLE;
        case COLI_XDNA_STATIC_ARTIFACT_INTEGRITY_FAILED:
            return COLI_XDNA_HARD_ARTIFACT_INTEGRITY_FAILED;
        case COLI_XDNA_STATIC_ARTIFACT_UNQUALIFIED:
            return COLI_XDNA_HARD_ARTIFACT_UNQUALIFIED;
        case COLI_XDNA_STATIC_REGISTRY_INVALID:
            return COLI_XDNA_HARD_REGISTRY_INVALID;
        default:
            /* Family/shape/format verdicts cannot occur here: the lookup above
             * already matched a row on the full key. */
            return COLI_XDNA_HARD_SHAPE_UNSUPPORTED;
    }

    *row_out = row;
    return COLI_XDNA_HARD_ELIGIBLE;
}

/* Grow a staging buffer to at least `need` bytes. Transient operation state:
 * reused across operations so a steady stream of the same shape allocates once,
 * but carrying no validity and no authority of any kind. */
static int coli_xdna_stage(void **base, size_t *cap, size_t need){
    if(*cap >= need) return 1;
    void *p = malloc(need);
    if(!p) return 0;
    free(*base);
    *base = p; *cap = need;
    return 1;
}

/* The shared execution core. Both modes call this, so their gate order and
 * their failure classification cannot drift apart.
 *
 * It returns a CLASS, never a handled flag: deciding what a failure means for
 * the caller belongs to the mode, not to this function. It writes y only after
 * the helper reports successful completion, and never calls matmul_qt. */
static ColiXdnaExec coli_xdna_attempt(ColiXdnaFamily family,
                                      ColiXdnaPrepared **slot,
                                      int fmt, const unsigned char *q4, const float *scale,
                                      int I, int O, int gs, int planar,
                                      float *y, const float *x, int S)
{
    g_xdna_output_valid = 0;          /* nothing is valid until completion says so */

    if(!slot || !q4 || !scale || !y || !x){
        g_xdna_last_hard = COLI_XDNA_HARD_SHAPE_UNSUPPORTED;
        return (g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED);
    }

    /* A device that would not initialise stays refused for the process. This is
     * the whole of the retry policy, and deliberately not more: the sealed A5
     * taxonomy classifies device-init failure PROCESS-scoped, and re-attempting
     * it on every operation would cost without prospect. */
    if(g_xdna_lane_health == COLI_XDNA_LANE_UNAVAILABLE){
        g_xdna_last_hard = COLI_XDNA_HARD_DEVICE_UNAVAILABLE;
        return (g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED);
    }

    const ColiXdnaArtifact *row = NULL;
    ColiXdnaHard h = coli_xdna_engine_gates(family, fmt, I, O, gs, planar, S, &row);
    if(h != COLI_XDNA_HARD_ELIGIBLE){
        g_xdna_last_hard = h;
        return (g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED);
    }

    /* 10  helper binding. Checked before the weight is prepared: preparing
     *     24 MiB for a lane that cannot execute would be pure waste. The
     *     loader verdict is sticky, so this is a branch, not a module load. */
    {
        ColiXdnaBinding b = coli_xdna_binding();
        if(b != COLI_XDNA_AVAILABLE){
            g_xdna_last_hard = (b == COLI_XDNA_ABI_INCOMPATIBLE
                                || b == COLI_XDNA_SYMBOL_INCOMPLETE)
                             ? COLI_XDNA_HARD_HELPER_ABI_INCOMPATIBLE
                             : COLI_XDNA_HARD_HELPER_UNAVAILABLE;
            return (g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED);
        }
    }

    /* 11  prepared weight. Lazy, and reused while it stays VALID for these
     *     dimensions. No eviction or cache policy here. */
    if(!*slot){
        *slot = coli_xdna_prepared_create();
        if(!*slot){
            g_xdna_last_hard = COLI_XDNA_HARD_PREPARED_INVALID;
            return (g_xdna_last_exec = COLI_XDNA_EXEC_WEIGHT_PREPARE_FAILED);
        }
    }
    ColiXdnaPrepared *prep = *slot;
    if(coli_xdna_prepared_state(prep) != COLI_XDNA_PREP_VALID
       || coli_xdna_prepared_k(prep) != (unsigned)I
       || coli_xdna_prepared_n(prep) != (unsigned)O){
        if(coli_xdna_prepare_from_fmt4(prep, fmt, q4, scale, I, O, gs) != COLI_XDNA_PREP_OK){
            g_xdna_last_hard = COLI_XDNA_HARD_PREPARED_INVALID;
            return (g_xdna_last_exec = COLI_XDNA_EXEC_WEIGHT_PREPARE_FAILED);
        }
    }
    const void *wimg = coli_xdna_prepared_image(prep);   /* NULL unless VALID */
    if(!wimg){
        g_xdna_last_hard = COLI_XDNA_HARD_PREPARED_INVALID;
        return (g_xdna_last_exec = COLI_XDNA_EXEC_WEIGHT_PREPARE_FAILED);
    }

    /* 12  alignment */
    if(!coli_xdna_pointer_alignment_ok(wimg)){
        g_xdna_last_hard = COLI_XDNA_HARD_ALIGNMENT_INVALID;
        return (g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED);
    }

    const unsigned am = row->artifact_m;

    /* 13  device / runtime / artifact runtime object, opened lazily. */
    if(!g_lane.open || g_lane.m != am || g_lane.k != (unsigned)I || g_lane.n != (unsigned)O){
        char xb[2048], ib[2048];
        char rs2[1024];
        const char *r2 = coli_xdna_effective_root(rs2, sizeof rs2);
        if(!r2
           || !coli_xdna_join(xb, sizeof xb, r2, row->xclbin_name)
           || !coli_xdna_join(ib, sizeof ib, r2, row->insts_name)){
            g_xdna_last_hard = COLI_XDNA_HARD_ARTIFACT_RUNTIME_UNAVAILABLE;
            return (g_xdna_last_exec = COLI_XDNA_EXEC_ARTIFACT_OPEN_FAILED);
        }
        /* Record the transition BEFORE tearing the old artifact down, while
         * g_lane.m still names the bucket being left. A bucket switch is the
         * expensive case (shutdown + open + forced re-wrap), so it is counted
         * rather than inferred from artifact_opens, which also fires on the
         * very first open and on a K/N change. */
        if(g_lane.open){
            int from = coli_xdna_bucket_slot(g_lane.m), to = coli_xdna_bucket_slot(am);
            if(from >= 0 && to >= 0 && from != to) g_xdna_b_switches[from][to]++;
        }
        if(g_lane.open){ coli_xdna_lane_drop_wrapper(); coli_xdna_helper_shutdown_call(); }
        g_lane.open = 0;
        g_xdna_helper_calls++; g_xdna_device_opens++;
        int rc = coli_xdna_helper_open_call(xb, ib, am, (unsigned)I, (unsigned)O);
        if(rc != COLI_XDNA_H_OK){
            if(rc == COLI_XDNA_H_E_DEVICE){
                g_xdna_lane_health = COLI_XDNA_LANE_UNAVAILABLE;   /* process-scoped */
                g_xdna_last_hard = COLI_XDNA_HARD_DEVICE_UNAVAILABLE;
                return (g_xdna_last_exec = COLI_XDNA_EXEC_DEVICE_INIT_FAILED);
            }
            /* Artifact-runtime scope. The engine already verified these bytes,
             * so this is NOT the same thing as a missing or tampered artifact
             * and must stay distinguishable from both. The lane stays healthy:
             * another shape may well open. */
            g_xdna_last_hard = COLI_XDNA_HARD_ARTIFACT_RUNTIME_UNAVAILABLE;
            return (g_xdna_last_exec = COLI_XDNA_EXEC_ARTIFACT_OPEN_FAILED);
        }
        g_lane.open = 1; g_lane.m = am; g_lane.k = (unsigned)I; g_lane.n = (unsigned)O;
        g_lane.wrapped = NULL; g_lane.wrapped_gen = 0;
        g_xdna_lane_health = COLI_XDNA_LANE_HEALTHY;
        g_xdna_artifact_opens++;
        { int sl = coli_xdna_bucket_slot(am); if(sl >= 0) g_xdna_b_artifact_opens[sl]++; }
    }

    /* 14  userptr wrap, keyed on (pointer, generation).
     *
     *     The pointer alone is not an identity. prepare_begin reuses retained
     *     capacity IN PLACE, so re-preparing a different weight yields the same
     *     address with different contents -- and the helper snapshots at wrap
     *     time via sync(BO_TO_DEVICE). Keying on the pointer alone therefore
     *     skipped the re-wrap and left the device computing against a view that
     *     no longer matched the engine image. Measured on real XDNA2 hardware,
     *     not merely reasoned about. */
    {
        unsigned long long gen = coli_xdna_prepared_generation(prep);
        if(g_lane.wrapped == wimg && g_lane.wrapped_gen == gen){
            g_xdna_wrapper_reuses++;
        } else {
            if(g_lane.wrapped == wimg) g_xdna_stale_rejects++;   /* same address, new image */
            if(g_lane.wrapped) coli_xdna_lane_drop_wrapper();
            g_xdna_helper_calls++; g_xdna_userptr_wraps++;
            if(coli_xdna_helper_wrap_call((void *)wimg,
                                          (uint64_t)coli_xdna_prepared_bytes(prep)) != COLI_XDNA_H_OK){
                g_xdna_last_hard = COLI_XDNA_HARD_WEIGHT_WRAP_UNAVAILABLE;
                return (g_xdna_last_exec = COLI_XDNA_EXEC_WEIGHT_WRAP_FAILED);
            }
            g_lane.wrapped = wimg; g_lane.wrapped_gen = gen;
        }
    }

    g_xdna_last_hard = COLI_XDNA_HARD_ELIGIBLE;   /* FULL_XDNA_HARD_ELIGIBLE */
    { int sl = coli_xdna_bucket_slot(am); if(sl >= 0) g_xdna_b_hard_eligible[sl]++; }

    /* ---- activation staging ------------------------------------------------- */
    size_t abytes = (size_t)am * (size_t)I * 2u;
    size_t obytes = (size_t)am * (size_t)O * 4u;
    if(!coli_xdna_stage((void **)&g_lane.act, &g_lane.act_cap, abytes)
       || !coli_xdna_stage((void **)&g_lane.out, &g_lane.out_cap, obytes)){
        return (g_xdna_last_exec = COLI_XDNA_EXEC_ACTIVATION_FAILED);
    }
    for(size_t i = 0; i < (size_t)S * (size_t)I; i++)
        g_lane.act[i] = coli_xdna_f2b(x[i]);
    if((unsigned)S < am){
        memset(g_lane.act + (size_t)S * (size_t)I, 0,
               ((size_t)am - (size_t)S) * (size_t)I * 2u);
        g_xdna_padded_ops++;
        /* ROWS as well as OPS: with two buckets the op count no longer says how
         * much padding was computed. An S=65 operation on M256 pads 191 rows;
         * an S=63 operation on M64 pads 1. Both are one "padded op". */
        { int sl = coli_xdna_bucket_slot(am);
          if(sl >= 0){ g_xdna_b_padded_ops[sl]++;
                       g_xdna_b_padded_rows[sl] += (long long)(am - (unsigned)S); } }
    }
    g_xdna_act_preps++;

    /* ---- blocking execution ------------------------------------------------- */
    g_xdna_helper_calls++;
    int rc = coli_xdna_helper_execute_call(g_lane.act, (uint64_t)abytes,
                                           g_lane.out, (uint64_t)obytes);
    if(rc != COLI_XDNA_H_OK){
        /* The helper may already have written real bytes into the staging
         * buffer before deciding it had failed. They stay there and are simply
         * never copied: y is untouched, and no property of those bytes can make
         * them valid, because validity is a state and this state is failure. */
        return (g_xdna_last_exec = (rc == COLI_XDNA_H_E_COMPLETION)
                                 ? COLI_XDNA_EXEC_COMPLETION_FAILED
                                 : COLI_XDNA_EXEC_EXECUTE_FAILED);
    }
    g_xdna_dispatches++; g_xdna_completions++;
    { int sl = coli_xdna_bucket_slot(am);
      if(sl >= 0){ g_xdna_b_dispatches[sl]++; g_xdna_b_completions[sl]++; } }

    memcpy(y, g_lane.out, (size_t)S * (size_t)O * 4u);   /* logical rows only */
    g_xdna_output_valid = 1;
    return (g_xdna_last_exec = COLI_XDNA_EXEC_OK);
}

/* AUTO-LIKE mode: the production seam. Any decline or failure returns 0 and the
 * caller runs its current path, which is the exact matmul_qt call that stood
 * there before this lane existed. */
int coli_xdna_try_matmul(ColiXdnaFamily family,
                         ColiXdnaPrepared **slot,
                         int fmt, const unsigned char *q4, const float *scale,
                         int I, int O, int gs, int planar,
                         float *y, const float *x, int S)
{
    /* Gate 0: PERMISSION. Without it this function is inert, which is what keeps
     * the lane from being an automatic scheduler.
     *
     * Two independent sources, and they are not interchangeable:
     *   g_xdna_explicit  the user asked for it (`coli --xdna` -> COLI_XDNA=1)
     *   g_xdna_force     an internal qualification run forced it
     *
     * Neither is discovery. Present hardware, a loadable helper and valid
     * artifacts do not set either one, so a machine that COULD run this lane
     * still does not until someone says so. Permission is also not selection:
     * every hard gate below still decides, and an unqualified operation runs
     * the current path no matter who asked. */
    if(!(g_xdna_explicit || g_xdna_force)){
        /* Clear validity here too. Declining before the core runs still ends an
         * attempt, and a stale 1 left over from an earlier success would claim
         * an XDNA result for an operation the lane never touched. */
        g_xdna_output_valid = 0;
        g_xdna_fallbacks++;
        g_xdna_last_exec = COLI_XDNA_EXEC_DECLINED;
        return 0;
    }
    if(coli_xdna_attempt(family, slot, fmt, q4, scale, I, O, gs, planar, y, x, S)
       == COLI_XDNA_EXEC_OK)
        return 1;
    g_xdna_fallbacks++;
    return 0;
}

/* EXPLICIT mode, internal and test-facing. Returns the class and implies no
 * fallback: a caller here asked for XDNA specifically and gets a classified
 * failure rather than a silent substitution. It does not consult the force
 * control, because asking explicitly IS the request. Kept as a separate entry
 * point rather than a mode flag so that no global state can ever leave the
 * production seam in a no-fallback configuration. */
ColiXdnaExec coli_xdna_test_attempt(ColiXdnaFamily family,
                                    ColiXdnaPrepared **slot,
                                    int fmt, const unsigned char *q4, const float *scale,
                                    int I, int O, int gs, int planar,
                                    float *y, const float *x, int S)
{
    return coli_xdna_attempt(family, slot, fmt, q4, scale, I, O, gs, planar, y, x, S);
}

int coli_xdna_test_output_valid(void){ return g_xdna_output_valid; }

const float *coli_xdna_test_output_staging(size_t *floats){
    if(floats) *floats = g_lane.out_cap / sizeof(float);
    return g_lane.out;
}

ColiXdnaLaneHealth coli_xdna_lane_health(void){ return g_xdna_lane_health; }

const char *coli_xdna_lane_health_text(ColiXdnaLaneHealth h){
    switch(h){
        case COLI_XDNA_LANE_UNINITIALIZED: return "UNINITIALIZED";
        case COLI_XDNA_LANE_HEALTHY:       return "HEALTHY";
        case COLI_XDNA_LANE_UNAVAILABLE:   return "UNAVAILABLE";
    }
    return "UNKNOWN";
}
void coli_xdna_execution_shutdown(void){
    coli_xdna_lane_drop_wrapper();
    if(g_lane.open) coli_xdna_helper_shutdown_call();
    g_lane.open = 0; g_lane.wrapped = NULL; g_lane.wrapped_gen = 0;
    /* Lane health is lane runtime state, so it resets with the rest of it. A
     * fresh attempt after a full teardown is a meaningful thing to allow; a
     * fresh attempt on every operation is not, which is why nothing else
     * clears it. */
    g_xdna_lane_health = COLI_XDNA_LANE_UNINITIALIZED;
    free(g_lane.act); g_lane.act = NULL; g_lane.act_cap = 0;
    free(g_lane.out); g_lane.out = NULL; g_lane.out_cap = 0;
}