Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
69.1 KB
Raw Blame
/* gliner_decide.c -- GLiNER2.5-Decide (fastino/GLiNER2.5-Decide on Hugging Face,
 * Apache-2.0) as a decision engine, in C.
 *
 * GLiNER2.5-Decide is a GLiNER2 "span" checkpoint: a DeBERTa-v3-large encoder
 * and GLiNER2's heads. A decision uses one of them, the classification head:
 * every question of a request becomes one classification task of a single
 * classify_text() call, all tasks and the text are read in one sequence by the
 * encoder, and an MLP scores each label at its [L] marker. The extraction
 * heads (span_rep, count_pred, count_embed) are never run and never loaded.
 *
 * The specification is the model's own Python: the `gliner2` package (2.0.0)
 * and the DeBERTa-v2 modeling code of transformers it builds on. Each step
 * below names the function it follows:
 *
 *   words        processing/word_splitter.WhitespaceTokenSplitter: Python's
 *                regex (URLs, e-mails, @handles, \w+(?:[-_]\w+)*, \S), each
 *                word lower-cased with str.lower(); processor._collate_batch
 *                first appends "." to a text that does not end in . ! or ?
 *   schema       inference/schema.Schema.classification and
 *                processor._transform_schema: per task
 *                ( [P] task[: prompt][ [DESCRIPTION] label: text ...] ( [L] label ... ) )
 *                tasks joined by [SEP_STRUCT], then [SEP_TEXT] and the words
 *   tokens       processor._format_input_with_mapping: every element above is
 *                tokenized on its own (tokenizer.tokenize), no [CLS] / [SEP];
 *                a marker's embedding is its first sub-token
 *   tokenizer    tokenizer.json (DebertaV2TokenizerFast): added tokens split
 *                out, Replace(\s{2,}|[\n\r\t] -> " "), NFC, right strip,
 *                Metaspace (prepend always, split), Unigram (Viterbi, unknown
 *                characters fused into one [UNK])
 *   encoder      DebertaV2Model: word embeddings and LayerNorm (no absolute
 *                positions), 24 post-norm layers of disentangled attention
 *                (content-to-content + content-to-position + position-to-
 *                content, relative positions in log buckets, shared key and
 *                query projections, the relative embeddings LayerNorm'd), GELU
 *                FFN
 *   head         runtime._extract_classification_result: classifier
 *                (Linear -> ReLU -> Linear) at each [L], temperature 1,
 *                softmax over the task's labels (a single-label task)
 *
 * A System One question becomes a task as docs/gliner_decide.md describes: the
 * task name is the question id, the prompt its instructions, the labels a
 * choice's labels, a score's levels "0".."n-1" or a noul's "yes" then "no",
 * and the option texts become label descriptions.
 *
 * Weights are read as stored (F32 in the release) and kept in f32. The
 * relative embeddings are the same for every request, so their projections
 * through each layer's query and key are computed once at load.
 *
 * Ways to run it:
 *   SERVE=1 SNAP=<dir> gliner_decide              the mux serve protocol, DECIDE only
 *   gliner_decide --model DIR --records F [--ids] a JSON array of DECIDE records;
 *                                                 one DECISION object per line
 *   gliner_decide --model DIR --tokenize F        a JSON array of strings; their ids
 *   gliner_decide --split F                       a JSON array of strings; their words
 */
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <math.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#ifdef _OPENMP
#include <omp.h>
#endif

#include "compat.h"
#include "json.h"
#include "st.h"
#include "qwen38_nfc.h"
#include "qi_gemm.h"
#include "matmul_f32.h"
#include "serve_codec.h"
#include "decide_serve.h"
#include "omp_tune.h"
#include "gliner_unicode.h"

/* ------------------------------------------------------------------ utils */

static void die(const char *fmt, ...)
{
    va_list args;
    va_start(args, fmt);
    fprintf(stderr, "[gliner_decide] ");
    vfprintf(stderr, fmt, args);
    fputc('\n', stderr);
    va_end(args);
    exit(1);
}

static void *xmalloc(size_t size)
{
    void *p = malloc(size ? size : 1);
    if (!p) die("out of memory (%zu bytes)", size);
    return p;
}

static void *xcalloc(size_t count, size_t size)
{
    void *p = calloc(count ? count : 1, size ? size : 1);
    if (!p) die("out of memory (%zu x %zu bytes)", count, size);
    return p;
}

static void *xrealloc(void *p, size_t size)
{
    void *q = realloc(p, size ? size : 1);
    if (!q) die("out of memory (%zu bytes)", size);
    return q;
}

static char *read_text(const char *path)
{
    FILE *f = fopen(path, "rb");
    if (!f) return NULL;
    if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return NULL; }
    long n = ftell(f);
    if (n < 0 || n > (1L << 30)) { fclose(f); return NULL; }
    rewind(f);
    char *b = (char *)xmalloc((size_t)n + 1);
    if (fread(b, 1, (size_t)n, f) != (size_t)n) { fclose(f); free(b); return NULL; }
    b[n] = 0;
    fclose(f);
    return b;
}

static jval *read_json(const char *dir, const char *name)
{
    char path[4096];
    snprintf(path, sizeof(path), "%s/%s", dir, name);
    char *text = read_text(path);
    if (!text) die("cannot read %s", path);
    jval *root = json_parse_checked(text);
    free(text);
    if (!root || root->t != J_OBJ) die("%s is not a JSON object", path);
    return root;
}

static double num_or(jval *o, const char *key, double fallback)
{
    jval *v = json_get(o, key);
    return v && v->t == J_NUM ? v->num : fallback;
}

static int bool_or(jval *o, const char *key, int fallback)
{
    jval *v = json_get(o, key);
    return v && v->t == J_BOOL ? v->boolean : fallback;
}

static const char *str_or(jval *o, const char *key, const char *fallback)
{
    jval *v = json_get(o, key);
    return v && v->t == J_STR ? v->str : fallback;
}

static double now_ms(void)
{
    struct timespec t;
    clock_gettime(CLOCK_MONOTONIC, &t);
    return t.tv_sec * 1e3 + t.tv_nsec / 1e6;
}

/* A growable byte string and a growable int array. */
typedef struct { char *s; size_t n, cap; } Str;
typedef struct { int *v; int n, cap; } IntVec;

static void str_put(Str *b, const char *bytes, size_t count)
{
    if (b->n + count + 1 > b->cap) {
        size_t cap = b->cap ? b->cap : 64;
        while (cap < b->n + count + 1) cap *= 2;
        b->s = (char *)xrealloc(b->s, cap);
        b->cap = cap;
    }
    memcpy(b->s + b->n, bytes, count);
    b->n += count;
    b->s[b->n] = 0;
}

static void str_cp(Str *b, uint32_t cp)
{
    char u[4];
    size_t n;
    if (cp < 0x80) { u[0] = (char)cp; n = 1; }
    else if (cp < 0x800) { u[0] = (char)(0xC0 | (cp >> 6)); u[1] = (char)(0x80 | (cp & 0x3F)); n = 2; }
    else if (cp < 0x10000) {
        u[0] = (char)(0xE0 | (cp >> 12)); u[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
        u[2] = (char)(0x80 | (cp & 0x3F)); n = 3;
    } else {
        u[0] = (char)(0xF0 | (cp >> 18)); u[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
        u[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); u[3] = (char)(0x80 | (cp & 0x3F)); n = 4;
    }
    str_put(b, u, n);
}

static void ivec_push(IntVec *a, int x)
{
    if (a->n == a->cap) {
        a->cap = a->cap ? a->cap * 2 : 64;
        a->v = (int *)xrealloc(a->v, (size_t)a->cap * sizeof(int));
    }
    a->v[a->n++] = x;
}

/* One UTF-8 code point at s[*at] (< n). Bytes that are not valid UTF-8 come
 * back one at a time as themselves; the gateway only ever sends valid UTF-8. */
static uint32_t utf8_next(const unsigned char *s, size_t n, size_t *at)
{
    size_t i = *at;
    unsigned char c = s[i];
    uint32_t cp;
    size_t w;
    if (c < 0x80) { *at = i + 1; return c; }
    if (c >= 0xC2 && c <= 0xDF) { cp = c & 0x1F; w = 2; }
    else if (c >= 0xE0 && c <= 0xEF) { cp = c & 0x0F; w = 3; }
    else if (c >= 0xF0 && c <= 0xF4) { cp = c & 0x07; w = 4; }
    else { *at = i + 1; return c; }
    if (i + w > n) { *at = i + 1; return c; }
    for (size_t k = 1; k < w; k++) {
        if ((s[i + k] & 0xC0) != 0x80) { *at = i + 1; return c; }
        cp = (cp << 6) | (s[i + k] & 0x3F);
    }
    *at = i + w;
    return cp;
}

/* ------------------------------------------------- Python's view of a text */

static int in_ranges(const GlUniRange *r, int n, uint32_t cp)
{
    int lo = 0, hi = n - 1;
    while (lo <= hi) {
        int mid = (lo + hi) / 2;
        if (cp < r[mid].lo) hi = mid - 1;
        else if (cp > r[mid].hi) lo = mid + 1;
        else return 1;
    }
    return 0;
}

#define NRANGES(t) ((int)(sizeof(t) / sizeof((t)[0])))

static int py_word(uint32_t cp)       /* \w of a str pattern */
{
    if (cp < 0x80) return (cp >= '0' && cp <= '9') || (cp >= 'a' && cp <= 'z') ||
                          (cp >= 'A' && cp <= 'Z') || cp == '_';
    return in_ranges(gl_uni_word, NRANGES(gl_uni_word), cp);
}

static int py_space(uint32_t cp)      /* \s of a str pattern: str.isspace() */
{
    return in_ranges(gl_uni_space, NRANGES(gl_uni_space), cp);
}

/* [a-z] under re.IGNORECASE: the ASCII letters plus the four code points whose
 * case folding reaches them (U+0130, U+0131, U+017F, U+212A). */
static int py_az(uint32_t cp)
{
    return (cp >= 'a' && cp <= 'z') || (cp >= 'A' && cp <= 'Z') ||
           cp == 0x130 || cp == 0x131 || cp == 0x17F || cp == 0x212A;
}

static int py_digit_ascii(uint32_t cp) { return cp >= '0' && cp <= '9'; }

/* A literal ASCII letter under re.IGNORECASE ('s' also matches U+017F). */
static int py_lit(uint32_t cp, char want)
{
    if (cp == (uint32_t)want || (want >= 'a' && want <= 'z' && cp == (uint32_t)(want - 32))) return 1;
    return want == 's' && cp == 0x17F;
}

static uint32_t py_lower1(uint32_t cp)
{
    if (cp < 0x80) return cp >= 'A' && cp <= 'Z' ? cp + 32 : cp;
    int lo = 0, hi = NRANGES(gl_uni_lower) - 1;
    while (lo <= hi) {
        int mid = (lo + hi) / 2;
        if (cp < gl_uni_lower[mid].cp) hi = mid - 1;
        else if (cp > gl_uni_lower[mid].cp) lo = mid + 1;
        else return gl_uni_lower[mid].lower;
    }
    return cp;
}

static int py_ignorable(uint32_t cp) { return in_ranges(gl_uni_ignorable, NRANGES(gl_uni_ignorable), cp); }
static int py_cased(uint32_t cp) { return in_ranges(gl_uni_cased, NRANGES(gl_uni_cased), cp); }

/* str.lower() of cps[0..n), appended to out as UTF-8. Capital sigma follows
 * Python's handle_capital_sigma (the Final_Sigma context of Unicode 3.13). */
static void py_lower(const uint32_t *cps, int n, Str *out)
{
    for (int i = 0; i < n; i++) {
        uint32_t cp = cps[i];
        if (cp == 0x130) { str_cp(out, 'i'); str_cp(out, 0x307); continue; }
        if (cp == 0x3A3) {
            int j = i - 1;
            while (j >= 0 && py_ignorable(cps[j])) j--;
            int final_sigma = j >= 0 && py_cased(cps[j]);
            if (final_sigma && i + 1 < n) {
                j = i + 1;
                while (j < n && py_ignorable(cps[j])) j++;
                final_sigma = j == n || !py_cased(cps[j]);
            }
            str_cp(out, final_sigma ? 0x3C2 : 0x3C3);
            continue;
        }
        str_cp(out, py_lower1(cp));
    }
}

/* ------------------------------------------- WhitespaceTokenSplitter */

typedef struct { char **w; int n, cap; } Words;

static void words_free(Words *w)
{
    for (int i = 0; i < w->n; i++) free(w->w[i]);
    free(w->w);
    memset(w, 0, sizeof(*w));
}

/* The pattern of processing/word_splitter.py, re.VERBOSE | re.IGNORECASE:
 *     (?:https?://[^\s]+|www\.[^\s]+)
 *   | [a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}
 *   | @[a-z0-9_]+
 *   | \w+(?:[-_]\w+)*
 *   | \S
 * tried in that order at each position, as re.finditer does; returns the end
 * of the match at p, or -1. */
/* [^\s]+ from k: the end of the run, or -1 when it is empty */
static int nonspace_run(const uint32_t *c, int n, int k)
{
    if (k >= n || py_space(c[k])) return -1;
    while (k < n && !py_space(c[k])) k++;
    return k;
}

/* "://" then [^\s]+ */
static int url_rest(const uint32_t *c, int n, int k)
{
    if (k + 3 > n || c[k] != ':' || c[k + 1] != '/' || c[k + 2] != '/') return -1;
    return nonspace_run(c, n, k + 3);
}

static int split_match(const uint32_t *c, int n, int p)
{
    /* https?://[^\s]+ (the 's' tried first, as the greedy ? does), www\.[^\s]+ */
    if (p + 4 <= n && py_lit(c[p], 'h') && py_lit(c[p + 1], 't') && py_lit(c[p + 2], 't') &&
        py_lit(c[p + 3], 'p')) {
        int k = p + 4, e = -1;
        if (k < n && py_lit(c[k], 's')) e = url_rest(c, n, k + 1);
        if (e < 0) e = url_rest(c, n, k);
        if (e >= 0) return e;
    }
    if (p + 4 <= n && py_lit(c[p], 'w') && py_lit(c[p + 1], 'w') && py_lit(c[p + 2], 'w') && c[p + 3] == '.') {
        int e = nonspace_run(c, n, p + 4);
        if (e >= 0) return e;
    }
    /* e-mail: the local part cannot contain '@', so only its longest run can be
     * followed by one; the domain backtracks to the last ".xx" that fits */
    {
        int k = p;
        while (k < n && (py_az(c[k]) || py_digit_ascii(c[k]) || c[k] == '.' || c[k] == '_' ||
                         c[k] == '%' || c[k] == '+' || c[k] == '-'))
            k++;
        if (k > p && k < n && c[k] == '@') {
            int d0 = k + 1, d1 = d0;
            while (d1 < n && (py_az(c[d1]) || py_digit_ascii(c[d1]) || c[d1] == '.' || c[d1] == '-')) d1++;
            for (int split = d1 - 1; split >= d0 + 1; split--) {
                if (c[split] != '.') continue;
                int e = split + 1;
                while (e < n && py_az(c[e])) e++;
                if (e - (split + 1) >= 2) return e;
            }
        }
    }
    /* @handle */
    if (c[p] == '@') {
        int k = p + 1;
        while (k < n && (py_az(c[k]) || py_digit_ascii(c[k]) || c[k] == '_')) k++;
        if (k > p + 1) return k;
    }
    /* \w+(?:[-_]\w+)* */
    if (py_word(c[p])) {
        int k = p + 1;
        for (;;) {
            while (k < n && py_word(c[k])) k++;
            if (k + 1 < n && (c[k] == '-' || c[k] == '_') && py_word(c[k + 1])) { k += 1; continue; }
            break;
        }
        return k;
    }
    /* \S */
    if (!py_space(c[p])) return p + 1;
    return -1;
}

/* The words of `text`, lower-cased, as WhitespaceTokenSplitter yields them. */
static void split_words(const char *text, size_t len, Words *out)
{
    memset(out, 0, sizeof(*out));
    int n = 0, cap = 64;
    uint32_t *c = (uint32_t *)xmalloc((size_t)cap * sizeof(uint32_t));
    for (size_t at = 0; at < len;) {
        uint32_t cp = utf8_next((const unsigned char *)text, len, &at);
        if (n == cap) { cap *= 2; c = (uint32_t *)xrealloc(c, (size_t)cap * sizeof(uint32_t)); }
        c[n++] = cp;
    }
    for (int p = 0; p < n;) {
        int e = split_match(c, n, p);
        if (e < 0) { p++; continue; }
        Str w = {0};
        py_lower(c + p, e - p, &w);
        if (!w.s) str_put(&w, "", 0);
        if (out->n == out->cap) {
            out->cap = out->cap ? out->cap * 2 : 64;
            out->w = (char **)xrealloc(out->w, (size_t)out->cap * sizeof(char *));
        }
        out->w[out->n++] = w.s;
        p = e;
    }
    free(c);
}

/* -------------------------------------------------------------- tokenizer */

typedef struct { char *s; int len, id, normalized; } AddedTok;

typedef struct {
    int n;                       /* Unigram pieces */
    char **piece;
    int *plen;
    double *score;
    int *slot;                   /* open addressing over the pieces, -1 empty */
    uint64_t mask;
    int max_piece;               /* longest piece, in bytes */
    double unk_score;            /* min score - 10 (Unigram's K_UNK_PENALTY) */
    int unk_id;
    AddedTok *added;
    int n_added;
} GlTok;

static uint64_t fnv1a(const char *s, int n)
{
    uint64_t h = 1469598103934665603ULL;
    for (int i = 0; i < n; i++) { h ^= (unsigned char)s[i]; h *= 1099511628211ULL; }
    return h;
}

static int piece_find(const GlTok *T, const char *s, int n)
{
    uint64_t h = fnv1a(s, n) & T->mask;
    for (;;) {
        int id = T->slot[h];
        if (id < 0) return -1;
        if (T->plen[id] == n && !memcmp(T->piece[id], s, (size_t)n)) return id;
        h = (h + 1) & T->mask;
    }
}

/* White_Space, which both the normalizer's \s (Oniguruma) and its Strip
 * (Rust's char::is_whitespace) mean. */
static int uni_white(uint32_t cp)
{
    return (cp >= 0x9 && cp <= 0xD) || cp == 0x20 || cp == 0x85 || cp == 0xA0 || cp == 0x1680 ||
           (cp >= 0x2000 && cp <= 0x200A) || cp == 0x2028 || cp == 0x2029 || cp == 0x202F ||
           cp == 0x205F || cp == 0x3000;
}

static void load_tokenizer(GlTok *T, const char *dir)
{
    memset(T, 0, sizeof(*T));
    jval *tj = read_json(dir, "tokenizer.json");
    jval *model = json_get(tj, "model");
    if (!model || strcmp(str_or(model, "type", ""), "Unigram"))
        die("tokenizer.json: only a Unigram model is supported (this is %s)",
            model ? str_or(model, "type", "unknown") : "missing");
    if (bool_or(model, "byte_fallback", 0)) die("tokenizer.json: byte_fallback is not supported");
    /* the normalizer of the release: Replace(\s{2,}|[\n\r\t] -> " "), NFC, Strip(right) */
    jval *norm = json_get(tj, "normalizer"), *seq = norm ? json_get(norm, "normalizers") : NULL;
    int ok = norm && !strcmp(str_or(norm, "type", ""), "Sequence") && seq && seq->t == J_ARR && seq->len == 3;
    if (ok) {
        jval *rep = seq->kids[0], *pat = json_get(rep, "pattern");
        ok = !strcmp(str_or(rep, "type", ""), "Replace") && pat &&
             !strcmp(str_or(pat, "Regex", ""), "\\s{2,}|[\\n\\r\\t]") &&
             !strcmp(str_or(rep, "content", ""), " ") &&
             !strcmp(str_or(seq->kids[1], "type", ""), "NFC") &&
             !strcmp(str_or(seq->kids[2], "type", ""), "Strip") &&
             !bool_or(seq->kids[2], "strip_left", 1) && bool_or(seq->kids[2], "strip_right", 0);
    }
    if (!ok) die("tokenizer.json: expected the normalizer Sequence[Replace(\\s{2,}|[\\n\\r\\t] -> \" \"), "
                 "NFC, Strip(right)] of the DeBERTa-v3 tokenizer");
    jval *pre = json_get(tj, "pre_tokenizer");
    if (pre && !strcmp(str_or(pre, "type", ""), "Sequence")) {
        jval *ps = json_get(pre, "pretokenizers");
        pre = ps && ps->t == J_ARR && ps->len == 1 ? ps->kids[0] : NULL;
    }
    if (!pre || strcmp(str_or(pre, "type", ""), "Metaspace") || strcmp(str_or(pre, "replacement", ""), "\xE2\x96\x81") ||
        strcmp(str_or(pre, "prepend_scheme", "always"), "always") || !bool_or(pre, "split", 1))
        die("tokenizer.json: expected a Metaspace pre-tokenizer (replacement U+2581, prepend always, split)");
    jval *vocab = json_get(model, "vocab");
    if (!vocab || vocab->t != J_ARR || vocab->len < 1) die("tokenizer.json: Unigram vocab missing");
    T->n = vocab->len;
    T->piece = (char **)xcalloc((size_t)T->n, sizeof(char *));
    T->plen = (int *)xcalloc((size_t)T->n, sizeof(int));
    T->score = (double *)xcalloc((size_t)T->n, sizeof(double));
    uint64_t size = 1;
    while (size < (uint64_t)T->n * 2) size <<= 1;
    T->mask = size - 1;
    T->slot = (int *)xmalloc((size_t)size * sizeof(int));
    for (uint64_t i = 0; i < size; i++) T->slot[i] = -1;
    double min_score = INFINITY;
    for (int i = 0; i < T->n; i++) {
        jval *e = vocab->kids[i];
        if (!e || e->t != J_ARR || e->len != 2 || e->kids[0]->t != J_STR || e->kids[1]->t != J_NUM)
            die("tokenizer.json: vocab entry %d is not [piece, score]", i);
        T->piece[i] = strdup(e->kids[0]->str);
        T->plen[i] = (int)strlen(T->piece[i]);
        T->score[i] = e->kids[1]->num;
        if (T->score[i] < min_score) min_score = T->score[i];
        if (T->plen[i] > T->max_piece) T->max_piece = T->plen[i];
        /* a piece listed twice keeps its last id, as the reference's map does */
        uint64_t h = fnv1a(T->piece[i], T->plen[i]) & T->mask;
        for (;;) {
            int id = T->slot[h];
            if (id < 0 || (T->plen[id] == T->plen[i] && !memcmp(T->piece[id], T->piece[i], (size_t)T->plen[i]))) {
                T->slot[h] = i;
                break;
            }
            h = (h + 1) & T->mask;
        }
    }
    T->unk_score = min_score - 10.0;
    jval *unk = json_get(model, "unk_id");
    T->unk_id = unk && unk->t == J_NUM ? (int)unk->num : -1;
    if (T->unk_id < 0 || T->unk_id >= T->n) die("tokenizer.json: the Unigram model needs an unk_id");
    jval *added = json_get(tj, "added_tokens");
    T->added = (AddedTok *)xcalloc((size_t)(added && added->t == J_ARR ? added->len : 0) + 1, sizeof(AddedTok));
    for (int i = 0; added && added->t == J_ARR && i < added->len; i++) {
        jval *a = added->kids[i];
        const char *content = str_or(a, "content", NULL);
        jval *id = json_get(a, "id");
        if (!content || !content[0] || !id || id->t != J_NUM) continue;
        if (bool_or(a, "lstrip", 0) || bool_or(a, "rstrip", 0) || bool_or(a, "single_word", 0))
            die("tokenizer.json: added token %s strips whitespace, which this engine does not implement", content);
        AddedTok *t = &T->added[T->n_added++];
        t->s = strdup(content);
        t->len = (int)strlen(content);
        t->id = (int)id->num;
        t->normalized = bool_or(a, "normalized", 0);
    }
    json_free(tj);
}

/* The longest added token of the given kind starting at s[p], or -1. */
static int added_at(const GlTok *T, const char *s, size_t n, size_t p, int normalized)
{
    int best = -1;
    for (int k = 0; k < T->n_added; k++) {
        const AddedTok *a = &T->added[k];
        if (a->normalized != normalized || (size_t)a->len > n - p) continue;
        if (!memcmp(s + p, a->s, (size_t)a->len) && (best < 0 || a->len > T->added[best].len)) best = k;
    }
    return best;
}

/* Unigram.encode_optimized on one pre-token: the best-scoring segmentation,
 * ties kept by the first candidate (shorter piece, earlier start); a
 * character no single piece covers costs unk_score, and adjacent unknowns
 * come out as one [UNK]. */
static void unigram(const GlTok *T, const char *s, int n, IntVec *out)
{
    if (n <= 0) return;
    double *best = (double *)xmalloc((size_t)(n + 1) * sizeof(double));
    int *from = (int *)xmalloc((size_t)(n + 1) * sizeof(int));
    int *id = (int *)xmalloc((size_t)(n + 1) * sizeof(int));
    for (int i = 0; i <= n; i++) { from[i] = -1; best[i] = 0; id[i] = -1; }
    from[0] = 0;
    for (int start = 0; start < n;) {
        size_t at = (size_t)start;
        utf8_next((const unsigned char *)s, (size_t)n, &at);
        int mblen = (int)at - start;
        double here = best[start];
        int single = 0;
        int limit = n - start < T->max_piece ? n - start : T->max_piece;
        for (int len = 1; len <= limit; len++) {
            int pid = piece_find(T, s + start, len);
            if (pid < 0) continue;
            int key = start + len;
            double cand = here + T->score[pid];
            if (from[key] < 0 || cand > best[key]) { best[key] = cand; from[key] = start; id[key] = pid; }
            if (len == mblen) single = 1;
        }
        if (!single) {
            int key = start + mblen;
            double cand = here + T->unk_score;
            if (from[key] < 0 || cand > best[key]) { best[key] = cand; from[key] = start; id[key] = T->unk_id; }
        }
        start += mblen;
    }
    /* backtrack, fusing runs of [UNK] */
    int count = 0;
    int *rev = (int *)xmalloc((size_t)(n + 1) * sizeof(int));
    for (int end = n; end > 0;) {
        int start = from[end];
        if (start < 0 || start >= end) die("unigram: broken lattice");
        if (id[end] == T->unk_id && count > 0 && rev[count - 1] == T->unk_id) { end = start; continue; }
        rev[count++] = id[end];
        end = start;
    }
    for (int i = count - 1; i >= 0; i--) ivec_push(out, rev[i]);
    free(rev); free(best); free(from); free(id);
}

/* Metaspace (prepend always, split) and Unigram on one normalized text. */
static void metaspace_unigram(const GlTok *T, const char *s, size_t n, IntVec *out)
{
    if (!n) return;
    Str m = {0};
    if (!(n >= 3 && (unsigned char)s[0] == 0xE2 && (unsigned char)s[1] == 0x96 && (unsigned char)s[2] == 0x81) &&
        s[0] != ' ')
        str_put(&m, "\xE2\x96\x81", 3);
    for (size_t i = 0; i < n; i++) {
        if (s[i] == ' ') str_put(&m, "\xE2\x96\x81", 3);
        else str_put(&m, s + i, 1);
    }
    /* split before every U+2581, the delimiter kept with what follows it */
    size_t start = 0;
    for (size_t i = 3; i + 3 <= m.n; i++) {
        if ((unsigned char)m.s[i] == 0xE2 && (unsigned char)m.s[i + 1] == 0x96 && (unsigned char)m.s[i + 2] == 0x81) {
            unigram(T, m.s + start, (int)(i - start), out);
            start = i;
            i += 2;
        }
    }
    unigram(T, m.s + start, (int)(m.n - start), out);
    free(m.s);
}

/* Replace(\s{2,}|[\n\r\t] -> " "), NFC, Strip(right); then the normalized
 * added tokens, then Metaspace + Unigram on what is left. */
static void encode_segment(const GlTok *T, const char *s, size_t n, IntVec *out)
{
    Str r = {0};
    int ascii = 1;
    for (size_t at = 0; at < n;) {
        size_t here = at;
        uint32_t cp = utf8_next((const unsigned char *)s, n, &at);
        if (cp >= 0x80) ascii = 0;
        if (!uni_white(cp)) { str_put(&r, s + here, at - here); continue; }
        size_t run_end = at;
        int run = 1;
        while (run_end < n) {
            size_t peek = run_end;
            uint32_t next = utf8_next((const unsigned char *)s, n, &peek);
            if (!uni_white(next)) break;
            run++;
            run_end = peek;
        }
        if (run >= 2) { str_put(&r, " ", 1); at = run_end; }
        else if (cp == '\n' || cp == '\r' || cp == '\t') str_put(&r, " ", 1);
        else str_put(&r, s + here, at - here);
    }
    char *norm = r.s ? r.s : NULL;
    size_t norm_len = r.n;
    if (!ascii && norm_len) {
        char *nfc = NULL;
        size_t nfc_len = 0;
        if (q38_nfc_normalize(r.s, r.n, &nfc, &nfc_len) != 0) die("NFC normalization failed");
        free(r.s);
        norm = nfc;
        norm_len = nfc_len;
    }
    /* strip trailing White_Space */
    while (norm_len > 0) {
        size_t k = norm_len - 1;
        while (k > 0 && ((unsigned char)norm[k] & 0xC0) == 0x80) k--;
        size_t at = k;
        uint32_t cp = utf8_next((const unsigned char *)norm, norm_len, &at);
        if (at != norm_len || !uni_white(cp)) break;
        norm_len = k;
    }
    size_t seg = 0;
    for (size_t p = 0; p < norm_len;) {
        int a = added_at(T, norm, norm_len, p, 1);
        if (a < 0) { p++; continue; }
        metaspace_unigram(T, norm + seg, p - seg, out);
        ivec_push(out, T->added[a].id);
        p += (size_t)T->added[a].len;
        seg = p;
    }
    metaspace_unigram(T, norm + seg, norm_len - seg, out);
    free(norm);
}

/* tokenizer.tokenize(text) as ids: the added tokens split out of the raw text
 * (leftmost, longest), every other segment normalized and encoded. */
static void gl_encode(const GlTok *T, const char *s, size_t n, IntVec *out)
{
    size_t seg = 0;
    for (size_t p = 0; p < n;) {
        int a = added_at(T, s, n, p, 0);
        if (a < 0) { p++; continue; }
        encode_segment(T, s + seg, p - seg, out);
        ivec_push(out, T->added[a].id);
        p += (size_t)T->added[a].len;
        seg = p;
    }
    encode_segment(T, s + seg, n - seg, out);
}

static int added_id(const GlTok *T, const char *content)
{
    for (int k = 0; k < T->n_added; k++)
        if (!strcmp(T->added[k].s, content)) return T->added[k].id;
    return -1;
}

/* ------------------------------------------------------------------ model */

typedef struct {
    QiMat wqkv; float *bqkv;     /* query_proj, key_proj, value_proj stacked: [3d][d] */
    QiMat wo; float *bo;          /* attention.output.dense */
    float *ln1_w, *ln1_b;         /* attention.output.LayerNorm */
    QiMat wi; float *bi;          /* intermediate.dense */
    QiMat wo2; float *bo2;        /* output.dense */
    float *ln2_w, *ln2_b;         /* output.LayerNorm */
    float *pos_q, *pos_k;         /* LayerNorm(rel_embeddings) through query_proj / key_proj: [2*span][d] */
} GlLayer;

typedef struct {
    int d, layers, heads, hd, inter, vocab;
    int span;                     /* pos_ebd_size: relative positions are clamped to [-span, span) */
    int bucket_size, max_position;  /* make_log_bucket_position's arguments */
    int c2p, p2c;                 /* which disentangled terms pos_att_type enables */
    float eps;
    float *tok_emb, *emb_ln_w, *emb_ln_b;
    GlLayer *L;
    QiMat cls1; float *cls1_b;    /* classifier.0: [2d][d] */
    float *cls2_w; float cls2_b;  /* classifier.2: [1][2d] */
    int cls_hidden;
    int max_len;                  /* tokens per request sequence (COLI_GLINER_MAX_LEN) */
    GlTok tok;
    int id_p, id_l, id_sep_struct, id_sep_text;
    char model_name[256];
} Gl;

static float *load_f32(shards *S, const char *name, int64_t expect)
{
    st_tensor *t = st_find(S, name);
    if (!t) die("model.safetensors has no tensor %s", name);
    if (expect >= 0 && t->numel != expect)
        die("%s has %lld elements, expected %lld", name, (long long)t->numel, (long long)expect);
    float *out = (float *)xmalloc((size_t)t->numel * sizeof(float));
    st_read_f32(S, name, out, 0);
    return out;
}

static void layernorm(float *out, const float *x, const float *w, const float *b, int rows, int d, float eps)
{
    #pragma omp parallel for schedule(static)
    for (int r = 0; r < rows; r++) {
        const float *xr = x + (size_t)r * d;
        float *o = out + (size_t)r * d;
        double mean = 0, var = 0;
        for (int i = 0; i < d; i++) mean += xr[i];
        mean /= d;
        for (int i = 0; i < d; i++) { double c = xr[i] - mean; var += c * c; }
        var /= d;
        float rstd = (float)(1.0 / sqrt(var + eps));
        for (int i = 0; i < d; i++) {
            float v = (float)(xr[i] - mean) * rstd * w[i];
            o[i] = b ? v + b[i] : v;
        }
    }
}

static int pos_att_has(jval *types, const char *name)
{
    if (!types) return 0;
    if (types->t == J_STR) return strstr(types->str, name) != NULL;   /* "p2c|c2p" */
    for (int i = 0; types->t == J_ARR && i < types->len; i++)
        if (types->kids[i]->t == J_STR && !strcmp(types->kids[i]->str, name)) return 1;
    return 0;
}

static void load_model(Gl *M, const char *dir)
{
    memset(M, 0, sizeof(*M));
    /* config.json: the GLiNER2 extractor */
    jval *cfg = read_json(dir, "config.json");
    if (strcmp(str_or(cfg, "model_type", ""), "extractor"))
        die("config.json: model_type must be extractor (a GLiNER2 checkpoint)");
    const char *arch = str_or(cfg, "architecture", "span");
    if (strcmp(arch, "span"))
        die("config.json: architecture %s is not supported (only GLiNER2's span architecture)", arch);
    const char *pooling = str_or(cfg, "token_pooling", "first");
    if (strcmp(pooling, "first"))
        die("config.json: token_pooling %s is not supported (only first)", pooling);
    snprintf(M->model_name, sizeof(M->model_name), "%s", str_or(cfg, "model_name", "gliner2"));
    json_free(cfg);

    /* encoder_config/config.json: DeBERTa-v2/v3 */
    jval *ec = read_json(dir, "encoder_config/config.json");
    if (strcmp(str_or(ec, "model_type", ""), "deberta-v2"))
        die("encoder_config/config.json: model_type must be deberta-v2");
    M->d = (int)num_or(ec, "hidden_size", 0);
    M->layers = (int)num_or(ec, "num_hidden_layers", 0);
    M->heads = (int)num_or(ec, "num_attention_heads", 0);
    M->inter = (int)num_or(ec, "intermediate_size", 0);
    M->vocab = (int)num_or(ec, "vocab_size", 0);
    M->eps = (float)num_or(ec, "layer_norm_eps", 1e-7);
    if (M->d < 1 || M->layers < 1 || M->heads < 1 || M->d % M->heads || M->inter < 1 || M->vocab < 1 ||
        M->d > 16384 || M->layers > 256 || M->inter > 1 << 18)
        die("encoder_config/config.json: invalid geometry");
    M->hd = M->d / M->heads;
    if (json_get(ec, "attention_head_size") && (int)num_or(ec, "attention_head_size", 0) != M->hd)
        die("encoder_config/config.json: attention_head_size other than hidden/heads is not supported");
    if (strcmp(str_or(ec, "hidden_act", "gelu"), "gelu"))
        die("encoder_config/config.json: hidden_act %s is not supported (only gelu)", str_or(ec, "hidden_act", ""));
    if (bool_or(ec, "position_biased_input", 1))
        die("encoder_config/config.json: absolute position embeddings (position_biased_input) are not supported");
    if (num_or(ec, "type_vocab_size", 0) > 0) die("encoder_config/config.json: token type embeddings are not supported");
    if (json_get(ec, "embedding_size") && (int)num_or(ec, "embedding_size", M->d) != M->d)
        die("encoder_config/config.json: an embedding_size other than hidden_size is not supported");
    if (num_or(ec, "conv_kernel_size", 0) > 0) die("encoder_config/config.json: the convolution layer is not supported");
    if (!bool_or(ec, "relative_attention", 0)) die("encoder_config/config.json: relative_attention must be true");
    if (!bool_or(ec, "share_att_key", 0)) die("encoder_config/config.json: share_att_key must be true");
    jval *types = json_get(ec, "pos_att_type");
    M->c2p = pos_att_has(types, "c2p");
    M->p2c = pos_att_has(types, "p2c");
    int max_rel = (int)num_or(ec, "max_relative_positions", -1);
    if (max_rel < 1) max_rel = (int)num_or(ec, "max_position_embeddings", 512);
    int buckets = (int)num_or(ec, "position_buckets", -1);
    M->span = buckets > 0 ? buckets : max_rel;
    M->bucket_size = buckets;
    M->max_position = max_rel;
    if (M->span < 1 || M->span > 1 << 16) die("encoder_config/config.json: invalid relative span");
    const char *norm_rel = str_or(ec, "norm_rel_ebd", "none");
    int rel_ln = strstr(norm_rel, "layer_norm") != NULL;
    json_free(ec);

    /* tokenizer */
    load_tokenizer(&M->tok, dir);
    M->id_p = added_id(&M->tok, "[P]");
    M->id_l = added_id(&M->tok, "[L]");
    M->id_sep_struct = added_id(&M->tok, "[SEP_STRUCT]");
    M->id_sep_text = added_id(&M->tok, "[SEP_TEXT]");
    if (M->id_p < 0 || M->id_l < 0 || M->id_sep_struct < 0 || M->id_sep_text < 0 || added_id(&M->tok, "[DESCRIPTION]") < 0)
        die("tokenizer.json: GLiNER2's special tokens ([P], [L], [SEP_STRUCT], [SEP_TEXT], [DESCRIPTION]) "
            "are not added tokens");

    /* weights: the encoder and the classifier; span_rep, count_pred and
     * count_embed serve extraction and are not read */
    shards S;
    st_init(&S, dir);
    int d = M->d, I = M->inter;
    st_tensor *emb = st_find(&S, "encoder.embeddings.word_embeddings.weight");
    if (!emb || emb->rank != 2 || emb->shape[1] != d) die("encoder.embeddings.word_embeddings.weight: expected [vocab, %d]", d);
    M->vocab = (int)emb->shape[0];
    for (int k = 0; k < M->tok.n_added; k++)
        if (M->tok.added[k].id < 0 || M->tok.added[k].id >= M->vocab) die("tokenizer.json: added token id outside the embeddings");
    if (M->tok.n > M->vocab) die("tokenizer.json: %d pieces for %d embedding rows", M->tok.n, M->vocab);
    M->tok_emb = load_f32(&S, "encoder.embeddings.word_embeddings.weight", (int64_t)M->vocab * d);
    M->emb_ln_w = load_f32(&S, "encoder.embeddings.LayerNorm.weight", d);
    M->emb_ln_b = load_f32(&S, "encoder.embeddings.LayerNorm.bias", d);
    float *rel = load_f32(&S, "encoder.encoder.rel_embeddings.weight", (int64_t)2 * M->span * d);
    if (rel_ln) {
        float *w = load_f32(&S, "encoder.encoder.LayerNorm.weight", d);
        float *b = load_f32(&S, "encoder.encoder.LayerNorm.bias", d);
        layernorm(rel, rel, w, b, 2 * M->span, d, M->eps);
        free(w); free(b);
    }
    M->L = (GlLayer *)xcalloc((size_t)M->layers, sizeof(GlLayer));
    char n[256];
    for (int l = 0; l < M->layers; l++) {
        GlLayer *E = &M->L[l];
#define NAME(fmt) (snprintf(n, sizeof(n), "encoder.encoder.layer.%d." fmt, l), n)
        float *wqkv = (float *)xmalloc((size_t)3 * d * d * sizeof(float));
        E->bqkv = (float *)xmalloc((size_t)3 * d * sizeof(float));
        const char *parts[3] = {"query_proj", "key_proj", "value_proj"};
        for (int part = 0; part < 3; part++) {
            char wname[256], bname[256];
            snprintf(wname, sizeof(wname), "encoder.encoder.layer.%d.attention.self.%s.weight", l, parts[part]);
            snprintf(bname, sizeof(bname), "encoder.encoder.layer.%d.attention.self.%s.bias", l, parts[part]);
            float *w = load_f32(&S, wname, (int64_t)d * d), *b = load_f32(&S, bname, d);
            memcpy(wqkv + (size_t)part * d * d, w, (size_t)d * d * sizeof(float));
            memcpy(E->bqkv + (size_t)part * d, b, (size_t)d * sizeof(float));
            free(w); free(b);
        }
        E->wqkv = (QiMat){QI_F32, 3 * d, d, wqkv, NULL, 0};
        E->wo = (QiMat){QI_F32, d, d, load_f32(&S, NAME("attention.output.dense.weight"), (int64_t)d * d), NULL, 0};
        E->bo = load_f32(&S, NAME("attention.output.dense.bias"), d);
        E->ln1_w = load_f32(&S, NAME("attention.output.LayerNorm.weight"), d);
        E->ln1_b = load_f32(&S, NAME("attention.output.LayerNorm.bias"), d);
        E->wi = (QiMat){QI_F32, I, d, load_f32(&S, NAME("intermediate.dense.weight"), (int64_t)I * d), NULL, 0};
        E->bi = load_f32(&S, NAME("intermediate.dense.bias"), I);
        E->wo2 = (QiMat){QI_F32, d, I, load_f32(&S, NAME("output.dense.weight"), (int64_t)d * I), NULL, 0};
        E->bo2 = load_f32(&S, NAME("output.dense.bias"), d);
        E->ln2_w = load_f32(&S, NAME("output.LayerNorm.weight"), d);
        E->ln2_b = load_f32(&S, NAME("output.LayerNorm.bias"), d);
#undef NAME
        /* DisentangledSelfAttention.disentangled_attention_bias with
         * share_att_key: pos_query_layer = query_proj(rel), pos_key_layer =
         * key_proj(rel). The same for every request: computed here, once. */
        QiMat wq = {QI_F32, d, d, wqkv, NULL, 0}, wk = {QI_F32, d, d, wqkv + (size_t)d * d, NULL, 0};
        E->pos_q = (float *)xmalloc((size_t)2 * M->span * d * sizeof(float));
        E->pos_k = (float *)xmalloc((size_t)2 * M->span * d * sizeof(float));
        qi_gemm(E->pos_q, rel, 2 * M->span, &wq, E->bqkv);
        qi_gemm(E->pos_k, rel, 2 * M->span, &wk, E->bqkv + d);
    }
    free(rel);
    st_tensor *c0 = st_find(&S, "classifier.0.weight");
    if (!c0 || c0->rank != 2 || c0->shape[1] != d) die("classifier.0.weight: expected [hidden, %d]", d);
    M->cls_hidden = (int)c0->shape[0];
    M->cls1 = (QiMat){QI_F32, M->cls_hidden, d, load_f32(&S, "classifier.0.weight", (int64_t)M->cls_hidden * d), NULL, 0};
    M->cls1_b = load_f32(&S, "classifier.0.bias", M->cls_hidden);
    M->cls2_w = load_f32(&S, "classifier.2.weight", M->cls_hidden);
    float *b2 = load_f32(&S, "classifier.2.bias", 1);
    M->cls2_b = b2[0];
    free(b2);
    st_destroy(&S);

    M->max_len = 4096;
    const char *env = getenv("COLI_GLINER_MAX_LEN");
    if (env && atoi(env) > 0) M->max_len = atoi(env);
}

/* ------------------------------------------------------------- the forward */

/* make_log_bucket_position, in float32 like the reference (checked against
 * torch for every |relative position| up to 200000). */
static int log_bucket(int rel, int bucket_size, int max_position)
{
    int mid = bucket_size / 2;
    int a = rel < 0 ? -rel : rel;
    if (a <= mid) return rel;
    float lp = ceilf(logf((float)a / (float)mid) / logf((float)(max_position - 1) / (float)mid) *
                     (float)(mid - 1)) + (float)mid;
    return rel < 0 ? -(int)lp : (int)lp;
}

static inline float gelu(float x) { return 0.5f * x * (1.0f + erff(x * 0.70710678118654752f)); }

#ifdef COLI_VULKAN
#include "decide_vk.h"         /* COLI_VULKAN=1: the forward, or its matrices, on a Vulkan device */
#endif

/* Y = X W^T + bias: the encoder's and the classifier's matrices */
static void gemm(float *Y, const float *X, int M, const QiMat *W, const float *bias)
{
#ifdef COLI_VULKAN
    if (dvk_gemm(Y, X, M, W, bias)) return;   /* matrix by matrix (COLI_VK_CHAIN=0) */
#endif
    qi_gemm(Y, X, M, W, bias);
}

static void add_rows(float *x, const float *y, size_t n)
{
    #pragma omp parallel for schedule(static)
    for (size_t i = 0; i < n; i++) x[i] += y[i];
}

/* DisentangledSelfAttention over one sequence of T rows. qkv rows are
 * [q | k | v], heads side by side. With delta = i - j and
 * r = clamp(bucket(delta) + span, 0, 2 span - 1), the score of key j for
 * query i is
 *     (q_i . k_j + q_i . pos_k[r] + k_j . pos_q[r]) / sqrt(hd * (1 + c2p + p2c))
 * (c2p is the second term, p2c the third: the reference gathers p2c at
 * -bucket(j - i), which is the same r because the bucket is odd). */
typedef struct {
    float *p2c;                  /* [H][T][2 span]: k_j . pos_q[r], every head */
    float *vt;                   /* [H][hd][T]: the values, transposed per head */
    int *ridx;                   /* [2T - 1]: r for every delta = i - j */
} AttnScratch;

/* Query rows per task of the attention: a task is one head and ATT_BLOCK rows. */
#define ATT_BLOCK 16

/* DisentangledSelfAttention over one sequence of T rows. qkv rows are
 * [q | k | v], heads side by side. With delta = i - j and
 * r = clamp(bucket(delta) + span, 0, 2 span - 1), the score of key j for
 * query i is
 *     (q_i . k_j + q_i . pos_k[r] + k_j . pos_q[r]) / sqrt(hd * (1 + c2p + p2c))
 * c2p is the second term and p2c the third: the reference gathers p2c at
 * -bucket(j - i), which is the same r because the bucket is odd.
 *
 * Two parallel loops per layer, whatever T: the first projects every key on
 * pos_q and transposes the values, the second runs (head, block of query
 * rows) tasks, each with its own products (qi_gemm on one thread). A loop of
 * small parallel regions per head would be quicker on an idle machine and
 * collapses on a busy one, where every region waits for its slowest thread. */
static void attention(const Gl *M, const GlLayer *E, const float *qkv, float *out, int T, AttnScratch *A)
{
    int d = M->d, H = M->heads, hd = M->hd, R = 2 * M->span;
    int sf = 1 + M->c2p + M->p2c;
    const float inv = 1.0f / sqrtf((float)(hd * sf));
    int *ridx = A->ridx;
    for (int delta = -(T - 1); delta <= T - 1; delta++) {
        int b = M->bucket_size > 0 && M->max_position > 0 ? log_bucket(delta, M->bucket_size, M->max_position) : delta;
        int r = b + M->span;
        ridx[delta + T - 1] = r < 0 ? 0 : r > R - 1 ? R - 1 : r;
    }
    int blocks = (T + ATT_BLOCK - 1) / ATT_BLOCK;
    #pragma omp parallel for schedule(dynamic, 1)
    for (int task = 0; task < H * blocks; task++) {
        int h = task / blocks, j0 = (task % blocks) * ATT_BLOCK;
        int n = T - j0 < ATT_BLOCK ? T - j0 : ATT_BLOCK;
        if (M->p2c) {
            QiMat Pq = {QI_F32, R, hd, E->pos_q + (size_t)h * hd, NULL, d};
            qi_gemm_ld(A->p2c + ((size_t)h * T + j0) * R, R, qkv + (size_t)j0 * 3 * d + d + (size_t)h * hd,
                       3 * d, n, &Pq, NULL);
        }
        float *vt = A->vt + (size_t)h * hd * T;
        for (int j = j0; j < j0 + n; j++) {
            const float *v = qkv + (size_t)j * 3 * d + 2 * d + (size_t)h * hd;
            for (int t = 0; t < hd; t++) vt[(size_t)t * T + j] = v[t];
        }
    }
    #pragma omp parallel
    {
        float *score = (float *)xmalloc((size_t)ATT_BLOCK * T * sizeof(float));
        float *c2p = (float *)xmalloc((size_t)ATT_BLOCK * R * sizeof(float));
        #pragma omp for schedule(dynamic, 1)
        for (int task = 0; task < H * blocks; task++) {
            int h = task / blocks, i0 = (task % blocks) * ATT_BLOCK;
            int n = T - i0 < ATT_BLOCK ? T - i0 : ATT_BLOCK;
            const float *q = qkv + (size_t)i0 * 3 * d + (size_t)h * hd;
            QiMat Kh = {QI_F32, T, hd, qkv + d + (size_t)h * hd, NULL, 3 * d};
            qi_gemm_ld(score, T, q, 3 * d, n, &Kh, NULL);
            if (M->c2p) {
                QiMat Pk = {QI_F32, R, hd, E->pos_k + (size_t)h * hd, NULL, d};
                qi_gemm_ld(c2p, R, q, 3 * d, n, &Pk, NULL);
            }
            const float *p2c = A->p2c + (size_t)h * T * R;
            for (int ii = 0; ii < n; ii++) {
                int i = i0 + ii;
                float *s = score + (size_t)ii * T;
                const float *c = c2p + (size_t)ii * R;
                const int *ri = ridx + i + T - 1;            /* ri[-j] is r for (i, j) */
                float mx = -INFINITY;
                for (int j = 0; j < T; j++) {
                    int r = ri[-j];
                    float x = s[j];
                    if (M->c2p) x += c[r];
                    if (M->p2c) x += p2c[(size_t)j * R + r];
                    x *= inv;
                    s[j] = x;
                    if (x > mx) mx = x;
                }
                double sum = 0;
                for (int j = 0; j < T; j++) { float e = expf(s[j] - mx); s[j] = e; sum += e; }
                float norm = (float)(1.0 / sum);
                for (int j = 0; j < T; j++) s[j] *= norm;
            }
            QiMat Vt = {QI_F32, hd, T, A->vt + (size_t)h * hd * T, NULL, 0};
            qi_gemm_ld(out + (size_t)i0 * d + (size_t)h * hd, d, score, T, n, &Vt, NULL);
        }
        free(score); free(c2p);
    }
}

/* The classifier (Linear -> ReLU -> Linear(., 1)) on the nm marker rows m: logits[nm]. */
static void classify(const Gl *M, float *m, int nm, double *logits)
{
    float *m1 = (float *)xmalloc((size_t)(nm > 0 ? nm : 1) * M->cls_hidden * sizeof(float));
    if (nm > 0) gemm(m1, m, nm, &M->cls1, M->cls1_b);
    for (int k = 0; k < nm; k++) {
        float *row = m1 + (size_t)k * M->cls_hidden;
        for (int j = 0; j < M->cls_hidden; j++) if (row[j] < 0.f) row[j] = 0.f;
        logits[k] = (double)(dot_f32_lanes(row, M->cls2_w, M->cls_hidden) + M->cls2_b);
    }
    free(m1);
}

/* DebertaV2Model.forward on ids[0..T), then the classifier at the rows in
 * marks[0..nm): logits[nm]. */
static void forward(const Gl *M, const int *ids, int T, const int *marks, int nm, double *logits)
{
    int d = M->d, I = M->inter;
    float *x = (float *)xmalloc((size_t)T * d * sizeof(float));
    float *h = (float *)xmalloc((size_t)T * d * sizeof(float));
    float *att = (float *)xmalloc((size_t)T * d * sizeof(float));
    float *big = (float *)xmalloc((size_t)T * (3 * d > I ? 3 * d : I) * sizeof(float));
    AttnScratch A;
    A.p2c = M->p2c ? (float *)xmalloc((size_t)M->heads * T * 2 * M->span * sizeof(float)) : NULL;
    A.vt = (float *)xmalloc((size_t)d * T * sizeof(float));
    A.ridx = (int *)xmalloc((size_t)(2 * T) * sizeof(int));
    for (int r = 0; r < T; r++) {
        int id = ids[r];
        if (id < 0 || id >= M->vocab) id = 0;
        memcpy(h + (size_t)r * d, M->tok_emb + (size_t)id * d, (size_t)d * sizeof(float));
    }
    layernorm(x, h, M->emb_ln_w, M->emb_ln_b, T, d, M->eps);
    for (int l = 0; l < M->layers; l++) {
        const GlLayer *E = &M->L[l];
        gemm(big, x, T, &E->wqkv, E->bqkv);
        attention(M, E, big, att, T, &A);
        gemm(h, att, T, &E->wo, E->bo);
        add_rows(h, x, (size_t)T * d);
        layernorm(x, h, E->ln1_w, E->ln1_b, T, d, M->eps);       /* x = attention_output */
        gemm(big, x, T, &E->wi, E->bi);
        #pragma omp parallel for schedule(static)
        for (size_t i = 0; i < (size_t)T * I; i++) big[i] = gelu(big[i]);
        gemm(h, big, T, &E->wo2, E->bo2);
        add_rows(h, x, (size_t)T * d);
        layernorm(x, h, E->ln2_w, E->ln2_b, T, d, M->eps);
    }
    float *m = (float *)xmalloc((size_t)(nm > 0 ? nm : 1) * d * sizeof(float));
    for (int k = 0; k < nm; k++) memcpy(m + (size_t)k * d, x + (size_t)marks[k] * d, (size_t)d * sizeof(float));
    classify(M, m, nm, logits);
    free(m); free(x); free(h); free(att); free(big);
    free(A.p2c); free(A.vt); free(A.ridx);
}

#ifdef COLI_VULKAN
/* ------------------------------------------- the forward on the device */

/* forward() as frames of the dense chain (decide_vk.h): the embedding rows go up, the
 * 24 layers run on the device in forward()'s order with its arithmetic (f32; the
 * disentangled attention with both relative terms, chain_enc.comp mode 5), and the [L]
 * rows come back for classify(). The relative embeddings' projections (pos_q, pos_k),
 * the norms' weights and the biases are uploaded once. */
typedef struct {
    int ok;
    VkcBuf *prm, *pos;
    VkcBuf *x, *h, *big, *att, *rng, *ridx, *ipos, *down, *c2p, *p2c;
    int emb_w, emb_b;
    int *bqkv, *bo, *ln1w, *ln1b, *bi, *bo2, *ln2w, *ln2b, *pq, *pk;
} GlVk;
static GlVk g_gvk;

static void gl_vk_setup(Gl *M)
{
    if (!g_dvk.ready || (!g_dvk.chain && !g_dvk.dense)) return;   /* nothing of it on the device */
    int n = 0, want = 0;
    for (int l = 0; l < M->layers; l++) {
        GlLayer *E = &M->L[l];
        const QiMat *w[4] = {&E->wqkv, &E->wo, &E->wi, &E->wo2};
        for (int k = 0; k < 4; k++, want++) n += dvk_tensor(w[k]) != NULL;
    }
    want++;
    n += dvk_tensor(&M->cls1) != NULL;
    size_t bytes = 0, tensors = 0;
    coli_vk_mem_info(&bytes, &tensors);
    fprintf(stderr, "[VK] gliner_decide: %d of %d matrices on the device (%.1f MiB)\n", n, want, bytes / 1048576.0);
    if (!g_dvk.chain) return;
    if (n != want || M->hd > 128 || M->d > 4096) {
        fprintf(stderr, "[VK] gliner_decide: the forward stays on the CPU (%s)\n",
                n != want ? "a matrix did not reach the device" : "a geometry past the chain's shaders");
        g_dvk.chain = 0;
        g_dvk.dense = coli_vk_dense();
        return;
    }
    GlVk *V = &g_gvk;
    int L = M->layers, d = M->d, R = 2 * M->span;
    int **lv[10] = {&V->bqkv, &V->bo, &V->ln1w, &V->ln1b, &V->bi, &V->bo2, &V->ln2w, &V->ln2b, &V->pq, &V->pk};
    for (int k = 0; k < 10; k++) *lv[k] = (int *)xcalloc((size_t)L, sizeof(int));
    DvkPrm P = {0}, Q = {0};
    V->emb_w = dvk_prm_add(&P, M->emb_ln_w, d);
    V->emb_b = dvk_prm_add(&P, M->emb_ln_b, d);
    for (int l = 0; l < L; l++) {
        GlLayer *E = &M->L[l];
        V->bqkv[l] = dvk_prm_add(&P, E->bqkv, (size_t)3 * d);
        V->bo[l] = dvk_prm_add(&P, E->bo, d);
        V->ln1w[l] = dvk_prm_add(&P, E->ln1_w, d);
        V->ln1b[l] = dvk_prm_add(&P, E->ln1_b, d);
        V->bi[l] = dvk_prm_add(&P, E->bi, M->inter);
        V->bo2[l] = dvk_prm_add(&P, E->bo2, d);
        V->ln2w[l] = dvk_prm_add(&P, E->ln2_w, d);
        V->ln2b[l] = dvk_prm_add(&P, E->ln2_b, d);
        V->pq[l] = dvk_prm_add(&Q, E->pos_q, (size_t)R * d);
        V->pk[l] = dvk_prm_add(&Q, E->pos_k, (size_t)R * d);
    }
    V->prm = dvk_prm_upload(&P);
    V->pos = dvk_prm_upload(&Q);
    free(P.v); free(Q.v);
    if (!V->prm || !V->pos) {
        fprintf(stderr, "[VK] gliner_decide: the forward stays on the CPU (no device memory for its parameters)\n");
        g_dvk.chain = 0;
        g_dvk.dense = coli_vk_dense();
        return;
    }
    V->ok = 1;
}

static int gvk_lin(const QiMat *W, VkcBuf *x, VkcBuf *y, int T, int bias, int act)
{
    ColiVkTensor *t = dvk_tensor(W);
    if (!t || !vkc_matmul(t, x, 0, y, 0, T)) return 0;
    VkcEncBias p = {0, T * W->N, W->N, 0, W->N, bias, act};
    return vkc_enc_bias(y, g_gvk.prm, &p);
}

/* forward() on the device; 0 = the caller runs forward(). */
static int gl_vk_forward(const Gl *M, const int *ids, int T, const int *marks, int nm, double *logits)
{
    GlVk *V = &g_gvk;
    if (!g_dvk.chain || !V->ok || !vkc_ready() || T < 1 || T > 65535) return 0;
    double t0 = dvk_now_ms();
    int d = M->d, I = M->inter, R = 2 * M->span;
    int wide = 3 * d > I ? 3 * d : I, nread = nm > 0 ? nm : 1;
    size_t fd = sizeof(float), id = sizeof(int);
    if (!vkc_reserve(&V->x, (size_t)T * d * fd, VKC_DEV) || !vkc_reserve(&V->h, (size_t)T * d * fd, VKC_DEV) ||
        !vkc_reserve(&V->att, (size_t)T * d * fd, VKC_DEV) || !vkc_reserve(&V->big, (size_t)T * wide * fd, VKC_DEV) ||
        !vkc_reserve(&V->rng, (size_t)2 * T * id, VKC_DEV) || !vkc_reserve(&V->ridx, (size_t)2 * T * id, VKC_DEV) ||
        !vkc_reserve(&V->ipos, (size_t)T * id, VKC_DEV) || !vkc_reserve(&V->down, (size_t)nread * d * fd, VKC_DOWN))
        return 0;
    /* the buckets a step can reach: ridx of delta -(T-1) .. T-1, which only grows */
    int rlo_ = 0, rhi_ = R - 1;
    {
        int b = M->bucket_size > 0 && M->max_position > 0 ? log_bucket(-(T - 1), M->bucket_size, M->max_position) : -(T - 1);
        int e = M->bucket_size > 0 && M->max_position > 0 ? log_bucket(T - 1, M->bucket_size, M->max_position) : T - 1;
        rlo_ = b + M->span < 0 ? 0 : b + M->span > R - 1 ? R - 1 : b + M->span;
        rhi_ = e + M->span < 0 ? 0 : e + M->span > R - 1 ? R - 1 : e + M->span;
    }
    int nr = rhi_ - rlo_ + 1;
    if ((M->c2p && !vkc_reserve(&V->c2p, (size_t)M->heads * T * nr * fd, VKC_DEV)) ||
        (M->p2c && !vkc_reserve(&V->p2c, (size_t)M->heads * T * nr * fd, VKC_DEV)))
        return 0;
    /* the host's part: the embedding rows; every row sees the whole sequence; the
     * bucketed relative position of every delta, attention()'s ridx */
    float *emb = (float *)xmalloc((size_t)T * d * fd);
    int *rng = (int *)xmalloc((size_t)2 * T * id), *ridx = (int *)xmalloc((size_t)2 * T * id);
    int *pos = (int *)xmalloc((size_t)T * id);
    for (int r = 0; r < T; r++) {
        int t = ids[r];
        if (t < 0 || t >= M->vocab) t = 0;
        memcpy(emb + (size_t)r * d, M->tok_emb + (size_t)t * d, (size_t)d * fd);
        rng[2 * r] = 0; rng[2 * r + 1] = T - 1; pos[r] = r;
    }
    for (int delta = -(T - 1); delta <= T - 1; delta++) {
        int b = M->bucket_size > 0 && M->max_position > 0 ? log_bucket(delta, M->bucket_size, M->max_position) : delta;
        int r = b + M->span;
        ridx[delta + T - 1] = r < 0 ? 0 : r > R - 1 ? R - 1 : r;
    }
    ridx[2 * T - 1] = 0;
    VkcRegion *reg = (VkcRegion *)xmalloc((size_t)nread * sizeof(VkcRegion));
    for (int k = 0; k < nm; k++) reg[k] = (VkcRegion){(size_t)k * d, (size_t)marks[k] * d, (size_t)d};
    VkcBuf *x = V->x, *h = V->h, *big = V->big, *att = V->att;
    int ok = vkc_begin() && vkc_write(h, 0, emb, (size_t)T * d * fd) && vkc_write(V->rng, 0, rng, (size_t)2 * T * id) &&
             vkc_write(V->ridx, 0, ridx, (size_t)2 * T * id) && vkc_write(V->ipos, 0, pos, (size_t)T * id);
    free(emb); free(rng); free(ridx); free(pos);
    VkcEncNorm np = {0, T, d, 0, d, 0, d, 0, d, V->emb_w, V->emb_b, 0, M->eps};
    ok = ok && vkc_enc_norm(h, NULL, V->prm, x, &np);
    int sf = 1 + M->c2p + M->p2c;
    float inv = 1.0f / sqrtf((float)(M->hd * sf));
    int flags = (M->c2p ? VKC_ENC_C2P : 0) | (M->p2c ? VKC_ENC_P2C : 0);
    for (int l = 0; ok && l < M->layers; l++) {
        const GlLayer *E = &M->L[l];
        ok = gvk_lin(&E->wqkv, x, big, T, V->bqkv[l], VKC_ENC_ACT_NONE);
        /* the relative terms' products first, attention()'s c2p (q against the keys of
         * the relative embeddings) and p2c (k against their queries), for the buckets
         * this step reaches */
        VkcEncRel rc = {0, T, M->heads, M->hd, 0, 3 * d, V->pk[l], d, 0, nr, rlo_};
        VkcEncRel rp = {0, T, M->heads, M->hd, d, 3 * d, V->pq[l], d, 0, nr, rlo_};
        if (M->c2p) ok = ok && vkc_enc_rel(big, V->pos, V->c2p, &rc);
        if (M->p2c) ok = ok && vkc_enc_rel(big, V->pos, V->p2c, &rp);
        VkcEncAttn ap = {0, T, M->heads, M->hd, 0, d, 2 * d, 3 * d, 0, d, flags, 0, 0, nr, T - 1, inv, rlo_};
        ok = ok && vkc_enc_attn(big, V->c2p, V->p2c, att, V->rng, V->ridx, V->ipos, &ap);
        ok = ok && gvk_lin(&E->wo, att, h, T, V->bo[l], VKC_ENC_ACT_NONE);
        VkcEncNorm n1 = {0, T, d, 0, d, 0, d, 0, d, V->ln1w[l], V->ln1b[l], VKC_ENC_ADD, M->eps};   /* x = LN(h + x) */
        ok = ok && vkc_enc_norm(h, x, V->prm, x, &n1);
        ok = ok && gvk_lin(&E->wi, x, big, T, V->bi[l], VKC_ENC_ACT_GELU);
        ok = ok && gvk_lin(&E->wo2, big, h, T, V->bo2[l], VKC_ENC_ACT_NONE);
        VkcEncNorm n2 = {0, T, d, 0, d, 0, d, 0, d, V->ln2w[l], V->ln2b[l], VKC_ENC_ADD, M->eps};
        ok = ok && vkc_enc_norm(h, x, V->prm, x, &n2);
    }
    ok = ok && (nm == 0 || vkc_copy_regions(V->down, x, reg, nm)) && vkc_submit(1);
    free(reg);
    if (!ok) {
        if (vkc_lost()) fprintf(stderr, "[VK] gliner_decide: the device was lost: the forward runs on the CPU\n");
        else vkc_finish();
        return 0;
    }
    float *m = (float *)xmalloc((size_t)nread * d * fd);
    memcpy(m, vkc_ptr(V->down), (size_t)nm * d * fd);
    classify(M, m, nm, logits);
    free(m);
    g_dvk.fwd_dev++; g_dvk.rows_dev += (unsigned long long)T;
    g_dvk.dev_ms += dvk_now_ms() - t0;
    return 1;
}
#endif

/* ------------------------------------------------------- the rendering */

typedef struct {
    IntVec ids;
    int *label_marks;            /* the [L] rows, every question's labels in turn */
    int n_marks;
    int *first_mark;             /* [Q+1]: question q owns label_marks[first_mark[q] .. first_mark[q+1]) */
    int schema_tokens;           /* up to and including [SEP_TEXT] */
    int state_tokens, state_used;
    int words, words_used;
} Rendered;

static void rendered_free(Rendered *r)
{
    free(r->ids.v); free(r->label_marks); free(r->first_mark);
    memset(r, 0, sizeof(*r));
}

/* A question's labels in the order the model reads them: a choice's labels, a
 * score's levels; a noul is asked as "yes" then "no", the order the model
 * card writes every yes/no question in. label_text[i] is option i's text in
 * that order, NULL when it has none. */
static int question_labels(const DecideQuestion *q, const char **label, const char **text)
{
    if (q->type == DECIDE_NOUL) {
        label[0] = "yes"; text[0] = q->options[1].text;
        label[1] = "no";  text[1] = q->options[0].text;
        return 2;
    }
    for (int i = 0; i < q->n_options; i++) { label[i] = q->options[i].label; text[i] = q->options[i].text; }
    return q->n_options;
}

static void push_encoded(const Gl *M, Rendered *r, const char *s)
{
    gl_encode(&M->tok, s, strlen(s), &r->ids);
}

/* processor._transform_record for one record: the schema of every question,
 * [SEP_TEXT], then as many words of the state as fit in max_len tokens.
 * Returns 1, or 0 with the reason when the questions alone do not fit. */
static int render(const Gl *M, const DecideRecord *rec, Rendered *r, char *err, size_t cap)
{
    memset(r, 0, sizeof(*r));
    int total_labels = 0;
    for (int q = 0; q < rec->n_questions; q++) total_labels += rec->questions[q].n_options;
    r->label_marks = (int *)xmalloc((size_t)(total_labels > 0 ? total_labels : 1) * sizeof(int));
    r->first_mark = (int *)xmalloc((size_t)(rec->n_questions + 1) * sizeof(int));
    const char *label[DECIDE_MAX_OPTIONS], *text[DECIDE_MAX_OPTIONS];
    for (int q = 0; q < rec->n_questions; q++) {
        const DecideQuestion *Q = &rec->questions[q];
        int k = question_labels(Q, label, text);
        if (q) ivec_push(&r->ids, M->id_sep_struct);
        /* ( [P] prompt_str ( [L] label ... ) ) */
        push_encoded(M, r, "(");
        ivec_push(&r->ids, M->id_p);
        Str prompt = {0};
        str_put(&prompt, Q->id, strlen(Q->id));
        if (Q->instructions[0]) { str_put(&prompt, ": ", 2); str_put(&prompt, Q->instructions, strlen(Q->instructions)); }
        for (int i = 0; i < k; i++) {
            if (!text[i] || !text[i][0]) continue;
            str_put(&prompt, " [DESCRIPTION] ", 15);
            str_put(&prompt, label[i], strlen(label[i]));
            str_put(&prompt, ": ", 2);
            str_put(&prompt, text[i], strlen(text[i]));
        }
        gl_encode(&M->tok, prompt.s, prompt.n, &r->ids);
        free(prompt.s);
        push_encoded(M, r, "(");
        r->first_mark[q] = r->n_marks;
        for (int i = 0; i < k; i++) {
            r->label_marks[r->n_marks++] = r->ids.n;
            ivec_push(&r->ids, M->id_l);
            push_encoded(M, r, label[i]);
        }
        push_encoded(M, r, ")");
        push_encoded(M, r, ")");
    }
    r->first_mark[rec->n_questions] = r->n_marks;
    ivec_push(&r->ids, M->id_sep_text);
    r->schema_tokens = r->ids.n;
    if (r->schema_tokens > M->max_len) {
        snprintf(err, cap, "questions: the questions and their options take %d tokens, more than "
                 "max_len=%d (COLI_GLINER_MAX_LEN); ask fewer questions or shorten the options",
                 r->schema_tokens, M->max_len);
        return 0;
    }
    /* the state: "." appended unless it ends in . ! or ? (processor._collate_batch) */
    size_t sl = strlen(rec->state);
    char *state = (char *)xmalloc(sl + 2);
    memcpy(state, rec->state, sl + 1);
    if (!sl) { state[0] = '.'; state[1] = 0; sl = 1; }
    else if (state[sl - 1] != '.' && state[sl - 1] != '!' && state[sl - 1] != '?') { state[sl] = '.'; state[sl + 1] = 0; sl++; }
    Words w;
    split_words(state, sl, &w);
    free(state);
    r->words = w.n;
    int full = 1;
    IntVec piece = {0};
    for (int i = 0; i < w.n; i++) {
        piece.n = 0;
        gl_encode(&M->tok, w.w[i], strlen(w.w[i]), &piece);
        r->state_tokens += piece.n;
        if (full && r->ids.n + piece.n <= M->max_len) {
            for (int t = 0; t < piece.n; t++) ivec_push(&r->ids, piece.v[t]);
            r->state_used += piece.n;
            r->words_used++;
        } else {
            full = 0;          /* max_len counts whole words from the start */
        }
    }
    free(piece.v);
    words_free(&w);
    return 1;
}

/* ------------------------------------------------------- the DECIDE call */

typedef struct {
    Gl *M;
    int dump_ids;
    Rendered last;               /* the last record's sequence, for --records --ids */
} GlCtx;

static int gl_decide(void *opaque, const DecideRecord *rec, DecideAnswer *answers,
                     int *input_tokens, char *err, size_t cap)
{
    GlCtx *ctx = (GlCtx *)opaque;
    Gl *M = ctx->M;
    rendered_free(&ctx->last);
    Rendered r;
    if (!render(M, rec, &r, err, cap)) { rendered_free(&r); return 0; }
    double *logits = (double *)xcalloc((size_t)(r.n_marks > 0 ? r.n_marks : 1), sizeof(double));
#ifdef COLI_VULKAN
    if (!gl_vk_forward(M, r.ids.v, r.ids.n, r.label_marks, r.n_marks, logits)) {
        if (g_dvk.chain) g_dvk.fwd_cpu++;
        forward(M, r.ids.v, r.ids.n, r.label_marks, r.n_marks, logits);
    }
    dvk_report();
#else
    forward(M, r.ids.v, r.ids.n, r.label_marks, r.n_marks, logits);
#endif
    for (int q = 0; q < rec->n_questions; q++) {
        const DecideQuestion *Q = &rec->questions[q];
        DecideAnswer *a = &answers[q];
        int k = r.first_mark[q + 1] - r.first_mark[q];
        const double *z = logits + r.first_mark[q];
        a->n = k;
        a->logits = (double *)xcalloc((size_t)k, sizeof(double));
        a->probs = (double *)xcalloc((size_t)k, sizeof(double));
        a->temperature = NAN;            /* no calibration ships with the checkpoint */
        /* softmax over the task's labels, as _extract_classification_result does
         * for a single-label task (temperature 1) */
        double mx = -INFINITY, sum = 0;
        for (int i = 0; i < k; i++) if (z[i] > mx) mx = z[i];
        double *p = (double *)xmalloc((size_t)k * sizeof(double));
        for (int i = 0; i < k; i++) { p[i] = exp(z[i] - mx); sum += p[i]; }
        for (int i = 0; i < k; i++) p[i] /= sum;
        for (int i = 0; i < k; i++) {
            /* back to the record's option order: a noul was read as yes, no */
            int src = Q->type == DECIDE_NOUL ? 1 - i : i;
            a->logits[i] = z[src];
            a->probs[i] = p[src];
        }
        free(p);
        a->n_actions = 0;
        a->tokens = r.ids.n;
        a->state_tokens = r.state_tokens;
        a->state_dropped = r.state_tokens - r.state_used;
    }
    free(logits);
    *input_tokens = r.ids.n;
    if (ctx->dump_ids) ctx->last = r;
    else rendered_free(&r);
    return 1;
}

/* ------------------------------------------------------------------- main */

static double rss_gb(void)
{
#ifdef __linux__
    FILE *f = fopen("/proc/self/statm", "r");
    long pages = 0, resident = 0;
    if (f) { if (fscanf(f, "%ld %ld", &pages, &resident) != 2) resident = 0; fclose(f); }
    return resident * (double)sysconf(_SC_PAGESIZE) / 1e9;
#else
    return 0.0;
#endif
}

static void print_json_string(FILE *out, const char *s)
{
    DecideBuf b = {0};
    decide_buf_string(&b, s);
    fputs(b.data ? b.data : "\"\"", out);
    free(b.data);
}

static jval *read_string_array(const char *path, char **text_out)
{
    char *text = read_text(path);
    if (!text) die("cannot read %s", path);
    jval *root = json_parse_checked(text);
    if (!root || root->t != J_ARR) die("%s: expected a JSON array", path);
    *text_out = text;
    return root;
}

static int run_records(Gl *M, const char *path, int dump_ids)
{
    char *text;
    jval *root = read_string_array(path, &text);
    GlCtx ctx;
    memset(&ctx, 0, sizeof(ctx));
    ctx.M = M;
    ctx.dump_ids = dump_ids;
    /* Each record is the text of a DECIDE payload and goes through
     * decide_record_parse, the path the serve loop takes. */
    for (int i = 0; i < root->len; i++) {
        jval *payload = json_get(root->kids[i], "payload");
        if (!payload || payload->t != J_STR) die("%s: record %d has no string \"payload\"", path, i);
        DecideRecord rec;
        char reason[1024];
        if (!decide_record_parse(payload->str, &rec, reason, sizeof(reason))) {
            printf("{\"error\":"); print_json_string(stdout, reason); printf("}\n");
            continue;
        }
        DecideAnswer *answers = (DecideAnswer *)xcalloc((size_t)rec.n_questions, sizeof(DecideAnswer));
        int tokens = 0;
        double t0 = now_ms();
        int ok = gl_decide(&ctx, &rec, answers, &tokens, reason, sizeof(reason));
        double ms = now_ms() - t0;
        if (!ok) {
            printf("{\"error\":"); print_json_string(stdout, reason); printf("}\n");
        } else {
            size_t len = 0;
            char *json = decide_answers_json(&rec, answers, tokens, ms, &len);
            if (dump_ids && json && len > 1) {
                json[len - 1] = 0;                 /* reopen the object */
                fputs(json, stdout);
                printf(",\"sequence\":{\"ids\":[");
                for (int t = 0; t < ctx.last.ids.n; t++) printf("%s%d", t ? "," : "", ctx.last.ids.v[t]);
                printf("],\"markers\":[");
                for (int t = 0; t < ctx.last.n_marks; t++) printf("%s%d", t ? "," : "", ctx.last.label_marks[t]);
                printf("],\"words\":%d,\"words_used\":%d}}\n", ctx.last.words, ctx.last.words_used);
            } else if (json) {
                fputs(json, stdout); fputc('\n', stdout);
            }
            free(json);
        }
        fflush(stdout);
        decide_answers_free(answers, rec.n_questions);
        free(answers);
        decide_record_free(&rec);
    }
    rendered_free(&ctx.last);
    json_free(root);
    free(text);
    return 0;
}

static int run_tokenize(Gl *M, const char *path)
{
    char *text;
    jval *root = read_string_array(path, &text);
    printf("[");
    for (int i = 0; i < root->len; i++) {
        jval *s = root->kids[i];
        if (s->t != J_STR) die("%s: entry %d is not a string", path, i);
        IntVec ids = {0};
        gl_encode(&M->tok, s->str, strlen(s->str), &ids);
        printf("%s[", i ? "," : "");
        for (int t = 0; t < ids.n; t++) printf("%s%d", t ? "," : "", ids.v[t]);
        printf("]");
        free(ids.v);
    }
    printf("]\n");
    json_free(root);
    free(text);
    return 0;
}

static int run_split(const char *path)
{
    char *text;
    jval *root = read_string_array(path, &text);
    printf("[");
    for (int i = 0; i < root->len; i++) {
        jval *s = root->kids[i];
        if (s->t != J_STR) die("%s: entry %d is not a string", path, i);
        Words w;
        split_words(s->str, strlen(s->str), &w);
        printf("%s[", i ? "," : "");
        for (int t = 0; t < w.n; t++) { if (t) printf(","); print_json_string(stdout, w.w[t]); }
        printf("]");
        words_free(&w);
    }
    printf("]\n");
    json_free(root);
    free(text);
    return 0;
}

int main(int argc, char **argv)
{
    const char *model = NULL, *records = NULL, *tokenize = NULL, *split = NULL;
    int dump_ids = 0;
    for (int i = 1; i < argc; i++) {
        if (!strcmp(argv[i], "--model") && i + 1 < argc) model = argv[++i];
        else if (!strcmp(argv[i], "--records") && i + 1 < argc) records = argv[++i];
        else if (!strcmp(argv[i], "--tokenize") && i + 1 < argc) tokenize = argv[++i];
        else if (!strcmp(argv[i], "--split") && i + 1 < argc) split = argv[++i];
        else if (!strcmp(argv[i], "--ids")) dump_ids = 1;
        else if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-h")) {
            printf("usage: SERVE=1 SNAP=<dir> gliner_decide            serve DECIDE on stdin/stdout\n"
                   "       gliner_decide --model DIR --records FILE [--ids]\n"
                   "       gliner_decide --model DIR --tokenize FILE\n"
                   "       gliner_decide --split FILE\n");
            return 0;
        }
        /* a bare number is the gateway's cache-cap argument: nothing to cache here */
    }
    if (split) return run_split(split);
    const char *serve = getenv("SERVE");
    int serving = serve && !strcmp(serve, "1") && !records && !tokenize;
    if (!model) model = getenv("SNAP");
    if (!model) die("no model: pass --model DIR or set SNAP");
    if (serving) coli_serve_stdio_init();
    coli_omp_tune_threads("gliner_decide");   /* physical cores: SMT halves the GEMMs here too */
#ifdef _OPENMP
    omp_set_max_active_levels(1);             /* the attention's qi_gemm calls run inside its tasks */
#endif
    double t0 = now_ms();
    Gl *M = (Gl *)xcalloc(1, sizeof(Gl));
    load_model(M, model);
    fprintf(stderr, "[gliner_decide] %s: DeBERTa-v2 %d layers x %d, %d heads, relative span %d; "
            "max_len %d; loaded in %.1f s\n", M->model_name, M->layers, M->d, M->heads, M->span,
            M->max_len, (now_ms() - t0) / 1e3);
    if (tokenize) return run_tokenize(M, tokenize);
#ifdef COLI_VULKAN
    dvk_init("gliner_decide");   /* COLI_VULKAN=1: the device */
    gl_vk_setup(M);
#endif
    if (records) return run_records(M, records, dump_ids);
    if (!serving) die("nothing to do: set SERVE=1, or pass --records / --tokenize / --split");
    int slots = getenv("KV_SLOTS") ? atoi(getenv("KV_SLOTS")) : 1;
    if (slots < 1) slots = 1;
    char caps[256];
    snprintf(caps, sizeof(caps), "decide=1 chat=0 max_len=%d", M->max_len);
    GlCtx ctx;
    memset(&ctx, 0, sizeof(ctx));
    ctx.M = M;
    return decide_serve(stdin, stdout, gl_decide, &ctx, caps, rss_gb(), slots);
}