Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
61.1 KB
Raw Blame
/* laya.c -- Laya, a decision model by Convai Innovations
 * (convaiinnovations/laya on Hugging Face, Apache-2.0), in C.
 *
 * Laya does not generate. Given a state and typed questions (choice, score,
 * noul) it scores every option of every question in one forward pass and
 * returns calibrated probabilities. It is a bidirectional encoder,
 * ModernBERT-large in the English checkpoint, plus a decision head trained
 * from scratch: two transformer layers, an option scorer read at one [MASK]
 * marker per option, and an act/escalate head.
 *
 * The specification is the model's own Python: the `laya` package (common.py,
 * agent.py) and the ModernBERT modeling code it builds on. Each step below
 * names the function it follows:
 *
 *   sequence     common.build_head / build_sequence / render_options
 *   encoder      ModernBertModel.forward: LayerNorm'd embeddings, 28 pre-norm
 *                layers (no norm before layer 0's attention), RoPE per layer
 *                type (global theta 160000 on every third layer starting at 0,
 *                local theta 10000 elsewhere with a +-64 window), GeGLU MLP,
 *                final LayerNorm; no biases
 *   head         DecisionModel.forward: + type_emb[qtype], two post-attention
 *                nn.TransformerEncoderLayer (norm_first, ReLU, d/64 heads),
 *                scorer LN -> Linear -> GELU -> Linear at each marker,
 *                act head on [h[0], top1, top1-top2, entropy, k/255]
 *   calibration  Agent._decode_answers: one temperature per (type, option
 *                count) bucket, clamped to [0.5, 5] (common.clamp_temperature),
 *                softmax over the question's options
 *
 * Weights are read as stored (F16 in the release) and kept in f32. The
 * forward runs every question of a request as one batch of rows; attention
 * stays per sequence, so a question never sees another's tokens.
 *
 * Two ways to run it:
 *   SERVE=1 SNAP=<dir> laya          the mux serve protocol, DECIDE only
 *                                    (decide_serve.h; what the gateway runs)
 *   laya --model DIR --records F     a JSON array of DECIDE records; one
 *                                    DECISION object per line (tests)
 *   laya --model DIR --tokenize F    a JSON array of strings; their ids
 */
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <math.h>
#include <stdarg.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#ifdef _OPENMP
#include <omp.h>
#endif

#include "compat.h"
#include "json.h"
#include "st.h"
#include "tok.h"
#include "qwen38_nfc.h"
#include "qi_gemm.h"
#include "matmul_f32.h"
#include "serve_codec.h"
#include "decide_serve.h"
#include "omp_tune.h"

/* ------------------------------------------------------------------ utils */

static void die(const char *fmt, ...)
{
    va_list args;
    va_start(args, fmt);
    fprintf(stderr, "[laya] ");
    vfprintf(stderr, fmt, args);
    fputc('\n', stderr);
    va_end(args);
    exit(1);
}

static void *xmalloc(size_t size)
{
    void *p = malloc(size ? size : 1);
    if (!p) die("out of memory (%zu bytes)", size);
    return p;
}

static void *xcalloc(size_t count, size_t size)
{
    void *p = calloc(count ? count : 1, size ? size : 1);
    if (!p) die("out of memory (%zu x %zu bytes)", count, size);
    return p;
}

static char *read_text(const char *path)
{
    FILE *f = fopen(path, "rb");
    if (!f) return NULL;
    if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return NULL; }
    long n = ftell(f);
    if (n < 0 || n > (1L << 30)) { fclose(f); return NULL; }
    rewind(f);
    char *b = (char *)xmalloc((size_t)n + 1);
    if (fread(b, 1, (size_t)n, f) != (size_t)n) { fclose(f); free(b); return NULL; }
    b[n] = 0;
    fclose(f);
    return b;
}

static jval *read_json(const char *dir, const char *name)
{
    char path[4096];
    snprintf(path, sizeof(path), "%s/%s", dir, name);
    char *text = read_text(path);
    if (!text) die("cannot read %s", path);
    jval *root = json_parse_checked(text);
    free(text);
    if (!root || root->t != J_OBJ) die("%s is not a JSON object", path);
    return root;
}

static double num_or(jval *o, const char *key, double fallback)
{
    jval *v = json_get(o, key);
    return v && v->t == J_NUM ? v->num : fallback;
}

static int bool_or(jval *o, const char *key, int fallback)
{
    jval *v = json_get(o, key);
    return v && v->t == J_BOOL ? v->boolean : fallback;
}

static double now_ms(void)
{
    struct timespec t;
    clock_gettime(CLOCK_MONOTONIC, &t);
    return t.tv_sec * 1e3 + t.tv_nsec / 1e6;
}

/* Python's floor division, for the one place the reference divides a value
 * that can be negative (build_head's per-option cap). */
static int floordiv(int a, int b)
{
    int q = a / b;
    if ((a % b) && ((a < 0) != (b < 0))) q--;
    return q;
}

/* ------------------------------------------------------------------ model */

typedef struct {
    float *attn_norm_w, *attn_norm_b;   /* NULL on layer 0: ModernBERT has no norm there */
    QiMat wqkv, wo, wi, wo2;
    float *bqkv, *bo, *bi, *bo2;        /* NULL without attention_bias / mlp_bias */
    float *mlp_norm_w, *mlp_norm_b;
    int global;                          /* full attention; otherwise the local window */
} EncLayer;

typedef struct {
    float *n1w, *n1b, *n2w, *n2b;
    QiMat in_proj, out_proj, lin1, lin2;
    float *in_proj_b, *out_proj_b, *lin1_b, *lin2_b;
} HeadLayer;

typedef struct {
    /* encoder config */
    int d, layers, heads, hd, inter, vocab, max_pos, window;
    float eps, theta_global, theta_local;
    int gelu_tanh;
    float *tok_emb;                      /* [vocab][d] */
    float *emb_norm_w, *emb_norm_b, *final_norm_w, *final_norm_b;
    EncLayer *L;
    float *rope_g, *rope_l;              /* [max_pos][hd/2] cos then [max_pos][hd/2] sin */
    /* decision head */
    int head_layers, head_heads;
    HeadLayer *H;
    float *type_emb;                     /* [3][d] */
    float *sc_ln_w, *sc_ln_b, *sc2_w, sc2_b;
    QiMat sc1; float *sc1_b;
    int act_hidden, n_act;
    QiMat act1; float *act1_b;
    float *act2_w, *act2_b;              /* [n_act][act_hidden] */
    char *action_names[DECIDE_MAX_ACTIONS];
    /* rl_agent_config.json */
    int max_len, head_max_len;
    double temperature[3];
    int n_buckets;
    char *bucket_key[32];
    double bucket_t[32];
    /* tokenizer */
    Tok tok;
    int nfc;
    int cls_id, sep_id, mask_id, pad_id;
    char *mask_str;
    unsigned char added_first[256];      /* first bytes of the added tokens */
    char model_name[128];
} Laya;

/* common.clamp_temperature: a usable temperature, confined to [0.5, 5]; a value
 * that is not a finite number (or is a JSON bool) falls back to 1.0. */
static double clamp_temperature(jval *v)
{
    if (!v || v->t != J_NUM || !isfinite(v->num)) return 1.0;
    double t = v->num;
    return t < 0.5 ? 0.5 : t > 5.0 ? 5.0 : t;
}

static float *load_f32(shards *S, const char *name, int64_t expect)
{
    st_tensor *t = st_find(S, name);
    if (!t) die("model.safetensors has no tensor %s", name);
    if (expect >= 0 && t->numel != expect)
        die("%s has %lld elements, expected %lld", name, (long long)t->numel, (long long)expect);
    float *out = (float *)xmalloc((size_t)t->numel * sizeof(float));
    st_read_f32(S, name, out, 0);
    return out;
}

static float *load_opt(shards *S, const char *name, int64_t expect)
{
    return st_find(S, name) ? load_f32(S, name, expect) : NULL;
}

static QiMat load_mat(shards *S, const char *name, int N, int K)
{
    QiMat m = {QI_F32, N, K, load_f32(S, name, (int64_t)N * K), NULL, 0};
    return m;
}

/* No int8 option: measured on the release (tools/compare_laya.py, 36 requests,
 * 117 questions), int8 weights flip 1 decision and int8 weights with int8
 * activations flip 4, against 0 for f32. The decisions are the contract. */

static void rope_table(float *table, int max_pos, int hd, float theta)
{
    /* ModernBertRotaryEmbedding.compute_default_rope_parameters, in float32 like
     * the reference: inv_freq = 1 / theta^(2i/hd), angle = pos * inv_freq. */
    int half = hd / 2;
    float *cosv = table, *sinv = table + (size_t)max_pos * half;
    for (int i = 0; i < half; i++) {
        float inv = 1.0f / powf(theta, (float)(2 * i) / (float)hd);
        for (int p = 0; p < max_pos; p++) {
            float a = (float)p * inv;
            cosv[(size_t)p * half + i] = cosf(a);
            sinv[(size_t)p * half + i] = sinf(a);
        }
    }
}

static const char *special_content(jval *cfg, const char *key)
{
    jval *v = json_get(cfg, key);
    if (v && v->t == J_STR) return v->str;
    if (v && v->t == J_OBJ) { jval *c = json_get(v, "content"); if (c && c->t == J_STR) return c->str; }
    return NULL;
}

static void load_tokenizer(Laya *M, const char *dir)
{
    char path[4096];
    snprintf(path, sizeof(path), "%s/tokenizer/tokenizer.json", dir);
    jval *tj = read_json(dir, "tokenizer/tokenizer.json");
    jval *model = json_get(tj, "model"), *mt = model ? json_get(model, "type") : NULL;
    if (!mt || mt->t != J_STR || strcmp(mt->str, "BPE"))
        die("tokenizer: only a byte-level BPE tokenizer is supported (this is %s)",
            mt && mt->t == J_STR ? mt->str : "unknown");
    jval *pt = json_get(tj, "pre_tokenizer"), *ptt = pt ? json_get(pt, "type") : NULL;
    if (!ptt || ptt->t != J_STR || strcmp(ptt->str, "ByteLevel") || !bool_or(pt, "use_regex", 1) ||
        bool_or(pt, "add_prefix_space", 0))
        die("tokenizer: expected a ByteLevel pre-tokenizer with the GPT-2 regex and no prefix space");
    jval *norm = json_get(tj, "normalizer");
    if (norm && norm->t == J_OBJ) {
        jval *nt = json_get(norm, "type");
        if (nt && nt->t == J_STR && !strcmp(nt->str, "NFC")) M->nfc = 1;
        else die("tokenizer: unsupported normalizer %s", nt && nt->t == J_STR ? nt->str : "?");
    } else if (norm && norm->t != J_NULL) {
        die("tokenizer: unsupported normalizer");
    }
    /* Added tokens: the reference matches them on the raw text (special ones) or
     * on the normalized text (the rest); NFC cannot create or break any of these
     * ASCII strings, so matching once after normalizing is the same split. The
     * one lstrip/rstrip token is [MASK], which never reaches the tokenizer:
     * build_sequence replaces it in every text first. */
    jval *added = json_get(tj, "added_tokens");
    for (int i = 0; added && i < added->len; i++) {
        jval *a = added->kids[i], *c = json_get(a, "content");
        if (!c || c->t != J_STR || !c->str[0]) continue;
        int is_mask = M->mask_str && !strcmp(c->str, M->mask_str);
        if (!is_mask && (bool_or(a, "lstrip", 0) || bool_or(a, "rstrip", 0) || bool_or(a, "single_word", 0)))
            die("tokenizer: added token %s strips whitespace, which this engine does not implement", c->str);
        M->added_first[(unsigned char)c->str[0]] = 1;
    }
    json_free(tj);
    tok_load(&M->tok, path);
    if (!M->tok.gpt2) die("tokenizer: not the GPT-2 byte-level pre-tokenizer");
}

static void load_model(Laya *M, const char *dir)
{
    memset(M, 0, sizeof(*M));
    /* rl_agent_config.json: the head and the calibration */
    jval *rc = read_json(dir, "rl_agent_config.json");
    M->head_layers = (int)num_or(rc, "head_layers", 2);
    M->max_len = (int)num_or(rc, "max_len", 512);
    M->head_max_len = (int)num_or(rc, "head_max_len", 192);
    const char *env;
    if ((env = getenv("COLI_LAYA_MAX_LEN")) && atoi(env) > 0) M->max_len = atoi(env);
    if ((env = getenv("COLI_LAYA_HEAD_MAX_LEN")) && atoi(env) > 0) M->head_max_len = atoi(env);
    jval *name = json_get(rc, "model_name");
    snprintf(M->model_name, sizeof(M->model_name), "%s", name && name->t == J_STR ? name->str : "laya");
    jval *temps = json_get(rc, "temperature");
    for (int i = 0; i < 3; i++)
        M->temperature[i] = temps && temps->t == J_ARR && temps->len == 3 ? clamp_temperature(temps->kids[i]) : 1.0;
    if (temps && (temps->t != J_ARR || temps->len != 3))
        die("rl_agent_config.json: temperature must be a list of 3 numbers");
    jval *tbo = json_get(rc, "temperature_by_options");
    for (int i = 0; tbo && tbo->t == J_OBJ && i < tbo->len && M->n_buckets < 32; i++) {
        M->bucket_key[M->n_buckets] = strdup(tbo->keys[i]);
        M->bucket_t[M->n_buckets++] = clamp_temperature(tbo->kids[i]);
    }
    jval *costs = json_get(rc, "act_costs");
    M->n_act = 1 + (costs && costs->t == J_OBJ ? costs->len : 0);
    if (M->n_act > DECIDE_MAX_ACTIONS) die("rl_agent_config.json: too many act_costs");
    M->action_names[0] = strdup("act");
    for (int i = 1; i < M->n_act; i++) M->action_names[i] = strdup(costs->keys[i - 1]);
    json_free(rc);

    /* encoder/config.json: ModernBERT */
    jval *ec = read_json(dir, "encoder/config.json");
    jval *mtype = json_get(ec, "model_type");
    if (!mtype || mtype->t != J_STR || strcmp(mtype->str, "modernbert"))
        die("encoder/config.json: model_type must be modernbert");
    M->d = (int)num_or(ec, "hidden_size", 0);
    M->layers = (int)num_or(ec, "num_hidden_layers", 0);
    M->heads = (int)num_or(ec, "num_attention_heads", 0);
    M->inter = (int)num_or(ec, "intermediate_size", 0);
    M->vocab = (int)num_or(ec, "vocab_size", 0);
    M->max_pos = (int)num_or(ec, "max_position_embeddings", 8192);
    M->window = (int)num_or(ec, "local_attention", 128) / 2;
    M->eps = (float)num_or(ec, "norm_eps", 1e-5);
    if (M->d < 1 || M->layers < 1 || M->heads < 1 || M->d % M->heads || M->inter < 1 || M->vocab < 1 ||
        M->max_pos < 1 || M->d > 16384 || M->layers > 256 || M->max_pos > 1 << 20)
        die("encoder/config.json: invalid geometry");
    M->hd = M->d / M->heads;
    if (M->hd % 2) die("encoder/config.json: head dimension must be even");
    jval *act = json_get(ec, "hidden_activation");
    if (act && act->t == J_STR) {
        if (!strcmp(act->str, "gelu")) M->gelu_tanh = 0;
        else if (!strcmp(act->str, "gelu_pytorch_tanh") || !strcmp(act->str, "gelu_new")) M->gelu_tanh = 1;
        else die("encoder/config.json: unsupported hidden_activation %s", act->str);
    }
    /* RoPE: transformers 5 writes rope_parameters per layer type; older configs
     * global_rope_theta / local_rope_theta (defaults 160000 / 10000). */
    M->theta_global = (float)num_or(ec, "global_rope_theta", 160000.0);
    M->theta_local = (float)num_or(ec, "local_rope_theta", 10000.0);
    jval *rp = json_get(ec, "rope_parameters");
    if (rp && rp->t == J_OBJ) {
        jval *f = json_get(rp, "full_attention"), *s = json_get(rp, "sliding_attention");
        if (f && f->t == J_OBJ) M->theta_global = (float)num_or(f, "rope_theta", M->theta_global);
        if (s && s->t == J_OBJ) M->theta_local = (float)num_or(s, "rope_theta", M->theta_local);
        jval *ft = f ? json_get(f, "rope_type") : NULL, *stp = s ? json_get(s, "rope_type") : NULL;
        if ((ft && ft->t == J_STR && strcmp(ft->str, "default")) ||
            (stp && stp->t == J_STR && strcmp(stp->str, "default")))
            die("encoder/config.json: only the default RoPE is supported");
    }
    M->L = (EncLayer *)xcalloc((size_t)M->layers, sizeof(EncLayer));
    jval *types = json_get(ec, "layer_types");
    int every = (int)num_or(ec, "global_attn_every_n_layers", 3);
    for (int l = 0; l < M->layers; l++) {
        if (types && types->t == J_ARR && types->len == M->layers && types->kids[l]->t == J_STR)
            M->L[l].global = !strcmp(types->kids[l]->str, "full_attention");
        else
            M->L[l].global = every > 0 ? l % every == 0 : 1;
    }
    json_free(ec);

    /* tokenizer: the special tokens by name, from tokenizer_config.json */
    jval *tc = read_json(dir, "tokenizer/tokenizer_config.json");
    const char *cls = special_content(tc, "cls_token"), *sep = special_content(tc, "sep_token");
    const char *mask = special_content(tc, "mask_token"), *pad = special_content(tc, "pad_token");
    if (!cls || !sep || !mask) die("tokenizer_config.json: needs cls_token, sep_token and mask_token");
    M->mask_str = strdup(mask);
    char *cls_s = strdup(cls), *sep_s = strdup(sep), *pad_s = pad ? strdup(pad) : NULL;
    json_free(tc);
    load_tokenizer(M, dir);
    M->cls_id = tok_id_of(&M->tok, cls_s);
    M->sep_id = tok_id_of(&M->tok, sep_s);
    M->mask_id = tok_id_of(&M->tok, M->mask_str);
    M->pad_id = pad_s ? tok_id_of(&M->tok, pad_s) : -1;
    if (M->cls_id < 0 || M->sep_id < 0 || M->mask_id < 0)
        die("tokenizer: cls/sep/mask are not added tokens of tokenizer.json");
    free(cls_s); free(sep_s); free(pad_s);

    /* weights */
    shards S;
    st_init(&S, dir);
    int d = M->d, I = M->inter;
    M->tok_emb = load_f32(&S, "encoder.embeddings.tok_embeddings.weight", (int64_t)M->vocab * d);
    M->emb_norm_w = load_f32(&S, "encoder.embeddings.norm.weight", d);
    M->emb_norm_b = load_opt(&S, "encoder.embeddings.norm.bias", d);
    M->final_norm_w = load_f32(&S, "encoder.final_norm.weight", d);
    M->final_norm_b = load_opt(&S, "encoder.final_norm.bias", d);
    char n[256];
    for (int l = 0; l < M->layers; l++) {
        EncLayer *E = &M->L[l];
#define NAME(fmt) (snprintf(n, sizeof(n), "encoder.layers.%d." fmt, l), n)
        if (l > 0) {
            E->attn_norm_w = load_f32(&S, NAME("attn_norm.weight"), d);
            E->attn_norm_b = load_opt(&S, NAME("attn_norm.bias"), d);
        }
        E->wqkv = load_mat(&S, NAME("attn.Wqkv.weight"), 3 * d, d);
        E->bqkv = load_opt(&S, NAME("attn.Wqkv.bias"), 3 * d);
        E->wo = load_mat(&S, NAME("attn.Wo.weight"), d, d);
        E->bo = load_opt(&S, NAME("attn.Wo.bias"), d);
        E->mlp_norm_w = load_f32(&S, NAME("mlp_norm.weight"), d);
        E->mlp_norm_b = load_opt(&S, NAME("mlp_norm.bias"), d);
        E->wi = load_mat(&S, NAME("mlp.Wi.weight"), 2 * I, d);
        E->bi = load_opt(&S, NAME("mlp.Wi.bias"), 2 * I);
        E->wo2 = load_mat(&S, NAME("mlp.Wo.weight"), d, I);
        E->bo2 = load_opt(&S, NAME("mlp.Wo.bias"), d);
#undef NAME
    }
    M->head_heads = d / 64 > 1 ? d / 64 : 1;        /* DecisionModel: nhead = max(1, d // 64) */
    if (d % M->head_heads) die("head: hidden size %d does not split into %d heads", d, M->head_heads);
    M->H = (HeadLayer *)xcalloc((size_t)(M->head_layers > 0 ? M->head_layers : 1), sizeof(HeadLayer));
    for (int l = 0; l < M->head_layers; l++) {
        HeadLayer *H = &M->H[l];
#define NAME(fmt) (snprintf(n, sizeof(n), "head.layers.%d." fmt, l), n)
        H->n1w = load_f32(&S, NAME("norm1.weight"), d);
        H->n1b = load_f32(&S, NAME("norm1.bias"), d);
        H->n2w = load_f32(&S, NAME("norm2.weight"), d);
        H->n2b = load_f32(&S, NAME("norm2.bias"), d);
        H->in_proj = load_mat(&S, NAME("self_attn.in_proj_weight"), 3 * d, d);
        H->in_proj_b = load_f32(&S, NAME("self_attn.in_proj_bias"), 3 * d);
        H->out_proj = load_mat(&S, NAME("self_attn.out_proj.weight"), d, d);
        H->out_proj_b = load_f32(&S, NAME("self_attn.out_proj.bias"), d);
        H->lin1 = load_mat(&S, NAME("linear1.weight"), 4 * d, d);
        H->lin1_b = load_f32(&S, NAME("linear1.bias"), 4 * d);
        H->lin2 = load_mat(&S, NAME("linear2.weight"), d, 4 * d);
        H->lin2_b = load_f32(&S, NAME("linear2.bias"), d);
#undef NAME
    }
    M->type_emb = load_f32(&S, "type_emb.weight", 3 * d);
    M->sc_ln_w = load_f32(&S, "scorer.0.weight", d);
    M->sc_ln_b = load_f32(&S, "scorer.0.bias", d);
    M->sc1 = load_mat(&S, "scorer.1.weight", d, d);
    M->sc1_b = load_f32(&S, "scorer.1.bias", d);
    M->sc2_w = load_f32(&S, "scorer.3.weight", d);
    float *b = load_f32(&S, "scorer.3.bias", 1);
    M->sc2_b = b[0];
    free(b);
    st_tensor *a1 = st_find(&S, "act_head.0.weight");
    if (!a1 || a1->rank != 2 || a1->shape[1] != d + 4) die("act_head.0.weight: expected [hidden, %d]", d + 4);
    M->act_hidden = (int)a1->shape[0];
    M->act1 = load_mat(&S, "act_head.0.weight", M->act_hidden, d + 4);
    M->act1_b = load_f32(&S, "act_head.0.bias", M->act_hidden);
    st_tensor *a2 = st_find(&S, "act_head.2.weight");
    if (!a2 || a2->rank != 2 || a2->shape[0] != M->n_act || a2->shape[1] != M->act_hidden)
        die("act_head.2.weight: expected [%d, %d] (one row per action of act_costs)", M->n_act, M->act_hidden);
    M->act2_w = load_f32(&S, "act_head.2.weight", (int64_t)M->n_act * M->act_hidden);
    M->act2_b = load_f32(&S, "act_head.2.bias", M->n_act);
    st_destroy(&S);

    int half = M->hd / 2;
    M->rope_g = (float *)xmalloc((size_t)M->max_pos * half * 2 * sizeof(float));
    M->rope_l = (float *)xmalloc((size_t)M->max_pos * half * 2 * sizeof(float));
    rope_table(M->rope_g, M->max_pos, M->hd, M->theta_global);
    rope_table(M->rope_l, M->max_pos, M->hd, M->theta_local);
    if (M->max_len > M->max_pos) M->max_len = M->max_pos;
}

/* -------------------------------------------------------------- tokenizer */

/* tokenizer.json applied to one text: NFC, the added tokens split out
 * (leftmost, longest), then the GPT-2 pre-tokenizer and BPE on what is left
 * (tok.h). Returns the ids; *count their number. */
static int *laya_encode(Laya *M, const char *text, size_t len, int *count)
{
    char *norm = NULL;
    size_t norm_len = len;
    const char *p = text;
    if (M->nfc) {
        if (q38_nfc_normalize(text, len, &norm, &norm_len) != 0) die("NFC normalization failed");
        p = norm;
    }
    int cap = (int)norm_len + 8, n = 0;
    int *out = (int *)xmalloc((size_t)cap * sizeof(int));
    const unsigned char *u = (const unsigned char *)p;
    int i = 0, L = (int)norm_len;
    Tok *T = &M->tok;
    while (i < L) {
        int hit = -1, hit_len = 0, hit_id = -1;
        for (int j = i; j < L && hit < 0; j++) {
            if (!M->added_first[u[j]]) continue;
            for (int k = 0; k < T->nsp; k++) {
                int sl = T->sp[k].len;
                if (sl > 0 && j + sl <= L && !memcmp(u + j, T->sp[k].str, (size_t)sl)) {
                    hit = j; hit_len = sl; hit_id = T->sp[k].id;
                    break;
                }
            }
        }
        int end = hit < 0 ? L : hit;
        if (end > i) pretok_chunk_gpt2(T, u, i, end, out, &n, cap);
        if (hit < 0) break;
        if (n < cap) out[n++] = hit_id;
        i = hit + hit_len;
    }
    free(norm);
    *count = n;
    return out;
}

/* str.replace(needle, " ") */
static char *replace_all(const char *s, const char *needle)
{
    size_t nl = strlen(needle), sl = strlen(s);
    char *out = (char *)xmalloc(sl + 1);
    size_t o = 0;
    for (size_t i = 0; i < sl;) {
        if (nl && i + nl <= sl && !memcmp(s + i, needle, nl)) { out[o++] = ' '; i += nl; }
        else out[o++] = s[i++];
    }
    out[o] = 0;
    return out;
}

/* ------------------------------------------------------------ the sequence */

typedef struct {
    int *ids, n;
    int *markers, n_markers;
    int qtype;
    int state_tokens, state_used;
} Seq;

static void seq_free(Seq *s) { free(s->ids); free(s->markers); memset(s, 0, sizeof(*s)); }

/* common.render_options: the option texts in label-index order. */
static char *render_option(const DecideQuestion *q, int i)
{
    const DecideOption *o = &q->options[i];
    const char *text = o->text && o->text[0] ? o->text : NULL;
    size_t size = strlen(o->label) + (o->text ? strlen(o->text) : 0) + 64;
    char *out = (char *)xmalloc(size);
    if (q->type == DECIDE_CHOICE) {
        if (text) snprintf(out, size, "%s: %s", o->label, text);
        else snprintf(out, size, "%s", o->label);
    } else if (q->type == DECIDE_SCORE) {
        snprintf(out, size, "level %d: %s", i, o->text ? o->text : "");
    } else {
        /* noul: [false, true], each with its default description */
        const char *fallback = i == 0 ? "no, the statement does not hold" : "yes, the statement holds";
        snprintf(out, size, "%s: %s", o->label, text ? text : fallback);
    }
    return out;
}

/* common.build_head + build_sequence:
 *   [CLS] <type> question: <ins> [SEP] [MASK] opt0 [MASK] opt1 ... [SEP] state [SEP]
 * Returns 1, or 0 with the reason when not every option marker fits. */
static int build_sequence(Laya *M, const DecideQuestion *q, const int *state, int n_state,
                          int truncate_left, Seq *out, char *err, size_t cap)
{
    memset(out, 0, sizeof(*out));
    out->qtype = q->type;
    const char *tname = decide_type_name(q->type);
    char *ins = replace_all(q->instructions, M->mask_str);
    size_t hl = strlen(tname) + strlen(ins) + 16;
    char *head_text = (char *)xmalloc(hl);
    snprintf(head_text, hl, "%s question: %s", tname, ins);
    int n_head = 0;
    int *head = laya_encode(M, head_text, strlen(head_text), &n_head);
    free(head_text); free(ins);

    int K = q->n_options;
    int **opt = (int **)xcalloc((size_t)K, sizeof(int *));
    int *olen = (int *)xcalloc((size_t)K, sizeof(int));
    int total = 0;
    for (int i = 0; i < K; i++) {
        char *rendered = render_option(q, i);
        char *clean = replace_all(rendered, M->mask_str);
        size_t cl = strlen(clean);
        char *spaced = (char *)xmalloc(cl + 2);
        spaced[0] = ' ';
        memcpy(spaced + 1, clean, cl + 1);
        int n = 0;
        int *ids = laya_encode(M, spaced, cl + 1, &n);
        if (n > 48) n = 48;                        /* truncation=True, max_length=48 */
        opt[i] = (int *)xmalloc((size_t)(n + 1) * sizeof(int));
        opt[i][0] = M->mask_id;
        memcpy(opt[i] + 1, ids, (size_t)n * sizeof(int));
        olen[i] = n + 1;
        total += n + 1;
        free(ids); free(spaced); free(clean); free(rendered);
    }
    int budget = M->head_max_len - total;
    if (budget < 16) {                             /* shrink every option evenly */
        int per = floordiv(M->head_max_len - 16, K > 1 ? K : 1);
        if (per < 4) per = 4;
        total = 0;
        for (int i = 0; i < K; i++) { if (olen[i] > per) olen[i] = per; total += olen[i]; }
        budget = M->head_max_len - total;
    }
    int keep = budget > 8 ? budget : 8;
    if (n_head > keep) n_head = keep;

    int room_cap = 2 + n_head + total + 1 + n_state + 1;
    int *ids = (int *)xmalloc((size_t)room_cap * sizeof(int));
    int *markers = (int *)xmalloc((size_t)K * sizeof(int));
    int n = 0;
    ids[n++] = M->cls_id;
    memcpy(ids + n, head, (size_t)n_head * sizeof(int)); n += n_head;
    ids[n++] = M->sep_id;
    for (int i = 0; i < K; i++) {
        markers[i] = n;
        memcpy(ids + n, opt[i], (size_t)olen[i] * sizeof(int));
        n += olen[i];
    }
    ids[n++] = M->sep_id;
    int room = M->max_len - n - 1;
    if (room < 0) room = 0;
    int take = n_state < room ? n_state : room;
    const int *from = truncate_left ? state + (n_state - take) : state;
    memcpy(ids + n, from, (size_t)take * sizeof(int)); n += take;
    ids[n++] = M->sep_id;
    if (n > M->max_len) n = M->max_len;
    int kept = 0;
    for (int i = 0; i < K; i++) if (markers[i] < M->max_len) markers[kept++] = markers[i];

    for (int i = 0; i < K; i++) free(opt[i]);
    free(opt); free(olen); free(head);
    out->ids = ids; out->n = n; out->markers = markers; out->n_markers = kept;
    out->state_tokens = n_state; out->state_used = take;
    if (kept != K) {
        snprintf(err, cap, "questions.%s: only %d of its %d option markers fit in max_len=%d with "
                 "head_max_len=%d spent on the question; lower head_max_len, raise max_len, or use "
                 "fewer options", q->id, kept, K, M->max_len, M->head_max_len);
        return 0;
    }
    return 1;
}

/* ------------------------------------------------------------- the forward */

static inline float gelu(float x, int tanh_form)
{
    if (tanh_form) return 0.5f * x * (1.0f + tanhf(0.7978845608028654f * (x + 0.044715f * x * x * x)));
    return 0.5f * x * (1.0f + erff(x * 0.70710678118654752f));
}

/* LayerNorm over rows of `d`: biased variance, eps inside the root. */
static void layernorm(float *out, const float *x, const float *w, const float *b, int rows, int d, float eps)
{
    #pragma omp parallel for schedule(static)
    for (int r = 0; r < rows; r++) {
        const float *xr = x + (size_t)r * d;
        float *o = out + (size_t)r * d;
        double mean = 0, var = 0;
        for (int i = 0; i < d; i++) mean += xr[i];
        mean /= d;
        for (int i = 0; i < d; i++) { double c = xr[i] - mean; var += c * c; }
        var /= d;
        float rstd = (float)(1.0 / sqrt(var + eps));
        for (int i = 0; i < d; i++) {
            float v = (float)(xr[i] - mean) * rstd * w[i];
            o[i] = b ? v + b[i] : v;
        }
    }
}

static void add_rows(float *x, const float *y, size_t n)
{
    #pragma omp parallel for schedule(static)
    for (size_t i = 0; i < n; i++) x[i] += y[i];
}

#ifdef COLI_VULKAN
#include "decide_vk.h"         /* COLI_VULKAN=1: the forward, or its matrices, on a Vulkan device */
#endif

static void gemm(float *Y, const float *X, int M, const QiMat *W, const float *bias)
{
#ifdef COLI_VULKAN
    if (dvk_gemm(Y, X, M, W, bias)) return;   /* matrix by matrix (COLI_VK_CHAIN=0) */
#endif
    qi_gemm(Y, X, M, W, bias);
}

/* Rotate q and k of every row in place: q*cos + rotate_half(q)*sin, positions
 * counted from each sequence's start. */
static void apply_rope(float *qkv, int T, const int *row_pos, int d, int heads, int hd,
                       const float *table, int max_pos)
{
    int half = hd / 2;
    const float *cosv = table, *sinv = table + (size_t)max_pos * half;
    #pragma omp parallel for schedule(static)
    for (int r = 0; r < T; r++) {
        const float *c = cosv + (size_t)row_pos[r] * half, *s = sinv + (size_t)row_pos[r] * half;
        for (int part = 0; part < 2; part++) {
            float *base = qkv + (size_t)r * 3 * d + (size_t)part * d;
            for (int h = 0; h < heads; h++) {
                float *v = base + (size_t)h * hd;
                for (int i = 0; i < half; i++) {
                    float a = v[i], b = v[i + half];
                    v[i] = a * c[i] - b * s[i];
                    v[i + half] = b * c[i] + a * s[i];
                }
            }
        }
    }
}

/* Softmax attention inside each sequence. qkv rows are [q | k | v], each d
 * wide with the heads side by side. window < 0 is full attention; otherwise
 * key j is visible to query i when |i - j| <= window. `only` (optional) marks
 * the query rows whose output is needed; the rest are skipped. */
static void attention(const float *qkv, float *out, int T, int d, int heads, int hd,
                      const int *row_seq, const int *seq_off, int window, const unsigned char *only)
{
    const float scale = 1.0f / sqrtf((float)hd);
    #pragma omp parallel
    {
        int cap = 0;
        for (int r = 0; r < T; r++) {
            int L = seq_off[row_seq[r] + 1] - seq_off[row_seq[r]];
            if (L > cap) cap = L;
        }
        float *score = (float *)xmalloc((size_t)(cap > 0 ? cap : 1) * sizeof(float));
        float *acc = (float *)xmalloc((size_t)hd * sizeof(float));
        #pragma omp for schedule(dynamic, 8)
        for (long task = 0; task < (long)T * heads; task++) {
            int r = (int)(task / heads), h = (int)(task % heads);
            if (only && !only[r]) continue;
            int s = row_seq[r], start = seq_off[s], L = seq_off[s + 1] - start, i = r - start;
            int lo = 0, hi = L - 1;
            if (window >= 0) { lo = i - window < 0 ? 0 : i - window; hi = i + window > L - 1 ? L - 1 : i + window; }
            const float *q = qkv + (size_t)r * 3 * d + (size_t)h * hd;
            float mx = -INFINITY;
            for (int j = lo; j <= hi; j++) {
                const float *k = qkv + (size_t)(start + j) * 3 * d + d + (size_t)h * hd;
                float v = dot_f32_lanes(q, k, hd) * scale;
                score[j - lo] = v;
                if (v > mx) mx = v;
            }
            double sum = 0;
            for (int j = lo; j <= hi; j++) { float e = expf(score[j - lo] - mx); score[j - lo] = e; sum += e; }
            float inv = (float)(1.0 / sum);
            for (int t = 0; t < hd; t++) acc[t] = 0.f;
            for (int j = lo; j <= hi; j++) {
                const float *v = qkv + (size_t)(start + j) * 3 * d + 2 * d + (size_t)h * hd;
                float p = score[j - lo] * inv;
                for (int t = 0; t < hd; t++) acc[t] += p * v[t];
            }
            memcpy(out + (size_t)r * d + (size_t)h * hd, acc, (size_t)hd * sizeof(float));
        }
        free(score); free(acc);
    }
}

typedef struct {
    double *logits;      /* [n_markers] */
    double actions[DECIDE_MAX_ACTIONS];
} SeqOut;

static void forward_tail(Laya *M, Seq *seqs, int S, const float *x, const int *first_row, const int *mark_row,
                         SeqOut *outs);

/* DecisionModel.forward over a batch of sequences. */
static void forward(Laya *M, Seq *seqs, int S, SeqOut *outs)
{
    int d = M->d, I = M->inter;
    int *seq_off = (int *)xmalloc((size_t)(S + 1) * sizeof(int));
    seq_off[0] = 0;
    for (int s = 0; s < S; s++) seq_off[s + 1] = seq_off[s] + seqs[s].n;
    int T = seq_off[S];
    int *row_seq = (int *)xmalloc((size_t)T * sizeof(int));
    int *row_pos = (int *)xmalloc((size_t)T * sizeof(int));
    for (int s = 0; s < S; s++)
        for (int i = 0; i < seqs[s].n; i++) { row_seq[seq_off[s] + i] = s; row_pos[seq_off[s] + i] = i; }

    float *x = (float *)xmalloc((size_t)T * d * sizeof(float));
    float *h = (float *)xmalloc((size_t)T * d * sizeof(float));
    float *att = (float *)xmalloc((size_t)T * d * sizeof(float));
    int wide = 3 * d;                    /* the widest activation: qkv, the GeGLU input or the head FFN */
    if (2 * I > wide) wide = 2 * I;
    if (4 * d > wide) wide = 4 * d;
    float *big = (float *)xmalloc((size_t)T * wide * sizeof(float));
    float *mid = (float *)xmalloc((size_t)T * (I > d ? I : d) * sizeof(float));

    /* embeddings: tok_embeddings, then LayerNorm */
    for (int r = 0; r < T; r++) {
        int id = seqs[row_seq[r]].ids[row_pos[r]];
        if (id < 0 || id >= M->vocab) id = 0;
        memcpy(h + (size_t)r * d, M->tok_emb + (size_t)id * d, (size_t)d * sizeof(float));
    }
    layernorm(x, h, M->emb_norm_w, M->emb_norm_b, T, d, M->eps);

    for (int l = 0; l < M->layers; l++) {
        EncLayer *E = &M->L[l];
        const float *in = x;
        if (E->attn_norm_w) { layernorm(h, x, E->attn_norm_w, E->attn_norm_b, T, d, M->eps); in = h; }
        gemm(big, in, T, &E->wqkv, E->bqkv);
        apply_rope(big, T, row_pos, d, M->heads, M->hd, E->global ? M->rope_g : M->rope_l, M->max_pos);
        attention(big, att, T, d, M->heads, M->hd, row_seq, seq_off, E->global ? -1 : M->window, NULL);
        gemm(h, att, T, &E->wo, E->bo);
        add_rows(x, h, (size_t)T * d);
        layernorm(h, x, E->mlp_norm_w, E->mlp_norm_b, T, d, M->eps);
        gemm(big, h, T, &E->wi, E->bi);
        #pragma omp parallel for schedule(static)
        for (int r = 0; r < T; r++) {
            const float *u = big + (size_t)r * 2 * I;
            float *g = mid + (size_t)r * I;
            for (int j = 0; j < I; j++) g[j] = gelu(u[j], M->gelu_tanh) * u[I + j];
        }
        gemm(h, mid, T, &E->wo2, E->bo2);
        add_rows(x, h, (size_t)T * d);
    }
    layernorm(h, x, M->final_norm_w, M->final_norm_b, T, d, M->eps);
    float *t = x; x = h; h = t;

    /* the decision head: + type_emb, then norm_first TransformerEncoderLayers */
    #pragma omp parallel for schedule(static)
    for (int r = 0; r < T; r++) {
        const float *te = M->type_emb + (size_t)seqs[row_seq[r]].qtype * d;
        float *xr = x + (size_t)r * d;
        for (int i = 0; i < d; i++) xr[i] += te[i];
    }
    int hh = M->head_heads, hhd = d / hh;
    for (int l = 0; l < M->head_layers; l++) {
        HeadLayer *H = &M->H[l];
        layernorm(h, x, H->n1w, H->n1b, T, d, 1e-5f);
        gemm(big, h, T, &H->in_proj, H->in_proj_b);
        attention(big, att, T, d, hh, hhd, row_seq, seq_off, -1, NULL);
        gemm(h, att, T, &H->out_proj, H->out_proj_b);
        add_rows(x, h, (size_t)T * d);
        layernorm(h, x, H->n2w, H->n2b, T, d, 1e-5f);
        gemm(big, h, T, &H->lin1, H->lin1_b);
        #pragma omp parallel for schedule(static)
        for (size_t i = 0; i < (size_t)T * 4 * d; i++) if (big[i] < 0.f) big[i] = 0.f;
        gemm(h, big, T, &H->lin2, H->lin2_b);
        add_rows(x, h, (size_t)T * d);
    }

    int nm = 0;
    for (int s = 0; s < S; s++) nm += seqs[s].n_markers;
    int *mark_row = (int *)xmalloc((size_t)(nm > 0 ? nm : 1) * sizeof(int));
    for (int s = 0, k = 0; s < S; s++)
        for (int i = 0; i < seqs[s].n_markers; i++, k++) mark_row[k] = seq_off[s] + seqs[s].markers[i];
    forward_tail(M, seqs, S, x, seq_off, mark_row, outs);
    free(mark_row);
    free(x); free(h); free(att); free(big); free(mid);
    free(seq_off); free(row_seq); free(row_pos);
}

/* The scorer at the markers and the act head, on the final rows x: sequence s's first
 * row is x[first_row[s]], its markers (all sequences' in order) x[mark_row[k]]. */
static void forward_tail(Laya *M, Seq *seqs, int S, const float *x, const int *first_row, const int *mark_row,
                         SeqOut *outs)
{
    int d = M->d;
    /* scorer at the markers: LayerNorm -> Linear -> GELU -> Linear(d, 1) */
    int nm = 0;
    for (int s = 0; s < S; s++) nm += seqs[s].n_markers;
    float *m = (float *)xmalloc((size_t)(nm > 0 ? nm : 1) * d * sizeof(float));
    float *mn = (float *)xmalloc((size_t)(nm > 0 ? nm : 1) * d * sizeof(float));
    float *m1 = (float *)xmalloc((size_t)(nm > 0 ? nm : 1) * d * sizeof(float));
    for (int k = 0; k < nm; k++)
        memcpy(m + (size_t)k * d, x + (size_t)mark_row[k] * d, (size_t)d * sizeof(float));
    layernorm(mn, m, M->sc_ln_w, M->sc_ln_b, nm, d, 1e-5f);
    gemm(m1, mn, nm, &M->sc1, M->sc1_b);
    for (int s = 0, k = 0; s < S; s++) {
        int K = seqs[s].n_markers;
        outs[s].logits = (double *)xcalloc((size_t)(K > 0 ? K : 1), sizeof(double));
        for (int i = 0; i < K; i++, k++) {
            float *row = m1 + (size_t)k * d;
            for (int j = 0; j < d; j++) row[j] = gelu(row[j], 0);
            outs[s].logits[i] = (double)(dot_f32_lanes(row, M->sc2_w, d) + M->sc2_b);
        }
    }

    /* act head: [pooled h[0], top1, top1 - top2, normalized entropy, k / 255],
     * the distribution being the raw (untempered) softmax of the logits. */
    float *feat = (float *)xmalloc((size_t)S * (d + 4) * sizeof(float));
    float *hid = (float *)xmalloc((size_t)S * M->act_hidden * sizeof(float));
    for (int s = 0; s < S; s++) {
        int K = seqs[s].n_markers;
        float *f = feat + (size_t)s * (d + 4);
        memcpy(f, x + (size_t)first_row[s] * d, (size_t)d * sizeof(float));
        float mx = -INFINITY;
        for (int i = 0; i < K; i++) if ((float)outs[s].logits[i] > mx) mx = (float)outs[s].logits[i];
        float *p = (float *)xmalloc((size_t)(K > 0 ? K : 1) * sizeof(float));
        float sum = 0.f;
        for (int i = 0; i < K; i++) { p[i] = expf((float)outs[s].logits[i] - mx); sum += p[i]; }
        float top1 = 0.f, top2 = 0.f, ent = 0.f;
        for (int i = 0; i < K; i++) {
            p[i] /= sum;
            if (p[i] > top1) { top2 = top1; top1 = p[i]; } else if (p[i] > top2) top2 = p[i];
            ent -= p[i] * logf(p[i] > 1e-9f ? p[i] : 1e-9f);
        }
        float kk = (float)(K < 2 ? 2 : K);
        f[d] = top1; f[d + 1] = top1 - top2; f[d + 2] = ent / logf(kk); f[d + 3] = kk / 255.0f;
        free(p);
    }
    gemm(hid, feat, S, &M->act1, M->act1_b);
    for (int s = 0; s < S; s++) {
        float *row = hid + (size_t)s * M->act_hidden;
        for (int j = 0; j < M->act_hidden; j++) row[j] = gelu(row[j], 0);
        double z[DECIDE_MAX_ACTIONS], mx = -INFINITY, sum = 0;
        for (int a = 0; a < M->n_act; a++) {
            z[a] = (double)(dot_f32_lanes(row, M->act2_w + (size_t)a * M->act_hidden, M->act_hidden) + M->act2_b[a]);
            if (z[a] > mx) mx = z[a];
        }
        for (int a = 0; a < M->n_act; a++) { z[a] = exp(z[a] - mx); sum += z[a]; }
        for (int a = 0; a < M->n_act; a++) outs[s].actions[a] = z[a] / sum;
    }

    free(feat); free(hid); free(m); free(mn); free(m1);
}

#ifdef COLI_VULKAN
/* ------------------------------------------- the forward on the device */

/* forward() as frames of the dense chain (decide_vk.h): the embedding rows go up, the
 * encoder and the head layers run on the device in forward()'s order with its
 * arithmetic (f32 throughout), and the rows forward_tail() reads come back. The
 * norms' weights, the biases, the type embedding and the RoPE tables are uploaded once. */
typedef struct {
    int ok;
    VkcBuf *prm, *tab;
    VkcBuf *x, *h, *big, *mid, *att, *ipos, *rg, *rl, *tid, *down;
    int emb_w, emb_b, fin_w, fin_b, type_emb;
    int *an_w, *an_b, *mn_w, *mn_b, *bqkv, *bo, *bi, *bo2;          /* per encoder layer */
    int *n1w, *n1b, *n2w, *n2b, *inb, *outb, *l1b, *l2b;            /* per head layer */
    int cos_g, sin_g, cos_l, sin_l;
} LayaVk;
static LayaVk g_lvk;

/* Every matrix's device copy now (the first request then pays nothing), and with the
 * forward on the device, its parameters and tables. */
static void laya_vk_setup(Laya *M)
{
    if (!g_dvk.ready || (!g_dvk.chain && !g_dvk.dense)) return;   /* nothing of it on the device */
    g_dvk.f16 = 1;   /* the release stores F16: the device reads those values in half the bytes */
    int n = 0, want = 0;
    for (int l = 0; l < M->layers; l++) {
        EncLayer *E = &M->L[l];
        const QiMat *w[4] = {&E->wqkv, &E->wo, &E->wi, &E->wo2};
        for (int k = 0; k < 4; k++, want++) n += dvk_tensor(w[k]) != NULL;
    }
    for (int l = 0; l < M->head_layers; l++) {
        HeadLayer *H = &M->H[l];
        const QiMat *w[4] = {&H->in_proj, &H->out_proj, &H->lin1, &H->lin2};
        for (int k = 0; k < 4; k++, want++) n += dvk_tensor(w[k]) != NULL;
    }
    want += 2;
    n += dvk_tensor(&M->sc1) != NULL;
    n += dvk_tensor(&M->act1) != NULL;
    size_t bytes = 0, tensors = 0;
    coli_vk_mem_info(&bytes, &tensors);
    fprintf(stderr, "[VK] laya: %d of %d matrices on the device (%.1f MiB; %u in f16, every value the checkpoint's, "
            "%u in f32)\n", n, want, bytes / 1048576.0, g_dvk.n_f16, g_dvk.n_f32);
    if (!g_dvk.chain) return;
    if (n != want || M->hd > 128 || M->d / M->head_heads > 128 || M->d > 4096) {
        fprintf(stderr, "[VK] laya: the forward stays on the CPU (%s)\n",
                n != want ? "a matrix did not reach the device" : "a geometry past the chain's shaders");
        g_dvk.chain = 0;
        g_dvk.dense = coli_vk_dense();
        return;
    }
    LayaVk *V = &g_lvk;
    DvkPrm P = {0};
    int L = M->layers, HL = M->head_layers > 0 ? M->head_layers : 1, d = M->d;
    int **lv[8] = {&V->an_w, &V->an_b, &V->mn_w, &V->mn_b, &V->bqkv, &V->bo, &V->bi, &V->bo2};
    int **hv[8] = {&V->n1w, &V->n1b, &V->n2w, &V->n2b, &V->inb, &V->outb, &V->l1b, &V->l2b};
    for (int k = 0; k < 8; k++) { *lv[k] = (int *)xcalloc((size_t)L, sizeof(int)); *hv[k] = (int *)xcalloc((size_t)HL, sizeof(int)); }
    V->emb_w = dvk_prm_add(&P, M->emb_norm_w, d);
    V->emb_b = dvk_prm_add(&P, M->emb_norm_b, d);
    V->fin_w = dvk_prm_add(&P, M->final_norm_w, d);
    V->fin_b = dvk_prm_add(&P, M->final_norm_b, d);
    V->type_emb = dvk_prm_add(&P, M->type_emb, (size_t)3 * d);
    for (int l = 0; l < L; l++) {
        EncLayer *E = &M->L[l];
        V->an_w[l] = dvk_prm_add(&P, E->attn_norm_w, d);
        V->an_b[l] = dvk_prm_add(&P, E->attn_norm_b, d);
        V->mn_w[l] = dvk_prm_add(&P, E->mlp_norm_w, d);
        V->mn_b[l] = dvk_prm_add(&P, E->mlp_norm_b, d);
        V->bqkv[l] = dvk_prm_add(&P, E->bqkv, (size_t)3 * d);
        V->bo[l] = dvk_prm_add(&P, E->bo, d);
        V->bi[l] = dvk_prm_add(&P, E->bi, (size_t)2 * M->inter);
        V->bo2[l] = dvk_prm_add(&P, E->bo2, d);
    }
    for (int l = 0; l < M->head_layers; l++) {
        HeadLayer *H = &M->H[l];
        V->n1w[l] = dvk_prm_add(&P, H->n1w, d);
        V->n1b[l] = dvk_prm_add(&P, H->n1b, d);
        V->n2w[l] = dvk_prm_add(&P, H->n2w, d);
        V->n2b[l] = dvk_prm_add(&P, H->n2b, d);
        V->inb[l] = dvk_prm_add(&P, H->in_proj_b, (size_t)3 * d);
        V->outb[l] = dvk_prm_add(&P, H->out_proj_b, d);
        V->l1b[l] = dvk_prm_add(&P, H->lin1_b, (size_t)4 * d);
        V->l2b[l] = dvk_prm_add(&P, H->lin2_b, d);
    }
    V->prm = dvk_prm_upload(&P);
    free(P.v);
    DvkPrm T = {0};
    size_t tn = (size_t)M->max_pos * (M->hd / 2);
    V->cos_g = dvk_prm_add(&T, M->rope_g, tn);
    V->sin_g = dvk_prm_add(&T, M->rope_g + tn, tn);
    V->cos_l = dvk_prm_add(&T, M->rope_l, tn);
    V->sin_l = dvk_prm_add(&T, M->rope_l + tn, tn);
    V->tab = dvk_prm_upload(&T);
    free(T.v);
    if (!V->prm || !V->tab) {
        fprintf(stderr, "[VK] laya: the forward stays on the CPU (no device memory for its parameters)\n");
        g_dvk.chain = 0;
        g_dvk.dense = coli_vk_dense();
        return;
    }
    V->ok = 1;
}

/* x = LN(x + h) with the sum kept in x, or LN(x) alone; into y */
static int lvk_norm(VkcBuf *x, VkcBuf *h, VkcBuf *y, int T, int d, int w, int b, int add, float eps)
{
    VkcEncNorm p = {0, T, d, 0, d, 0, d, 0, d, w, b, add ? VKC_ENC_ADD | VKC_ENC_SUM : 0, eps};
    return vkc_enc_norm(x, h, g_lvk.prm, y, &p);
}
static int lvk_lin(const QiMat *W, VkcBuf *x, VkcBuf *y, int T, int bias, int act)
{
    ColiVkTensor *t = dvk_tensor(W);
    if (!t || !vkc_matmul(t, x, 0, y, 0, T)) return 0;
    if (bias < 0 && act == VKC_ENC_ACT_NONE) return 1;
    VkcEncBias p = {0, T * W->N, W->N, 0, W->N, bias, act};
    return vkc_enc_bias(y, g_lvk.prm, &p);
}

/* forward() for the S sequences on the device; 0 = the caller runs forward(). */
static int laya_vk_forward(Laya *M, Seq *seqs, int S, SeqOut *outs)
{
    LayaVk *V = &g_lvk;
    if (!g_dvk.chain || !V->ok || !vkc_ready()) return 0;
    double t0 = dvk_now_ms();
    int d = M->d, I = M->inter, T = 0, nm = 0;
    for (int s = 0; s < S; s++) { T += seqs[s].n; nm += seqs[s].n_markers; }
    if (T < 1 || T > 65535) return 0;
    int wide = 3 * d;
    if (2 * I > wide) wide = 2 * I;
    if (4 * d > wide) wide = 4 * d;
    int nread = S + nm;
    size_t fd = sizeof(float), id = sizeof(int);
    if (!vkc_reserve(&V->x, (size_t)T * d * fd, VKC_DEV) || !vkc_reserve(&V->h, (size_t)T * d * fd, VKC_DEV) ||
        !vkc_reserve(&V->att, (size_t)T * d * fd, VKC_DEV) || !vkc_reserve(&V->big, (size_t)T * wide * fd, VKC_DEV) ||
        !vkc_reserve(&V->mid, (size_t)T * (I > d ? I : d) * fd, VKC_DEV) ||
        !vkc_reserve(&V->ipos, (size_t)T * id, VKC_DEV) || !vkc_reserve(&V->rg, (size_t)2 * T * id, VKC_DEV) ||
        !vkc_reserve(&V->rl, (size_t)2 * T * id, VKC_DEV) || !vkc_reserve(&V->tid, (size_t)T * id, VKC_DEV) ||
        !vkc_reserve(&V->down, (size_t)nread * d * fd, VKC_DOWN))
        return 0;
    /* the host's part: the embedding rows, and per row its position, its question type
     * and the rows its attention sees (the sequence; the window inside it) */
    float *emb = (float *)xmalloc((size_t)T * d * fd);
    int *pos = (int *)xmalloc((size_t)T * id), *tid = (int *)xmalloc((size_t)T * id);
    int *rg = (int *)xmalloc((size_t)2 * T * id), *rl = (int *)xmalloc((size_t)2 * T * id);
    VkcRegion *reg = (VkcRegion *)xmalloc((size_t)nread * sizeof(VkcRegion));
    for (int s = 0, r = 0, k = S; s < S; s++) {
        int start = r, L = seqs[s].n;
        reg[s] = (VkcRegion){(size_t)s * d, (size_t)start * d, (size_t)d};
        for (int i = 0; i < seqs[s].n_markers; i++, k++)
            reg[k] = (VkcRegion){(size_t)k * d, (size_t)(start + seqs[s].markers[i]) * d, (size_t)d};
        for (int i = 0; i < L; i++, r++) {
            int id_ = seqs[s].ids[i];
            if (id_ < 0 || id_ >= M->vocab) id_ = 0;
            memcpy(emb + (size_t)r * d, M->tok_emb + (size_t)id_ * d, (size_t)d * fd);
            pos[r] = i; tid[r] = seqs[s].qtype;
            rg[2 * r] = start; rg[2 * r + 1] = start + L - 1;
            int lo = i - M->window < 0 ? 0 : i - M->window, hi = i + M->window > L - 1 ? L - 1 : i + M->window;
            rl[2 * r] = start + lo; rl[2 * r + 1] = start + hi;
        }
    }
    VkcBuf *x = V->x, *h = V->h, *big = V->big, *mid = V->mid, *att = V->att;
    int ok = vkc_begin() && vkc_write(h, 0, emb, (size_t)T * d * fd) && vkc_write(V->ipos, 0, pos, (size_t)T * id) &&
             vkc_write(V->tid, 0, tid, (size_t)T * id) && vkc_write(V->rg, 0, rg, (size_t)2 * T * id) &&
             vkc_write(V->rl, 0, rl, (size_t)2 * T * id);
    free(emb); free(pos); free(tid); free(rg); free(rl);
    float eps = M->eps;
    ok = ok && lvk_norm(h, NULL, x, T, d, V->emb_w, V->emb_b, 0, eps);
    float scale = 1.0f / sqrtf((float)M->hd);
    for (int l = 0; ok && l < M->layers; l++) {
        EncLayer *E = &M->L[l];
        /* layer 0 has no attention norm; the others' norm joined the previous add below */
        VkcBuf *in = E->attn_norm_w ? h : x;
        if (l == 0 && E->attn_norm_w) ok = lvk_norm(x, NULL, h, T, d, V->an_w[l], V->an_b[l], 0, eps);
        ok = ok && lvk_lin(&E->wqkv, in, big, T, V->bqkv[l], VKC_ENC_ACT_NONE);
        VkcEncRope rp = {0, T * M->heads * M->hd, M->heads, M->hd, 0, 3 * d, d,
                         E->global ? V->cos_g : V->cos_l, E->global ? V->sin_g : V->sin_l};
        ok = ok && vkc_enc_rope(big, V->tab, V->ipos, &rp);
        VkcEncAttn ap = {0, T, M->heads, M->hd, 0, d, 2 * d, 3 * d, 0, d, 0, 0, 0, 0, 0, scale, 0};
        ok = ok && vkc_enc_attn(big, NULL, NULL, att, E->global ? V->rg : V->rl, NULL, NULL, &ap);
        ok = ok && lvk_lin(&E->wo, att, h, T, V->bo[l], VKC_ENC_ACT_NONE);
        ok = ok && lvk_norm(x, h, h, T, d, V->mn_w[l], V->mn_b[l], 1, eps);     /* x += h; h = mlp_norm(x) */
        ok = ok && lvk_lin(&E->wi, h, big, T, -1, VKC_ENC_ACT_NONE);
        VkcEncGeglu gp = {0, T * I, I, 0, 2 * I, 0, I, V->bi[l],
                          M->gelu_tanh ? VKC_ENC_ACT_GELU_TANH : VKC_ENC_ACT_GELU};
        ok = ok && vkc_enc_geglu(big, V->prm, mid, &gp);
        ok = ok && lvk_lin(&E->wo2, mid, h, T, V->bo2[l], VKC_ENC_ACT_NONE);
        if (l + 1 < M->layers && M->L[l + 1].attn_norm_w)                        /* x += h; h = attn_norm(x) */
            ok = ok && lvk_norm(x, h, h, T, d, V->an_w[l + 1], V->an_b[l + 1], 1, eps);
        else if (l + 1 < M->layers) {
            VkcEw ep = {VKC_EW_ADD, T * d, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.f};
            ok = ok && vkc_ew(x, x, h, NULL, NULL, &ep);
        }
    }
    /* x += h; the final norm into h, which carries the stream from here (forward()'s swap) */
    ok = ok && lvk_norm(x, h, h, T, d, V->fin_w, V->fin_b, 1, eps);
    { VkcBuf *t = x; x = h; h = t; }
    VkcEncAddRow tp = {0, T * d, d, 0, d, V->type_emb};
    ok = ok && vkc_enc_addrow(x, V->prm, V->tid, &tp);
    int hh = M->head_heads, hhd = d / hh;
    float hscale = 1.0f / sqrtf((float)hhd);
    for (int l = 0; ok && l < M->head_layers; l++) {
        HeadLayer *H = &M->H[l];
        if (l == 0) ok = lvk_norm(x, NULL, h, T, d, V->n1w[l], V->n1b[l], 0, 1e-5f);
        ok = ok && lvk_lin(&H->in_proj, h, big, T, V->inb[l], VKC_ENC_ACT_NONE);
        VkcEncAttn ap = {0, T, hh, hhd, 0, d, 2 * d, 3 * d, 0, d, 0, 0, 0, 0, 0, hscale, 0};
        ok = ok && vkc_enc_attn(big, NULL, NULL, att, V->rg, NULL, NULL, &ap);
        ok = ok && lvk_lin(&H->out_proj, att, h, T, V->outb[l], VKC_ENC_ACT_NONE);
        ok = ok && lvk_norm(x, h, h, T, d, V->n2w[l], V->n2b[l], 1, 1e-5f);     /* x += h; h = norm2(x) */
        ok = ok && lvk_lin(&H->lin1, h, big, T, V->l1b[l], VKC_ENC_ACT_RELU);
        ok = ok && lvk_lin(&H->lin2, big, h, T, V->l2b[l], VKC_ENC_ACT_NONE);
        if (l + 1 < M->head_layers)                                             /* x += h; h = norm1(x) */
            ok = ok && lvk_norm(x, h, h, T, d, V->n1w[l + 1], V->n1b[l + 1], 1, 1e-5f);
        else {
            VkcEw ep = {VKC_EW_ADD, T * d, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.f};
            ok = ok && vkc_ew(x, x, h, NULL, NULL, &ep);
        }
    }
    ok = ok && vkc_copy_regions(V->down, x, reg, nread) && vkc_submit(1);
    free(reg);
    if (!ok) {
        if (vkc_lost()) fprintf(stderr, "[VK] laya: the device was lost: the forward runs on the CPU\n");
        else vkc_finish();
        return 0;
    }
    int *first = (int *)xmalloc((size_t)S * id), *mark = (int *)xmalloc((size_t)(nm > 0 ? nm : 1) * id);
    for (int s = 0; s < S; s++) first[s] = s;
    for (int k = 0; k < nm; k++) mark[k] = S + k;
    forward_tail(M, seqs, S, (const float *)vkc_ptr(V->down), first, mark, outs);
    free(first); free(mark);
    g_dvk.fwd_dev++; g_dvk.rows_dev += (unsigned long long)T;
    g_dvk.dev_ms += dvk_now_ms() - t0;
    return 1;
}
#endif

/* ------------------------------------------------------------ calibration */

/* common.temp_bucket */
static double temperature_for(Laya *M, int type, int k)
{
    const char *size = k <= 2 ? "2" : k <= 5 ? "3-5" : k <= 10 ? "6-10" : "11+";
    char key[32];
    snprintf(key, sizeof(key), "%s:%s", decide_type_name(type), size);
    for (int i = 0; i < M->n_buckets; i++) if (!strcmp(M->bucket_key[i], key)) return M->bucket_t[i];
    return M->temperature[type];
}

/* ------------------------------------------------------- the DECIDE call */

typedef struct {
    Laya *M;
    int dump_ids;
    Seq *last;           /* the sequences of the last record, for --records --ids */
    int last_n;
} LayaCtx;

/* Most rows one forward takes before the batch is split: the activations of a
 * batch are about 12 * hidden floats per row. */
#define LAYA_MAX_BATCH_ROWS 16384

static int laya_decide(void *opaque, const DecideRecord *rec, DecideAnswer *answers,
                       int *input_tokens, char *err, size_t cap)
{
    LayaCtx *ctx = (LayaCtx *)opaque;
    Laya *M = ctx->M;
    for (int i = 0; i < ctx->last_n; i++) seq_free(&ctx->last[i]);
    free(ctx->last); ctx->last = NULL; ctx->last_n = 0;

    char *state = replace_all(rec->state, M->mask_str);
    int n_state = 0;
    int *state_ids = laya_encode(M, state, strlen(state), &n_state);
    free(state);
    /* Agent._encode_state: a list (a conversation) keeps its newest turns */
    int truncate_left = rec->state_type == DECIDE_STATE_ARRAY;
    int Q = rec->n_questions;
    Seq *seqs = (Seq *)xcalloc((size_t)Q, sizeof(Seq));
    for (int q = 0; q < Q; q++) {
        if (!build_sequence(M, &rec->questions[q], state_ids, n_state, truncate_left, &seqs[q], err, cap)) {
            for (int i = 0; i <= q; i++) seq_free(&seqs[i]);
            free(seqs); free(state_ids);
            return 0;
        }
    }
    free(state_ids);
    SeqOut *outs = (SeqOut *)xcalloc((size_t)Q, sizeof(SeqOut));
    for (int start = 0; start < Q;) {
        int end = start, rows = 0;
        while (end < Q && (end == start || rows + seqs[end].n <= LAYA_MAX_BATCH_ROWS)) rows += seqs[end++].n;
#ifdef COLI_VULKAN
        if (laya_vk_forward(M, seqs + start, end - start, outs + start)) { start = end; continue; }
        if (g_dvk.chain) g_dvk.fwd_cpu++;
#endif
        forward(M, seqs + start, end - start, outs + start);
        start = end;
    }
#ifdef COLI_VULKAN
    dvk_report();
#endif
    int total = 0;
    for (int q = 0; q < Q; q++) {
        DecideAnswer *a = &answers[q];
        int K = seqs[q].n_markers;
        a->n = K;
        a->logits = (double *)xcalloc((size_t)K, sizeof(double));
        a->probs = (double *)xcalloc((size_t)K, sizeof(double));
        a->temperature = temperature_for(M, seqs[q].qtype, K);
        double mx = -INFINITY, sum = 0;
        for (int i = 0; i < K; i++) {
            a->logits[i] = outs[q].logits[i];
            double z = (double)(float)outs[q].logits[i] / a->temperature;
            a->probs[i] = z;
            if (z > mx) mx = z;
        }
        for (int i = 0; i < K; i++) { a->probs[i] = exp(a->probs[i] - mx); sum += a->probs[i]; }
        for (int i = 0; i < K; i++) a->probs[i] /= sum;
        a->n_actions = M->n_act;
        for (int i = 0; i < M->n_act; i++) { a->action_names[i] = M->action_names[i]; a->actions[i] = outs[q].actions[i]; }
        a->tokens = seqs[q].n;
        a->state_tokens = seqs[q].state_tokens;
        a->state_dropped = seqs[q].state_tokens - seqs[q].state_used;
        total += seqs[q].n;
        free(outs[q].logits);
    }
    free(outs);
    *input_tokens = total;
    if (ctx->dump_ids) { ctx->last = seqs; ctx->last_n = Q; }
    else { for (int q = 0; q < Q; q++) seq_free(&seqs[q]); free(seqs); }
    return 1;
}

/* ------------------------------------------------------------------- main */

static double rss_gb(void)
{
#ifdef __linux__
    FILE *f = fopen("/proc/self/statm", "r");
    long pages = 0, resident = 0;
    if (f) { if (fscanf(f, "%ld %ld", &pages, &resident) != 2) resident = 0; fclose(f); }
    return resident * (double)sysconf(_SC_PAGESIZE) / 1e9;
#else
    return 0.0;
#endif
}

static void print_json_string(FILE *out, const char *s)
{
    DecideBuf b = {0};
    decide_buf_string(&b, s);
    fputs(b.data ? b.data : "\"\"", out);
    free(b.data);
}

static int run_records(Laya *M, const char *path, int dump_ids)
{
    char *text = read_text(path);
    if (!text) die("cannot read %s", path);
    jval *root = json_parse_checked(text);
    if (!root || root->t != J_ARR) die("%s: expected a JSON array of records", path);
    LayaCtx ctx = {M, dump_ids, NULL, 0};
    /* Each record is the text of a DECIDE payload and goes through
     * decide_record_parse, the path the serve loop takes. */
    for (int i = 0; i < root->len; i++) {
        jval *r = root->kids[i];
        jval *payload = json_get(r, "payload");
        if (!payload || payload->t != J_STR) die("%s: record %d has no string \"payload\"", path, i);
        DecideRecord rec;
        char reason[1024];
        if (!decide_record_parse(payload->str, &rec, reason, sizeof(reason))) {
            printf("{\"error\":"); print_json_string(stdout, reason); printf("}\n");
            continue;
        }
        DecideAnswer *answers = (DecideAnswer *)xcalloc((size_t)rec.n_questions, sizeof(DecideAnswer));
        int tokens = 0;
        double t0 = now_ms();
        int ok = laya_decide(&ctx, &rec, answers, &tokens, reason, sizeof(reason));
        double ms = now_ms() - t0;
        if (!ok) {
            printf("{\"error\":"); print_json_string(stdout, reason); printf("}\n");
        } else {
            size_t len = 0;
            char *json = decide_answers_json(&rec, answers, tokens, ms, &len);
            if (dump_ids && json && len > 1) {
                json[len - 1] = 0;                 /* reopen the object */
                fputs(json, stdout);
                printf(",\"sequences\":[");
                for (int q = 0; q < ctx.last_n; q++) {
                    printf("%s{\"ids\":[", q ? "," : "");
                    for (int t = 0; t < ctx.last[q].n; t++) printf("%s%d", t ? "," : "", ctx.last[q].ids[t]);
                    printf("],\"markers\":[");
                    for (int t = 0; t < ctx.last[q].n_markers; t++) printf("%s%d", t ? "," : "", ctx.last[q].markers[t]);
                    printf("]}");
                }
                printf("]}\n");
            } else if (json) {
                fputs(json, stdout); fputc('\n', stdout);
            }
            free(json);
        }
        fflush(stdout);
        decide_answers_free(answers, rec.n_questions);
        free(answers);
        decide_record_free(&rec);
    }
    json_free(root);
    free(text);
    return 0;
}

static int run_tokenize(Laya *M, const char *path)
{
    char *text = read_text(path);
    if (!text) die("cannot read %s", path);
    jval *root = json_parse_checked(text);
    if (!root || root->t != J_ARR) die("%s: expected a JSON array of strings", path);
    printf("[");
    for (int i = 0; i < root->len; i++) {
        jval *s = root->kids[i];
        if (s->t != J_STR) die("%s: entry %d is not a string", path, i);
        int n = 0;
        int *ids = laya_encode(M, s->str, strlen(s->str), &n);
        printf("%s[", i ? "," : "");
        for (int t = 0; t < n; t++) printf("%s%d", t ? "," : "", ids[t]);
        printf("]");
        free(ids);
    }
    printf("]\n");
    json_free(root);
    free(text);
    return 0;
}

int main(int argc, char **argv)
{
    const char *model = NULL, *records = NULL, *tokenize = NULL;
    int dump_ids = 0;
    for (int i = 1; i < argc; i++) {
        if (!strcmp(argv[i], "--model") && i + 1 < argc) model = argv[++i];
        else if (!strcmp(argv[i], "--records") && i + 1 < argc) records = argv[++i];
        else if (!strcmp(argv[i], "--tokenize") && i + 1 < argc) tokenize = argv[++i];
        else if (!strcmp(argv[i], "--ids")) dump_ids = 1;
        else if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-h")) {
            printf("usage: SERVE=1 SNAP=<dir> laya            serve DECIDE on stdin/stdout\n"
                   "       laya --model DIR --records FILE [--ids]\n"
                   "       laya --model DIR --tokenize FILE\n");
            return 0;
        }
        /* a bare number is the gateway's cache-cap argument: nothing to cache here */
    }
    const char *serve = getenv("SERVE");
    int serving = serve && !strcmp(serve, "1") && !records && !tokenize;
    if (!model) model = getenv("SNAP");
    if (!model) die("no model: pass --model DIR or set SNAP");
    if (serving) coli_serve_stdio_init();
    coli_omp_tune_threads("laya");          /* physical cores: SMT halves the GEMMs here too */
    double t0 = now_ms();
    Laya *M = (Laya *)xcalloc(1, sizeof(Laya));
    load_model(M, model);
    fprintf(stderr, "[laya] %s: ModernBERT %d layers x %d, %d heads; head %d layers; max_len %d "
            "(head %d); loaded in %.1f s\n", M->model_name, M->layers, M->d, M->heads,
            M->head_layers, M->max_len, M->head_max_len, (now_ms() - t0) / 1e3);
#ifdef COLI_VULKAN
    if (!tokenize) { dvk_init("laya"); laya_vk_setup(M); }   /* COLI_VULKAN=1: the device */
#endif
    if (tokenize) return run_tokenize(M, tokenize);
    if (records) return run_records(M, records, dump_ids);
    if (!serving) die("nothing to do: set SERVE=1, or pass --records / --tokenize");
    int slots = getenv("KV_SLOTS") ? atoi(getenv("KV_SLOTS")) : 1;
    if (slots < 1) slots = 1;
    char caps[256];
    snprintf(caps, sizeof(caps), "decide=1 chat=0 max_len=%d head_max_len=%d", M->max_len, M->head_max_len);
    LayaCtx ctx = {M, 0, NULL, 0};
    return decide_serve(stdin, stdout, laya_decide, &ctx, caps, rss_gb(), slots);
}