Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
/* Pure-C inference engine for Thinking Machines "Inkling" (text-only), Stage A.
* Goal, like olmoe.c before GLM-5.2: reproduce the EXACT token ids of the HF
* transformers reference (ref_inkling.json from tools/make_tiny_inkling.py)
* to validate the core math before scaling to the 975B checkpoint.
*
* Architecture (vs glm.c's MLA/RoPE/DSA — shares almost nothing):
* - hybrid attention: sliding-window layers (window=512, 16 KV heads) and
* global layers (8 KV heads) interleaved 5:1; conventional GQA, no RoPE
* - learned relative-position bias: r_proj(x) mixes a per-layer bank
* proj[d_rel, rel_extent] into one bias per backward distance
* - log-length scaling tau on global layers past n_floor tokens
* - depthwise-causal short convs (kernel 4, residual inside, fp32):
* on K and V inside attention, after attention, and after the MLP
* - MoE: sigmoid router + loss-free bias for top-k selection; combine
* weights are sigmoids of the raw logits jointly normalized over
* topk routed + n_shared shared experts, x route_scale x global_scale
* - logits: hidden / logits_mup_width_multiplier, sliced to unpadded vocab
*
* Dense weights (attn, norms, convs, router, shared experts, dense MLP)
* resident in RAM as f32; routed experts streamed from disk per-expert out
* of the fused [E, 2I, D] / [E, D, I] tensors, LRU-cached, optionally
* int-quantized (bits=0 keeps them f32 for bit-exact oracle validation).
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <time.h>
#include <pthread.h>
#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
#include <sys/resource.h>
#include <sys/select.h> /* serve-loop stdin poll (POSIX); inkling serves on Linux */
#endif
#include "cli_args.h"
#include "st.h"
#include "tok.h"
#ifdef _OPENMP
#include <omp.h>
#endif
#include "omp_tune.h"
#include "route_trace.h"
#include "kv_prefix.h"
#include "pin_pool.h" /* KV prefix reuse (shared) */ /* shared routing telemetry (#700) */
#include "serve_codec.h"
#include "serve_budget.h"
#ifdef COLI_SEGMENT_ADAPTER
#include "segment_runtime.h"
#include "segment_adapters.h"
#include "segment_adapter_internal.h"
#endif
#ifdef COLI_EDGE_ADAPTER
#include "edge_runtime.h"
#include "edge_adapters.h"
#include "edge_tok_internal.h"
#endif
#ifdef COLI_CUDA
#include "backend_cuda_ink.h"
static int g_cuda = 0;
#endif
#ifdef COLI_METAL
/* Apple-GPU expert MoE (opt-in, COLI_METAL=1). Reuses colibri's batched
* coli_metal_moe_block: inkling's container int4 (nibble-packed, -8 offset,
* per-row scales) is bit-identical to the Metal fmt=2 kernel, and int8 to
* fmt=1. Expert slots live in page-aligned per-layer slabs registered once
* for zero-copy resolve — unified memory, no upload. Attention and the dense
* path stay on the CPU (at high hit rates ~90% of decode is expert matmul). */
#include "backend_metal.h"
static int g_metal = 0;
#endif
#ifdef COLI_VULKAN
/* Vulkan (opt-in, VK=1 build + COLI_VULKAN=1): the RESIDENT matrices, in the form
* they are kept in RAM, uploaded on first use -- the dense-int4g64 container's int8
* (fmt 1) and int4-g64 (fmt 4), and the f32 (fmt 10) and bf16 (fmt 11) residents of
* an unconverted snapshot -- and the routed experts through the shared expert tier
* (vk_tier.h, ink_vk_tier_start). The embedding row lookups stay on the CPU; CUDA
* keeps its own bf16 residents, and with CUDA (or Metal) on the tier stays off. */
#include "backend_vulkan.h"
#include "vk_tier.h"
#include "vk_chain.h"
static int g_vk_ready = 0;
static int g_vk_tier_try = 0; /* the device opened with the expert tier to be tried */
static int g_vk_chain = 0; /* COLI_VK_CHAIN decided on, and the chain's pipelines are up (inkling_chain.h) */
static int g_ink_dho = 0; /* COLI_VK_DENSE_HOST: the dense matrices on the device only (ink_dho_place) */
/* The partial chain's fit (inkc_fit_start, inkling_chain.h): the first n of L layers on the
* device; L = 0 while there is none (the chain off, or one that declines as before). */
static VkcFit g_inkc_fit;
#endif
#ifndef COLI_VULKAN /* exclusive RAM/VRAM (vk_tier.h) is the Vulkan build's: no device holds an expert */
static inline int vkt_ram_first(int layer, int eid) { (void)layer; (void)eid; return 0; }
static inline void vkt_ram_gave(void) {}
#endif
#if defined(__APPLE__)
#include <mach/mach.h>
#include <sys/sysctl.h>
#endif
#define MAXL 256
/* ---------- config ---------- */
typedef struct {
int hidden, n_layers, vocab, unpad_vocab;
int n_heads, n_kv, head_dim; /* global ("hybrid") layers */
int swa_heads, swa_kv, swa_hd; /* sliding ("hybrid_sliding") layers */
int window, d_rel, rel_extent, conv_k;
double log_floor; /* <=0: log scaling off */
float log_alpha;
int n_experts, topk, n_shared, moe_inter, dense_inter;
int eos;
float eps, route_scale, mup;
int audio_tok, mel_bins, mel_vocab; /* DMel audio input (TMLv0 <|audio|> placeholder) */
unsigned char local[MAXL]; /* 1 = sliding-window layer */
unsigned char sparse[MAXL]; /* 1 = MoE layer, 0 = dense MLP */
} Cfg;
/* per-layer dims that depend on the attention type */
#define L_HEADS(c,i) ((c)->local[i] ? (c)->swa_heads : (c)->n_heads)
#define L_KV(c,i) ((c)->local[i] ? (c)->swa_kv : (c)->n_kv)
#define L_HD(c,i) ((c)->local[i] ? (c)->swa_hd : (c)->head_dim)
#define L_EXT(c,i) ((c)->local[i] ? (c)->window : (c)->rel_extent)
/* ---------- resident weights ----------
* Large matmul weights keep their on-disk dtype in RAM: bf16 for the real
* 975B checkpoint (f32 residents would need ~172 GB, over sabre's 187),
* f32 for the tiny oracle (bit-exact validation). Under CUDA, bf16 tensors
* move to VRAM (dev set, host freed): decode reads ~35 GB of residents per
* token, so this trades the DDR5 bandwidth wall for VRAM bandwidth AND
* frees the same RAM for the expert cache. */
/* q4/qs: densa pre-quantizzata int4 group-scaled (gs=64) da un container
* separato, opzionale — vedi load_w. La densa bf16 di Inkling e' 49.4 GB e non
* entra in 25 GB (il caricamento la espande pure a f32, ~99 GB al picco: e' li'
* l'OOM). A int4-gs64 diventa ~15 GB. Nessun campo q4 = comportamento invariato. */
typedef struct { float *f; uint16_t *h; void *dev;
uint8_t *q4; /* int4 nibble-packed (qbits=4) o int8 (qbits=8) */
float *qs; /* scale: [rows*ng] se qbits=4, [rows] se qbits=8 */
int gs, qbits; int64_t qn; /* qn = byte in q4, per il guard OOB */
#ifdef COLI_VULKAN
void *vk; /* InkVk of the loaded tensor, shared by its wt_off_i
* views (NULL = CPU only) */
int64_t vk_view; /* this view's first element in the tensor (wt_off_i):
* the key of its device copy, whatever the host holds */
const char *vk_name;/* the tensor's name, to read it back (ink_dho_reload) */
#endif
} Wt;
typedef struct {
float *in_ln, *post_ln;
Wt q, k, v, r, o; /* projections */
float *qn, *kn; /* per-head rmsnorm [head_dim] */
float *relp; /* [d_rel, ext] bias bank */
float *k_cw, *v_cw, *a_cw, *m_cw; /* sconv weights, [C*K] depthwise */
/* dense layers */
Wt dg, du, dd; float dgs;
/* MoE layers */
float *router, *rbias, rgs; /* [E+ns, D], [E], scalar */
Wt sh_g, sh_u, sh_d; /* shared experts [ns][I,D] etc. */
} Layer;
/* ---------- routed-expert cache: LRU + optional pinned set ----------
* Container snapshots keep the expert rows PACKED in RAM (int4 stays 4-bit:
* ~28 MB/expert instead of ~57 unpacked, so the same budget caches twice the
* experts); the matmul kernels unpack nibbles in-register. */
typedef struct {
int eid; uint64_t used;
int pinned; /* never evicted (usage-history pin) */
int filled; /* 0 while queued for a parallel fill */
uint8_t *p13, *p2; float *s13, *s2; /* container: packed rows + row scales */
int8_t *q13, *q2; /* bits>0: runtime-quantized int8 */
float *f13, *f2; /* bits==0: raw f32 (oracle) */
} Slot;
typedef struct {
Slot *slots;
int *slot_by_expert; /* expert id -> local slot, -1 if absent */
int n, cap;
} LCache;
/* One conversation's state for a multiplexed serve (KV_SLOTS>1, serve_mux below):
* its K and V (a sliding layer's ring), its four short-convolution banks, where it
* stands and the record of the tokens it holds. The Model holds the conversation a
* prefill runs on (ink_seq_swap trades it for a parked one); a multiplexed decode
* step parks them all and reads each row's from its InkRow. */
typedef struct { float **K, **V; float **cs[4]; int kv_len; kv_prefix kvp; } InkSeq;
typedef struct { InkSeq *seq; int pos; } InkRow;
static int g_ink_mux_slots = 1; /* KV_SLOTS: the conversations a serve decodes at once */
typedef struct {
Cfg c;
shards S;
shards Sq; /* container densa int4-gs64 (opzionale) */
int has_q, q_loaded; /* has_q: container presente; q_loaded: tensori presi da li */
int64_t q_bytes;
int quant_bits; /* 0 = f32 experts (oracle mode) */
int xq; /* experts on disk are a colibri container (U8 + .qs) */
Wt embed, lm_head;
float *embed_norm, *final_norm;
Wt audio_enc; /* [mel_bins*mel_vocab, D] embedding table */
float *audio_norm; /* audio tower RMSNorm [D]; NULL = no audio */
Layer *L;
LCache *cache;
int64_t rb13, rb2; /* container row-bytes (0 = not container) */
uint32_t **eusage; /* per-layer expert selection counts */
uint8_t **ehit; /* experts routed this turn, for HITS (dashboard Brain) */
int npin; /* pinned experts per sparse layer */
uint64_t clock, hits, miss;
uint64_t ereq, euse; /* routed richiesti (topk) vs usati dopo TOPP */
double t_fill, t_expert, t_shared, t_attn, t_route; /* phase timers */
float **K, **V; int kv_len, max_t; /* per-layer [kv][kv_ring_rows][hd]; sliding layers are a t%window ring */
float **cs[4]; /* conv states, [n_layers][C*(K-1)] */
double dense_load_s;
/* KV prefix reuse: what the current K/V and conv states were built from.
* See kv_prefix.h — recorded where the tokens are fed, never derived. */
kv_prefix kvp;
#ifdef COLI_VULKAN
void *vkchain; /* the dense chain's device state (inkling_chain.h), NULL until it runs */
void *vkchain2; /* its layers on COLI_VK_DEV2's device, after the primary's (inkling_chain.h) */
#endif
/* A multiplexed decode step (ink_step_rows): row s is the token at
* mux_rows[s].pos of mux_rows[s].seq; NULL in every other forward. */
const InkRow *mux_rows;
} Model;
/* ---------- utility ---------- */
static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; }
#if defined(__APPLE__)
static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0*1024.0); }
#else
static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0); }
#endif
static float *falloc(int64_t n) { float *p = malloc(n*sizeof(float)); if(!p){fprintf(stderr,"OOM %ld\n",(long)n);exit(1);} return p; }
static float sigmoidf(float x) { return 1.f / (1.f + expf(-x)); }
static float siluf(float x) { return x / (1.f + expf(-x)); }
/* TOPP=p (0..1): top-p adattivo sui routed — tieni gli esperti fino al peso
* cumulato p. 0 = spento (default): tutti i topk, calcolo invariato. */
static float g_topp = 0.f;
#include "matmul_f32.h" /* y[S,O] = x[S,I] @ W^T, W [O,I] f32 row-major */
#if defined(__AVX512BF16__) && defined(__AVX512F__)
#include <immintrin.h>
#define HAVE_BF16_DOT 1
#endif
#if defined(__AVX2__)
#include <immintrin.h>
#endif
/* bf16-weight matmul: activations rounded to bf16 per row (matches the HF
* bf16 reference numerics), hardware vdpbf16ps dot where available,
* shift-to-f32 scalar otherwise. */
static void matmul_h(float *y, const float *x, const uint16_t *W, int S, int I, int O) {
#ifdef HAVE_BF16_DOT
if (I % 32 == 0) {
uint16_t *xh = malloc((size_t)S * I * sizeof(uint16_t));
for (int s = 0; s < S; s++) {
const float *xs = x + (int64_t)s * I;
uint16_t *xd = xh + (int64_t)s * I;
for (int i = 0; i < I; i += 32) {
__m512 a = _mm512_loadu_ps(xs + i), b = _mm512_loadu_ps(xs + i + 16);
_mm512_storeu_si512(xd + i, (__m512i)_mm512_cvtne2ps_pbh(b, a));
}
}
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint16_t *w = W + (int64_t)o * I;
int s = 0;
for (; s + 3 < S; s += 4) {
const uint16_t *x0 = xh + (int64_t)(s+0)*I;
const uint16_t *x1 = xh + (int64_t)(s+1)*I;
const uint16_t *x2 = xh + (int64_t)(s+2)*I;
const uint16_t *x3 = xh + (int64_t)(s+3)*I;
__m512 a0 = _mm512_setzero_ps(), a1 = a0, a2 = a0, a3 = a0;
for (int i = 0; i < I; i += 32) {
__m512bh wb = (__m512bh)_mm512_loadu_si512(w + i);
a0 = _mm512_dpbf16_ps(a0, (__m512bh)_mm512_loadu_si512(x0 + i), wb);
a1 = _mm512_dpbf16_ps(a1, (__m512bh)_mm512_loadu_si512(x1 + i), wb);
a2 = _mm512_dpbf16_ps(a2, (__m512bh)_mm512_loadu_si512(x2 + i), wb);
a3 = _mm512_dpbf16_ps(a3, (__m512bh)_mm512_loadu_si512(x3 + i), wb);
}
y[(int64_t)(s+0)*O + o] = _mm512_reduce_add_ps(a0);
y[(int64_t)(s+1)*O + o] = _mm512_reduce_add_ps(a1);
y[(int64_t)(s+2)*O + o] = _mm512_reduce_add_ps(a2);
y[(int64_t)(s+3)*O + o] = _mm512_reduce_add_ps(a3);
}
for (; s < S; s++) {
const uint16_t *xs = xh + (int64_t)s*I;
__m512 acc = _mm512_setzero_ps();
for (int i = 0; i < I; i += 32)
acc = _mm512_dpbf16_ps(acc, (__m512bh)_mm512_loadu_si512(xs + i),
(__m512bh)_mm512_loadu_si512(w + i));
y[(int64_t)s*O + o] = _mm512_reduce_add_ps(acc);
}
}
free(xh);
return;
}
#endif
/* Prefill tile: decode each bf16 weight once for four independent rows.
* Explicit mul+add (rather than a C expression that the compiler may fuse)
* matches the non-FMA scalar oracle bit for bit. Every SIMD lane still
* accumulates i=0..I-1 in the original order. */
#if defined(__AVX2__)
if (S > 1) {
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint16_t *w = W + (int64_t)o * I;
int s = 0;
for (; s + 3 < S; s += 4) {
const float *x0 = x + (int64_t)(s+0)*I;
const float *x1 = x + (int64_t)(s+1)*I;
const float *x2 = x + (int64_t)(s+2)*I;
const float *x3 = x + (int64_t)(s+3)*I;
__m128 acc = _mm_setzero_ps();
for (int i = 0; i < I; i++) {
union { uint32_t u; float f; } v = { (uint32_t)w[i] << 16 };
__m128 xv = _mm_set_ps(x3[i], x2[i], x1[i], x0[i]);
acc = _mm_add_ps(acc, _mm_mul_ps(xv, _mm_set1_ps(v.f)));
}
float a[4]; _mm_storeu_ps(a, acc);
y[(int64_t)(s+0)*O + o] = a[0]; y[(int64_t)(s+1)*O + o] = a[1];
y[(int64_t)(s+2)*O + o] = a[2]; y[(int64_t)(s+3)*O + o] = a[3];
}
for (; s < S; s++) {
const float *xs = x + (int64_t)s * I;
float acc = 0.f;
for (int i = 0; i < I; i++) {
union { uint32_t u; float f; } v = { (uint32_t)w[i] << 16 };
acc += xs[i] * v.f;
}
y[(int64_t)s * O + o] = acc;
}
}
return;
}
#endif
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint16_t *w = W + (int64_t)o * I;
for (int s = 0; s < S; s++) {
const float *xs = x + (int64_t)s * I;
float acc = 0.f;
for (int i = 0; i < I; i++) {
union { uint32_t u; float f; } v = { (uint32_t)w[i] << 16 };
acc += xs[i] * v.f;
}
y[(int64_t)s * O + o] = acc;
}
}
}
/* dispatch on where the weight lives */
/* y[S,O] = x[S,I] @ W^T con W int4 GROUP-scaled: nibble +8, low = colonna pari,
* una scala f32 ogni `gs` elementi lungo I (ng = ceil(I/gs) scale per riga).
* Differenza da matmul_q4 (per-riga): la scala cambia DENTRO la riga, quindi
* l'accumulo va chiuso a ogni gruppo invece che una volta sola a fine riga. */
#if defined(__AVX2__) && defined(__FMA__)
static inline float matmul_i4g_row_fma(const float *xs, const uint8_t *w,
const float *sc, int I, int gs, int ng) {
__m128 acc = _mm_setzero_ps();
for (int g = 0; g < ng; g++) {
int i0 = g*gs, i1 = i0+gs; if (i1 > I) i1 = I;
__m128 part = _mm_setzero_ps();
for (int i = i0; i < i1; i++) {
uint8_t b = w[i >> 1]; int q = (i & 1) ? (b >> 4) : (b & 15);
part = _mm_fmadd_ss(_mm_set_ss(xs[i]), _mm_set_ss((float)(q-8)), part);
}
acc = _mm_add_ss(acc, _mm_mul_ss(part, _mm_set_ss(sc[g])));
}
return _mm_cvtss_f32(acc);
}
#endif
static void matmul_i4g(float *y, const float *x, const uint8_t *p, const float *scale,
int S, int I, int O, int gs) {
int ng = (I + gs - 1) / gs;
int64_t rb = (I + 1) / 2;
#if defined(__AVX2__) && defined(__FMA__)
/* Four prompt rows share nibble unpack and scale loads. The explicit FMA
* for each lane and the group-boundary mul+add match the scalar kernel's
* generated operation order bit for bit. */
if (S > 1) {
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint8_t *w = p + (int64_t)o * rb;
const float *sc = scale + (int64_t)o * ng;
int s = 0;
for (; s + 3 < S; s += 4) {
const float *x0 = x + (int64_t)(s+0)*I, *x1 = x + (int64_t)(s+1)*I;
const float *x2 = x + (int64_t)(s+2)*I, *x3 = x + (int64_t)(s+3)*I;
__m128 acc = _mm_setzero_ps();
for (int g = 0; g < ng; g++) {
int i0 = g*gs, i1 = i0+gs; if (i1 > I) i1 = I;
__m128 part = _mm_setzero_ps();
for (int i = i0; i < i1; i++) {
uint8_t b = w[i >> 1]; int q = (i & 1) ? (b >> 4) : (b & 15);
__m128 xv = _mm_set_ps(x3[i], x2[i], x1[i], x0[i]);
part = _mm_fmadd_ps(xv, _mm_set1_ps((float)(q-8)), part);
}
acc = _mm_add_ps(acc, _mm_mul_ps(part, _mm_set1_ps(sc[g])));
}
float a[4]; _mm_storeu_ps(a, acc);
y[(int64_t)(s+0)*O + o] = a[0]; y[(int64_t)(s+1)*O + o] = a[1];
y[(int64_t)(s+2)*O + o] = a[2]; y[(int64_t)(s+3)*O + o] = a[3];
}
for (; s < S; s++) {
const float *xs = x + (int64_t)s*I;
y[(int64_t)s*O + o] = matmul_i4g_row_fma(xs, w, sc, I, gs, ng);
}
}
return;
}
#endif
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint8_t *w = p + (int64_t)o * rb;
const float *sc = scale + (int64_t)o * ng;
for (int s = 0; s < S; s++) {
const float *xs = x + (int64_t)s * I;
#if defined(__AVX2__) && defined(__FMA__)
y[(int64_t)s * O + o] = matmul_i4g_row_fma(xs, w, sc, I, gs, ng);
#else
float acc = 0.f;
for (int g = 0; g < ng; g++) {
int i0 = g * gs, i1 = i0 + gs; if (i1 > I) i1 = I;
float part = 0.f;
for (int i = i0; i < i1; i++) {
uint8_t b = w[i >> 1];
int q = (i & 1) ? (b >> 4) : (b & 0x0F);
part += xs[i] * (float)(q - 8);
}
acc += part * sc[g]; /* scala chiusa per gruppo */
}
y[(int64_t)s * O + o] = acc;
#endif
}
}
}
/* int8 per-riga (embed/lm_head: sensibili, non vanno a 4 bit) */
static void matmul_i8r(float *y, const float *x, const int8_t *q, const float *scale,
int S, int I, int O) {
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const int8_t *w = q + (int64_t)o * I; float sc = scale[o];
for (int s = 0; s < S; s++) {
const float *xs = x + (int64_t)s * I;
float acc = 0.f;
for (int i = 0; i < I; i++) acc += xs[i] * (float)w[i];
y[(int64_t)s * O + o] = acc * sc;
}
}
}
#ifdef COLI_VULKAN
/* Device copies of one loaded tensor. A Wt travels by value and the shared
* experts are read through wt_off_i views of one fused [ns][R,I] tensor, so the
* cache cannot live in the copy matmul_w receives: it lives in a table the
* loaded Wt points to and every view copies, one entry per view (keyed by the
* view's first weight). Released with the tensor it belongs to. */
typedef struct { int64_t view; int used; ColiVkTensor *t; int dead; } InkVkView;
/* gone: the device holds the tensor alone (COLI_VK_DENSE_HOST): fmt and gs are the ones
* its views went up in, home the loaded Wt the CPU reads it back into. */
typedef struct { int n, fmt, gs, gone; Wt *home; InkVkView e[]; } InkVk;
/* The shader format that reads a weight exactly as the CPU kernel does, 0 for none.
* int8 per row (matmul_i8r) is fmt 1; int4 group-scaled with +8 nibbles and the low
* nibble on the even column (matmul_i4g) is fmt 4; f32 (matmul) is fmt 10; bf16
* (matmul_h) is fmt 11 -- except where matmul_h rounds the activations to bf16
* (the AVX512-BF16 dot), which the shader does not: there bf16 stays on the CPU. */
static int ink_vk_fmt(const Wt *w) {
if (w->q4) return w->qbits == 8 ? 1 : (w->qbits == 4 && w->gs >= 8 && w->gs % 8 == 0) ? 4 : 0;
if (w->f) return 10;
#ifndef HAVE_BF16_DOT
if (w->h) return 11;
#endif
return 0;
}
/* Attach a table to a loaded tensor the shader can read; returns its fmt, or 0. */
static int ink_vk_attach(Wt *w, int views) {
int fmt = ink_vk_fmt(w);
if (!fmt || w->vk || views < 1) return 0;
InkVk *v = calloc(1, sizeof(*v) + (size_t)views * sizeof(v->e[0]));
if (!v) return 0;
v->n = views; v->fmt = fmt; v->gs = w->q4 ? w->gs : 0; v->home = w;
w->vk = v;
return fmt;
}
static void ink_vk_release(Wt *w) {
InkVk *v = w->vk;
if (!v) return;
for (int k = 0; k < v->n; k++) if (v->e[k].t) coli_vk_tensor_free(v->e[k].t);
free(v);
w->vk = NULL;
}
/* 1 when y was computed on the device. One command buffer in the backend:
* main thread only, never from inside a parallel region. */
/* The entry of a view (keyed by its first element in the tensor), claimed on first use;
* NULL when every entry belongs to another view. */
static InkVkView *ink_vk_entry(InkVk *v, int64_t view) {
for (int k = 0; k < v->n; k++) {
InkVkView *e = &v->e[k];
if (e->used && e->view != view) continue;
e->used = 1; e->view = view;
return e;
}
return NULL;
}
static int ink_vk_matmul(float *y, const float *x, const Wt *W, int S, int I, int O) {
InkVk *v = W->vk;
if (!g_vk_ready || !v) return 0;
#ifdef _OPENMP
if (omp_in_parallel()) return 0;
#endif
int fmt = v->gone ? v->fmt : ink_vk_fmt(W);
const void *data = W->q4 ? (const void *)W->q4 : W->f ? (const void *)W->f : (const void *)W->h;
if (!fmt || (!data && !v->gone)) return 0;
InkVkView *e = ink_vk_entry(v, W->vk_view);
if (!e || e->dead || (v->gone && !e->t)) return 0; /* gone: only a copy made before the drop serves */
if (coli_vk_matmul(&e->t, y, x, data, W->q4 ? W->qs : NULL, fmt, S, I, O, v->gs))
return 1;
if (!e->t) e->dead = 1; /* refused at upload: this view stays on the CPU */
return 0;
}
/* A view's device copy, made now if it is not there yet (the chain's tensors and the
* eager upload of COLI_VK_DENSE_HOST); NULL when the view has none. */
static ColiVkTensor *ink_vk_view_tensor(const Wt *W, int I, int O) {
InkVk *v = W->vk;
if (!v) return NULL;
int fmt = v->gone ? v->fmt : ink_vk_fmt(W);
const void *data = W->q4 ? (const void *)W->q4 : W->f ? (const void *)W->f : (const void *)W->h;
if (!fmt || (!data && !v->gone)) return NULL;
InkVkView *e = ink_vk_entry(v, W->vk_view);
if (!e || e->dead) return NULL;
if (!e->t && (v->gone || !coli_vk_tensor_ensure(&e->t, data, W->q4 ? W->qs : NULL, fmt, I, O, v->gs))) {
e->dead = 1; return NULL;
}
return e->t;
}
static Wt ink_dho_reload(Wt W, int I); /* below, beside load_w */
static void ink_vk_report(void) {
if (g_vk_ready)
fprintf(stderr, "[VK] inkling: %llu matmuls on the GPU\n", coli_vk_matmul_calls());
}
#endif
#ifdef COLI_INKLING_SHARED_BATCH_TEST
static uint64_t g_matmul_w_calls;
#endif
static void matmul_w(float *y, const float *x, Wt W, int S, int I, int O) {
#ifdef COLI_INKLING_SHARED_BATCH_TEST
g_matmul_w_calls++;
#endif
#ifdef COLI_VULKAN
if (W.vk && ((InkVk *)W.vk)->gone) { /* the device holds it alone */
if (ink_vk_matmul(y, x, &W, S, I, O)) return;
W = ink_dho_reload(W, I); /* the CPU needs it after all (a lost device): from disk */
}
#endif
#ifdef COLI_CUDA
if (W.dev) {
if (ink_cuda_matmul_bf16(y, x, W.dev, S, I, O) == 0) return;
fprintf(stderr, "cuda matmul failed and host copy was freed\n"); exit(1);
}
#endif
if (W.q4) {
/* guard: il container e' un file, non un invariante — se la geometria non
* torna si esce invece di leggere fuori dal buffer. */
int64_t need = W.qbits == 8 ? (int64_t)O * I : (int64_t)O * ((I + 1) / 2);
if (need > W.qn) {
fprintf(stderr, "dense q4: geometria incoerente (serve %lld B, ho %lld) I=%d O=%d\n",
(long long)need, (long long)W.qn, I, O); exit(1);
}
#ifdef COLI_VULKAN
if (W.vk && ink_vk_matmul(y, x, &W, S, I, O)) return;
#endif
if (W.qbits == 8) matmul_i8r(y, x, (const int8_t*)W.q4, W.qs, S, I, O);
else matmul_i4g(y, x, W.q4, W.qs, S, I, O, W.gs);
return;
}
#ifdef COLI_VULKAN
if (W.vk && ink_vk_matmul(y, x, &W, S, I, O)) return;
#endif
if (W.f) matmul(y, x, W.f, S, I, O);
else matmul_h(y, x, W.h, S, I, O);
}
/* y[1,O] = x @ q^T, int8 weights + per-row scale. Fast path: activations
* quantized Q8 per 32-block, VNNI (or maddubs) int8 dot — same family as
* glm.c's IDOT kernels; IDOT=0 falls back to the byte-exact scalar route. */
#if defined(__AVX2__)
static inline __m256i i8dot_block(__m256i acc, __m256i a, __m256i b) {
__m256i ax = _mm256_sign_epi8(a, a); /* |a| as u8 */
__m256i sy = _mm256_sign_epi8(b, a); /* b * sign(a) */
#if defined(__AVX512VNNI__) && defined(__AVX512VL__)
return _mm256_dpbusd_epi32(acc, ax, sy);
#else
__m256i p = _mm256_maddubs_epi16(ax, sy);
return _mm256_add_epi32(acc, _mm256_madd_epi16(p, _mm256_set1_epi16(1)));
#endif
}
#endif
static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int I, int O) {
#if defined(__AVX2__)
static int idot = -1;
/* Opt-in (IDOT=1), not default: the fast path only exists under __AVX2__
* and quantizes ACTIVATIONS per 32-block — the same model produced
* different tokens on x86 vs ARM with the old on-by-default. Same class
* and same fix as olmoe (#1044) and qwen36 (#712 review). */
if (idot < 0) { const char *e = getenv("IDOT"); idot = (e && atoi(e)); }
if (idot && I % 32 == 0 && I <= 8192) {
int nb = I / 32;
int8_t xi[8192]; float xs[256];
for (int b = 0; b < nb; b++) {
const float *xb = x + b*32;
float am = 0.f; for (int i = 0; i < 32; i++) { float a = fabsf(xb[i]); if (a > am) am = a; }
float s = am/127.f; if (s < 1e-12f) s = 1e-12f;
xs[b] = s; float inv = 1.f/s;
for (int i = 0; i < 32; i++) xi[b*32+i] = (int8_t)lrintf(xb[i]*inv);
}
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const int8_t *w = q + (int64_t)o * I;
float acc = 0.f;
for (int b = 0; b < nb; b++) {
__m256i vacc = i8dot_block(_mm256_setzero_si256(),
_mm256_loadu_si256((const __m256i*)(xi + b*32)),
_mm256_loadu_si256((const __m256i*)(w + b*32)));
__m128i lo = _mm256_castsi256_si128(vacc), hi = _mm256_extracti128_si256(vacc, 1);
__m128i s4 = _mm_add_epi32(lo, hi);
s4 = _mm_hadd_epi32(s4, s4); s4 = _mm_hadd_epi32(s4, s4);
acc += xs[b] * (float)_mm_cvtsi128_si32(s4);
}
y[o] = acc * scale[o];
}
return;
}
#endif
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const int8_t *w = q + (int64_t)o * I;
float acc = 0.f;
for (int i = 0; i < I; i++) acc += x[i] * (float)w[i];
y[o] = acc * scale[o];
}
}
/* y[1,O] = x @ W^T with W kept PACKED int4 (low nibble = even column, +8
* offset, per-row scale — the on-disk container layout, cached as-is).
* Nibbles unpack in-register: same numeric result as unpack-to-int8 +
* matmul_q, half the cache footprint. IDOT=0 keeps the byte-exact scalar. */
static void matmul_q4(float *y, const float *x, const uint8_t *p, const float *scale, int I, int O) {
#if defined(__AVX2__)
static int idot = -1;
/* Opt-in (IDOT=1), not default: the fast path only exists under __AVX2__
* and quantizes ACTIVATIONS per 32-block — the same model produced
* different tokens on x86 vs ARM with the old on-by-default. Same class
* and same fix as olmoe (#1044) and qwen36 (#712 review). */
if (idot < 0) { const char *e = getenv("IDOT"); idot = (e && atoi(e)); }
if (idot && I % 32 == 0 && I <= 8192) {
int nb = I / 32;
int8_t xi[8192]; float xs[256];
for (int b = 0; b < nb; b++) {
const float *xb = x + b*32;
float am = 0.f; for (int i = 0; i < 32; i++) { float a = fabsf(xb[i]); if (a > am) am = a; }
float s = am/127.f; if (s < 1e-12f) s = 1e-12f;
xs[b] = s; float inv = 1.f/s;
for (int i = 0; i < 32; i++) xi[b*32+i] = (int8_t)lrintf(xb[i]*inv);
}
const __m128i m4 = _mm_set1_epi8(0x0F);
const __m256i b8 = _mm256_set1_epi8(8);
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint8_t *w = p + (int64_t)o * (I/2);
float acc = 0.f;
for (int b = 0; b < nb; b++) {
__m128i by = _mm_loadu_si128((const __m128i*)(w + b*16)); /* 16 B = 32 nibbles */
__m128i lo = _mm_and_si128(by, m4); /* even columns */
__m128i hi = _mm_and_si128(_mm_srli_epi16(by, 4), m4); /* odd columns */
__m256i nib = _mm256_set_m128i(_mm_unpackhi_epi8(lo, hi), /* cols 16..31 */
_mm_unpacklo_epi8(lo, hi)); /* cols 0..15 */
nib = _mm256_sub_epi8(nib, b8);
__m256i vacc = i8dot_block(_mm256_setzero_si256(),
_mm256_loadu_si256((const __m256i*)(xi + b*32)), nib);
__m128i l = _mm256_castsi256_si128(vacc), h = _mm256_extracti128_si256(vacc, 1);
__m128i s4 = _mm_add_epi32(l, h);
s4 = _mm_hadd_epi32(s4, s4); s4 = _mm_hadd_epi32(s4, s4);
acc += xs[b] * (float)_mm_cvtsi128_si32(s4);
}
y[o] = acc * scale[o];
}
return;
}
#endif
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const uint8_t *w = p + (int64_t)o * (I/2);
float acc = 0.f;
for (int i = 0; i < I; i += 2) {
uint8_t byte = w[i/2];
acc += x[i] * (float)((int)(byte & 0xF) - 8);
acc += x[i+1] * (float)((int)(byte >> 4) - 8);
}
y[o] = acc * scale[o];
}
}
static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits) {
int qmax = (1 << (bits - 1)) - 1;
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const float *wr = w + (int64_t)o * I;
float amax = 0.f; for (int i = 0; i < I; i++) { float a = fabsf(wr[i]); if (a > amax) amax = a; }
float s = amax / qmax; if (s < 1e-8f) s = 1e-8f;
scale[o] = s;
int8_t *qr = q + (int64_t)o * I;
for (int i = 0; i < I; i++) {
int v = (int)lrintf(wr[i] / s);
if (v > qmax) v = qmax;
if (v < -qmax-1) v = -qmax-1;
qr[i] = (int8_t)v;
}
}
}
/* rmsnorm computed in f64 accumulate like the f32->f32 reference */
static void rmsnorm_row(float *out, const float *x, const float *w, int D, float eps) {
double ms = 0; for (int i = 0; i < D; i++) ms += (double)x[i]*x[i];
float r = 1.f / sqrtf((float)(ms / D) + eps);
for (int i = 0; i < D; i++) out[i] = x[i] * r * w[i];
}
static void softmax_row(float *x, int n) {
float m = -1e30f; for (int i = 0; i < n; i++) if (x[i] > m) m = x[i];
float s = 0; for (int i = 0; i < n; i++) { x[i] = expf(x[i]-m); s += x[i]; }
for (int i = 0; i < n; i++) x[i] /= s;
}
/* ---------- depthwise causal short conv, residual inside (fp32) ----------
* seq[S,C] in-place: out[t] = sum_j w[c,j]*in[t+j-(K-1)] + in[t], history from
* state[C*(K-1)] (raw pre-conv inputs), which is updated to the new tail. */
static void sconv_apply(float *seq, int S, int C, const float *w, float *state, int K) {
int P = K - 1;
#pragma omp parallel
{
float *col = malloc((P + S) * sizeof(float));
#pragma omp for schedule(static)
for (int ch = 0; ch < C; ch++) {
for (int j = 0; j < P; j++) col[j] = state[(int64_t)ch*P + j];
for (int t = 0; t < S; t++) col[P + t] = seq[(int64_t)t*C + ch];
const float *wc = w + (int64_t)ch*K;
for (int t = 0; t < S; t++) {
float acc = 0.f;
for (int j = 0; j < K; j++) acc += wc[j] * col[t + j];
seq[(int64_t)t*C + ch] = acc + col[P + t];
}
for (int j = 0; j < P; j++) state[(int64_t)ch*P + j] = col[S + j];
}
free(col);
}
}
/* ---------- config loading ----------
* Accepts both the flat text config (tiny oracle via InklingForCausalLM) and
* the full multimodal config.json (real checkpoint, fields under text_config). */
static double jnum(jval *o, const char *k, double dflt) {
jval *v = json_get(o, k);
return (v && v->t == J_NUM) ? v->num : dflt;
}
static void load_cfg(Cfg *c, const char *snap) {
char path[2048]; snprintf(path, sizeof(path), "%s/config.json", snap);
FILE *f = fopen(path, "rb"); if(!f){perror(path);exit(1);}
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
char *buf = malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f);
char *arena=NULL; jval *root = json_parse(buf, &arena);
jval *r = json_get(root, "text_config"); if (!r) r = root;
c->hidden = (int)jnum(r,"hidden_size",6144);
c->n_layers = (int)jnum(r,"num_hidden_layers",66);
c->vocab = (int)jnum(r,"vocab_size",201024);
c->unpad_vocab = (int)jnum(r,"unpadded_vocab_size",c->vocab);
c->n_heads = (int)jnum(r,"num_attention_heads",64);
c->n_kv = (int)jnum(r,"num_key_value_heads",8);
c->head_dim = (int)jnum(r,"head_dim",128);
c->swa_heads = (int)jnum(r,"swa_num_attention_heads",c->n_heads);
c->swa_kv = (int)jnum(r,"swa_num_key_value_heads",16);
c->swa_hd = (int)jnum(r,"swa_head_dim",c->head_dim);
c->window = (int)jnum(r,"sliding_window_size",512);
c->d_rel = (int)jnum(r,"d_rel",16);
c->rel_extent = (int)jnum(r,"rel_extent",1024);
c->log_floor = jnum(r,"log_scaling_n_floor",0);
c->log_alpha = (float)jnum(r,"log_scaling_alpha",0.1);
c->conv_k = (int)jnum(r,"sconv_kernel_size", jnum(r,"conv_kernel_size",4));
c->n_experts = (int)jnum(r,"n_routed_experts",256);
c->topk = (int)jnum(r,"num_experts_per_tok",6);
c->n_shared = (int)jnum(r,"n_shared_experts",2);
c->eps = (float)jnum(r,"rms_norm_eps",1e-6);
c->route_scale = (float)jnum(r,"route_scale",8.0);
c->mup = (float)jnum(r,"logits_mup_width_multiplier",24.0);
/* eos lives at the top level in the real multimodal config, in the text
* config for a flat snapshot; may be null (tiny oracle) */
jval *eo = json_get(root,"eos_token_id");
if (!eo || eo->t != J_NUM) eo = json_get(r,"eos_token_id");
c->eos = (eo && eo->t == J_NUM) ? (int)eo->num : -1;
/* DMel audio: placeholder id at the top level (absent in the shipped
* config -> the TMLv0 constant), frame geometry under audio_config */
c->audio_tok = (int)jnum(root,"audio_token_id",200023);
jval *ac = json_get(root,"audio_config");
c->mel_bins = ac ? (int)jnum(ac,"n_mel_bins",80) : 80;
c->mel_vocab = ac ? (int)jnum(ac,"mel_vocab_size",16) : 16;
/* real config.json: intermediate_size = MoE, dense_intermediate_size = dense.
* HF-saved config (post_init applied): intermediate_size = dense, moe_intermediate_size = MoE. */
jval *dis = json_get(r,"dense_intermediate_size");
if (dis && dis->t == J_NUM) {
c->dense_inter = (int)dis->num;
c->moe_inter = (int)jnum(r,"intermediate_size",3072);
} else {
c->dense_inter = (int)jnum(r,"intermediate_size",24576);
c->moe_inter = (int)jnum(r,"moe_intermediate_size",3072);
}
if (c->n_layers > MAXL) { fprintf(stderr,"n_layers %d > MAXL\n", c->n_layers); exit(1); }
/* SEC: these loops run to n_layers -- a number from config.json, up to
* MAXL -- while indexing a JSON array whose length is a different, equally
* attacker-chosen number. "num_hidden_layers": 66 with a one-element
* layer_types[] read kids[1..65] past a malloc'd array that starts at
* capacity 8, then dereferenced whatever was there as a string. config.json
* alone was enough; no weights and no valid snapshot were needed.
*
* A short array now means "not specified for these layers", so the loop
* falls through to the same default it would have used had the key been
* absent. The element type is checked too: kids[i]->str is a union member
* that is only a valid pointer when t == J_STR. */
#define LT_STR(arr, i) \
((arr) && (arr)->t == J_ARR && (i) < (arr)->len && \
(arr)->kids[i] && (arr)->kids[i]->t == J_STR ? (arr)->kids[i]->str : NULL)
/* attention layer types: explicit layer_types[] > local_layer_ids[] > (i+1)%6 rule */
jval *lt = json_get(r,"layer_types");
jval *ll = json_get(r,"local_layer_ids");
for (int i = 0; i < c->n_layers; i++) {
const char *ltype = LT_STR(lt, i);
if (ltype) c->local[i] = (strcmp(ltype,"hybrid_sliding")==0);
else if (ll && ll->t == J_ARR) {
c->local[i] = 0;
for (int j = 0; j < ll->len; j++)
if (ll->kids[j] && ll->kids[j]->t == J_NUM &&
(int)ll->kids[j]->num == i) { c->local[i] = 1; break; }
} else c->local[i] = ((i + 1) % 6) != 0;
}
/* MLP types: explicit mlp_layer_types[] > dense_mlp_idx (first k layers dense) */
jval *mt = json_get(r,"mlp_layer_types");
int dense_idx = (int)jnum(r,"dense_mlp_idx",0);
for (int i = 0; i < c->n_layers; i++) {
const char *mtype = LT_STR(mt, i);
if (mtype) c->sparse[i] = (strcmp(mtype,"sparse")==0);
else c->sparse[i] = (i >= dense_idx);
}
#undef LT_STR
free(buf); free(arena);
}
/* ---------- weight loading ---------- */
/* The embedding norm has two names. The converter and every container written with
* transformers up to 5.17 call it model.embed_norm.weight; transformers 5.18 saves
* the same tensor as model.embed_tokens.embed_norm.weight. Read by the old name
* only, a checkpoint saved by 5.18 loaded with no embedding norm at all -- no
* error, and every answer wrong (the tiny oracle: 1 of 36 positions). */
static const char *embed_norm_name(shards *S) {
if (st_has(S, "model.embed_norm.weight")) return "model.embed_norm.weight";
if (st_has(S, "model.embed_tokens.embed_norm.weight")) return "model.embed_tokens.embed_norm.weight";
return NULL;
}
static float *load_t(Model *m, const char *name) {
int64_t n = st_numel(&m->S, name);
if (n < 0) { fprintf(stderr, "missing %s\n", name); exit(1); }
float *p = falloc(n);
st_read_f32(&m->S, name, p, 0);
return p;
}
static float load_scalar(Model *m, const char *name, float dflt) {
if (!st_has(&m->S, name)) return dflt;
/* SEC: `v` is four bytes on the stack, and st_read_f32 writes as many as
* the FILE says the tensor holds -- it validates the header against itself
* (numel*esz == nbytes) and cannot know the destination size. A crafted
* snapshot declaring e.g. model.layers.0.mlp.gate.global_scale as F32
* shape [4096] therefore drops 16 KiB of attacker bytes over this frame
* and its return address; the reported crash had RSP fully controlled.
*
* st_read_f32_cap is the same read with the caller's capacity passed in.
* A scalar's capacity is 1. Anything larger is a hostile or corrupt file
* and stops the load, which is the behaviour every other bounds check in
* st.h already has. */
float v; st_read_f32_cap(&m->S, name, &v, 1, 0); return v;
}
/* chunked pread: a single pread caps at ~2.1 GB on Linux, and the bf16
* embed/lm_head tensors are 2.47 GB — loop in 1 GB slices */
static void pread_all(int fd, void *buf, int64_t nb, int64_t off) {
char *p = buf;
while (nb > 0) {
int64_t chunk = nb < (1<<30) ? nb : (1<<30);
ssize_t got = pread(fd, p, (size_t)chunk, off);
if (got <= 0) { perror("pread chunk"); exit(1); }
p += got; off += got; nb -= got;
}
}
/* big matmul weights keep their on-disk dtype resident: BF16 raw (real
* checkpoint, halves RAM), anything else as f32 (tiny oracle: bit-exact).
* gpu_ok: bf16 tensors move to VRAM while budget lasts (embed stays host —
* it's a row lookup, not a matmul). */
/* Container densa pre-quantizzata (opzionale): <snap>/dense-int4g64/.
* Se il tensore c'e' li' dentro lo carichiamo int4-gs64 (U8 + sidecar .qs, la
* stessa convenzione che il container usa gia' per gli esperti); altrimenti si
* prosegue col percorso bf16/f32 di sempre. Container assente = nessun cambio. */
static int load_w_quant(Model *m, const char *name, int64_t orig_numel, Wt *out) {
if (!m->has_q) return 0;
st_tensor *t = st_find(&m->Sq, name);
if (!t || t->dtype != 3) return 0; /* 3 = U8/byte grezzi */
char qn[352]; snprintf(qn, sizeof(qn), "%s.qs", name);
st_tensor *s = st_find(&m->Sq, qn);
if (!s) return 0; /* senza scale non si decodifica */
/* Geometria dedotta dai CONTEGGI, non dalle shape (st_tensor non le porta):
* int4-gs64 -> byte ~= numel/2 e scale ~= numel/64
* int8 -> byte == numel e scale == righe (numel/I, I ignoto qui:
* basta che le scale siano molte meno dei byte)
* Il controllo forte sull'OOB lo fa matmul_w con l'I vero del chiamante. */
Wt w = {0};
if (t->nbytes == orig_numel && s->numel * 64 < (int64_t)t->nbytes) {
w.qbits = 8; /* int8 per riga */
} else if (t->nbytes * 2 >= orig_numel && t->nbytes * 2 <= orig_numel + 2 * s->numel) {
w.qbits = 4; w.gs = 64; /* int4 group-scaled */
} else {
fprintf(stderr, "[dense] %s: geometria non riconosciuta nel container, uso il bf16\n", name);
return 0; /* dubbio -> percorso originale */
}
w.q4 = malloc(t->nbytes); if (!w.q4) { fprintf(stderr, "OOM %s\n", name); exit(1); }
pread_all(t->fd, w.q4, t->nbytes, t->off);
w.qn = t->nbytes;
w.qs = falloc(s->numel);
st_read_f32(&m->Sq, qn, w.qs, 0);
*out = w;
m->q_loaded++; m->q_bytes += t->nbytes + (int64_t)s->numel * 4;
return 1;
}
#ifdef COLI_VULKAN
/* the names of loaded tensors, kept for the life of the process (a Wt travels by value) */
static const char *ink_name_keep(const char *name) {
char *n = strdup(name);
if (!n) { fprintf(stderr, "OOM weight name\n"); exit(1); }
return n;
}
#endif
static Wt load_w(Model *m, const char *name, int gpu_ok) {
Wt w = {0};
st_tensor *t = st_find(&m->S, name);
if (!t) { fprintf(stderr, "missing %s\n", name); exit(1); }
#ifdef COLI_VULKAN
if (load_w_quant(m, name, t->numel, &w)) { w.vk_name = ink_name_keep(name); return w; }
w.vk_name = ink_name_keep(name);
#else
if (load_w_quant(m, name, t->numel, &w)) return w;
#endif
if (t->dtype == 0) {
w.h = malloc(t->nbytes); if (!w.h) { fprintf(stderr,"OOM %s\n",name); exit(1); }
pread_all(t->fd, w.h, t->nbytes, t->off);
#ifdef COLI_CUDA
/* keep 3 GB VRAM headroom for the activation buffers + future tiers */
if (g_cuda && gpu_ok && ink_cuda_free_bytes() > (size_t)t->nbytes + (3ULL<<30)) {
w.dev = ink_cuda_upload(w.h, t->nbytes);
if (w.dev) { free(w.h); w.h = NULL; }
}
#else
(void)gpu_ok;
#endif
} else {
w.f = falloc(t->numel);
st_read_f32(&m->S, name, w.f, 0);
}
return w;
}
/* `off` conta ELEMENTI ed e' sempre un multiplo di I (si affetta per riga: un
* esperto condiviso dal tensore fuso [E,R,I]), quindi I arriva dal chiamante —
* st_tensor non porta le shape e non voglio indovinarle. */
static Wt wt_off_i(Wt w, int64_t off, int I) {
Wt r = w;
#ifdef COLI_VULKAN
r.vk_view = w.vk_view + off;
#endif
r.f = w.f ? w.f + off : NULL;
r.h = w.h ? w.h + off : NULL;
r.dev = w.dev ? (char*)w.dev + off*2 : NULL; /* dev is always bf16 */
if (w.q4) {
int64_t row = off / I;
if (w.qbits == 8) { r.q4 = w.q4 + off; r.qs = w.qs + row; r.qn = w.qn - off; }
else { r.q4 = w.q4 + off / 2; r.qs = w.qs + row * ((I + w.gs - 1) / w.gs);
r.qn = w.qn - off / 2; }
}
return r;
}
/* dequantizza UNA riga (int4-gs64: nibble +8, low = colonna pari, scala ogni gs) */
static void wt_deq_row(Wt w, int64_t row, float *out, int I) {
if (w.qbits == 8) {
const int8_t *q = (const int8_t*)w.q4 + row * I; float s = w.qs[row];
for (int i = 0; i < I; i++) out[i] = (float)q[i] * s;
return;
}
int ng = (I + w.gs - 1) / w.gs;
const uint8_t *p = w.q4 + row * ((I + 1) / 2);
const float *sc = w.qs + row * ng;
for (int i = 0; i < I; i++) {
uint8_t b = p[i >> 1];
int q = (i & 1) ? (b >> 4) : (b & 0x0F);
out[i] = (float)(q - 8) * sc[i / w.gs];
}
}
static void wt_row_f32(Wt w, int64_t off, float *out, int n) {
if (w.q4) { wt_deq_row(w, off / n, out, n); return; } /* n = dim di contrazione */
if (w.f) memcpy(out, w.f + off, n * sizeof(float));
else for (int i = 0; i < n; i++) { union { uint32_t u; float f; } v = { (uint32_t)w.h[off + i] << 16 }; out[i] = v.f; }
}
/* f32 slice of a (possibly bf16/f16) tensor: element offset + count.
* Needed to stream one expert out of the fused [E,2I,D]/[E,D,I] tensors. */
static void read_f32_slice(shards *S, const char *name, float *out, int64_t off, int64_t cnt) {
st_tensor *t = st_find(S, name);
if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); }
if (t->dtype == 3) { fprintf(stderr, "%s: U8 container has no f32 view\n", name); exit(1); }
int esz = (t->dtype == 2) ? 4 : 2;
void *raw = malloc((size_t)cnt * esz);
if (!raw) { fprintf(stderr,"OOM slice %s\n",name); exit(1); }
if (pread(t->fd, raw, (size_t)cnt*esz, t->off + off*esz) != (ssize_t)(cnt*esz)) { perror("pread slice"); exit(1); }
if (t->dtype == 2) memcpy(out, raw, (size_t)cnt*4);
else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < cnt; i++) out[i] = bf16_to_f32(p[i]); }
else { uint16_t *p = raw; for (int64_t i = 0; i < cnt; i++) out[i] = f16_to_f32(p[i]); }
free(raw);
posix_fadvise(t->fd, t->off + off*esz, cnt*esz, POSIX_FADV_DONTNEED);
}
/* raw byte slice of a U8 container tensor */
static void read_u8_slice(shards *S, const char *name, uint8_t *out, int64_t boff, int64_t nb) {
st_tensor *t = st_find(S, name);
if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); }
if (pread(t->fd, out, (size_t)nb, t->off + boff) != (ssize_t)nb) { perror("pread u8 slice"); exit(1); }
posix_fadvise(t->fd, t->off + boff, nb, POSIX_FADV_DONTNEED);
}
/* container rows -> int8: rowb==cols is int8 verbatim; rowb==cols/2 is packed
* int4 (low nibble = even column, offset +8 — convert_inkling_int4.py / glm.c) */
static void unpack_rows(const uint8_t *raw, int8_t *q, int64_t rows, int64_t cols, int64_t rowb) {
if (rowb == cols) { memcpy(q, raw, (size_t)(rows*cols)); return; }
if (rowb*2 != cols) { fprintf(stderr, "container row size %ld vs cols %ld unsupported\n", (long)rowb, (long)cols); exit(1); }
for (int64_t r = 0; r < rows; r++) {
const uint8_t *b = raw + r*rowb;
int8_t *qr = q + r*cols;
for (int64_t j = 0; j < rowb; j++) {
qr[2*j] = (int8_t)((b[j] & 0xF) - 8);
qr[2*j+1] = (int8_t)((b[j] >> 4) - 8);
}
}
}
static double mem_avail_bytes(void);
#ifdef COLI_VULKAN
/* After the weights: open the device (COLI_VULKAN=1) and mark the resident
* matrices it can take, which upload on first use; the line says which. The host
* copy stays (it is the CPU fallback), so the device holds a second copy: on a GPU
* that shares RAM with the CPU the resident set costs twice. */
static void ink_vk_mark(Wt *w, int views, int *by, int *cpu) {
if (!w->q4 && !w->f && !w->h) return; /* absent, or a CUDA resident */
int fmt = ink_vk_attach(w, views);
if (fmt) by[fmt]++; else (*cpu)++;
}
/* head 0: lm_head stays on the CPU (a partial chain's, inkc_fit_start) */
static void ink_vk_mark_dense(Model *m, int layer_begin, int layer_end, int head) {
int by[13] = {0}, cpu = 0;
if (head) ink_vk_mark(&m->lm_head, 1, by, &cpu);
for (int i = layer_begin; i < layer_end; i++) {
Layer *l = &m->L[i];
Wt *one[] = { &l->q, &l->k, &l->v, &l->r, &l->o, &l->dg, &l->du, &l->dd };
for (size_t j = 0; j < sizeof(one) / sizeof(one[0]); j++) ink_vk_mark(one[j], 1, by, &cpu);
ink_vk_mark(&l->sh_g, m->c.n_shared, by, &cpu); /* one device copy per shared expert */
ink_vk_mark(&l->sh_u, m->c.n_shared, by, &cpu);
ink_vk_mark(&l->sh_d, m->c.n_shared, by, &cpu);
}
fprintf(stderr, "[VK] inkling: %d resident matrices go to the GPU on first use "
"(%d int8, %d int4-g64, %d f32, %d bf16), %d stay on the CPU; "
"%s and embedding lookups stay on the CPU\n",
by[1] + by[4] + by[10] + by[11], by[1], by[4], by[10], by[11], cpu,
g_vk_tier_try ? "routed experts go to the expert tier (its own line)" : "routed experts");
}
/* Opens the device saying whether the routed-expert tier will be tried (vk_tier.h
* step 0): a whole model with routed experts, and neither CUDA nor Metal on, either
* of which wins. The dense matrices then go where coli_vk_dense() says: on a device
* sharing the CPU's RAM with the tier on, they stay on the CPU unless COLI_VK_DENSE=1;
* if the tier does not start after all, ink_vk_tier_start marks them then. */
/* ---- the dense matrices on the device only (COLI_VK_DENSE_HOST) ----------------
* Decided when the device opens (ink_vk_init_model), before the expert cache is sized:
* every resident matrix the shaders read goes up now, view by view (each shared expert
* of the fused tensors is a view), into the table the per-matrix path and the chain
* share, and its host copy is given back. The per-matrix path then answers on the device
* whatever coli_vk_dense() says, the chain finds the same copies, and the CPU reads a
* matrix back from disk only when it needs it after all (ink_dho_reload: a lost device).
* Kept on the host: the embedding and the audio table (row lookups), the routers (the
* chain holds its own copy, the host step reads them off the chain), norms, convolution
* taps and the bias banks, and bf16 matrices on a CPU whose bf16 dot rounds activations. */
#ifndef INKLING_CHAIN_IGPU /* inkling_chain.h's default, needed here first */
#define INKLING_CHAIN_IGPU COLI_VK_CHAIN_UNMEASURED
#endif
static Model *g_ink_dho_model;
static pthread_mutex_t g_ink_dho_mx = PTHREAD_MUTEX_INITIALIZER;
/* Host bytes of a loaded tensor of `n` elements, in the form RAM holds it. */
static size_t ink_wt_bytes(const Wt *w, int64_t n, int I) {
if (w->q4) return (size_t)w->qn + (size_t)(w->qbits == 8 ? n / I : (n / I) * ((I + w->gs - 1) / w->gs)) * 4;
if (w->f) return (size_t)n * 4;
if (w->h) return (size_t)n * 2;
return 0;
}
/* The CPU needs the view W of a tensor the device holds alone: read the tensor back
* into its home Wt (the bytes load_w read the first time), and hand the view of it. */
static Wt ink_dho_reload(Wt W, int I) {
InkVk *v = W.vk;
pthread_mutex_lock(&g_ink_dho_mx);
if (v->gone) {
Model *m = g_ink_dho_model;
Wt *h = v->home;
if (!m || !h->vk_name) { fprintf(stderr, "[VK] inkling: a dense matrix the device held alone cannot be read back\n"); exit(1); }
int q_loaded = m->q_loaded; int64_t q_bytes = m->q_bytes;
Wt t = load_w(m, h->vk_name, 0);
m->q_loaded = q_loaded; m->q_bytes = q_bytes;
if (!!t.q4 != (v->fmt == 1 || v->fmt == 4) || (t.q4 && t.qbits != h->qbits)) {
fprintf(stderr, "[VK] inkling: %s came back in another form\n", h->vk_name); exit(1);
}
size_t b = (size_t)t.qn;
h->f = t.f; h->h = t.h; h->q4 = t.q4; h->qs = t.qs; h->qn = t.qn;
free((void *)t.vk_name);
__atomic_store_n(&v->gone, 0, __ATOMIC_RELEASE);
coli_vk_dense_host_reloaded(b ? b : 1);
}
pthread_mutex_unlock(&g_ink_dho_mx);
return wt_off_i(*v->home, W.vk_view, I);
}
/* One loaded tensor up, view by view (`views` views `stride` elements apart, each
* [O, I]), then its host copy given back: all views or none. */
static int ink_dho_drop(Wt *w, int views, int64_t stride, int I, int O, int *n) {
InkVk *v = w->vk;
if (!v || v->gone || !w->vk_name || (!w->q4 && !w->f && !w->h)) return 0;
for (int j = 0; j < views; j++) {
Wt view = wt_off_i(*w, (int64_t)j * stride, I);
if (!ink_vk_view_tensor(&view, I, O)) return 0;
}
size_t b = ink_wt_bytes(w, (int64_t)views * I * O, I);
free(w->f); free(w->h); free(w->q4); free(w->qs);
w->f = NULL; w->h = NULL; w->q4 = NULL; w->qs = NULL; /* qbits, gs and qn stay: the form it went up in */
v->gone = 1;
coli_vk_dense_host_dropped(b);
(*n)++;
return 1;
}
static size_t ink_dho_count(const Wt *w, int64_t n, int I) { return ink_vk_fmt(w) ? ink_wt_bytes(w, n, I) : 0; }
static size_t ink_dho_bytes(const Model *m, int layer_begin, int layer_end, int head) {
const Cfg *c = &m->c; int64_t D = c->hidden;
size_t b = head ? ink_dho_count(&m->lm_head, (int64_t)c->unpad_vocab * D, (int)D) : 0;
for (int i = layer_begin; i < layer_end; i++) {
const Layer *l = &m->L[i];
int64_t q = (int64_t)L_HEADS(c, i) * L_HD(c, i), kv = (int64_t)L_KV(c, i) * L_HD(c, i), I = c->moe_inter;
b += ink_dho_count(&l->q, D * q, (int)D) + ink_dho_count(&l->k, D * kv, (int)D) + ink_dho_count(&l->v, D * kv, (int)D) +
ink_dho_count(&l->r, D * L_HEADS(c, i) * c->d_rel, (int)D) + ink_dho_count(&l->o, q * D, (int)q) +
ink_dho_count(&l->dg, D * c->dense_inter, (int)D) + ink_dho_count(&l->du, D * c->dense_inter, (int)D) +
ink_dho_count(&l->dd, D * c->dense_inter, c->dense_inter) +
ink_dho_count(&l->sh_g, c->n_shared * I * D, (int)D) + ink_dho_count(&l->sh_u, c->n_shared * I * D, (int)D) +
ink_dho_count(&l->sh_d, c->n_shared * I * D, (int)I);
}
return b;
}
/* Layer i's matrices up (each view of a fused tensor its own) and their host copies given back. */
static void ink_dho_drop_layer(Model *m, int i, int *n) {
const Cfg *c = &m->c; int D = c->hidden;
Layer *l = &m->L[i];
int q = L_HEADS(c, i) * L_HD(c, i), kv = L_KV(c, i) * L_HD(c, i), I = c->moe_inter, ns = c->n_shared;
ink_dho_drop(&l->q, 1, 0, D, q, n); ink_dho_drop(&l->k, 1, 0, D, kv, n); ink_dho_drop(&l->v, 1, 0, D, kv, n);
ink_dho_drop(&l->r, 1, 0, D, L_HEADS(c, i) * c->d_rel, n); ink_dho_drop(&l->o, 1, 0, q, D, n);
if (!c->sparse[i]) {
ink_dho_drop(&l->dg, 1, 0, D, c->dense_inter, n); ink_dho_drop(&l->du, 1, 0, D, c->dense_inter, n);
ink_dho_drop(&l->dd, 1, 0, c->dense_inter, D, n);
} else if (ns > 0) {
ink_dho_drop(&l->sh_g, ns, (int64_t)I * D, D, I, n); ink_dho_drop(&l->sh_u, ns, (int64_t)I * D, D, I, n);
ink_dho_drop(&l->sh_d, ns, (int64_t)D * I, I, D, n);
}
}
/* inkling_chain.h: the partial chain's fit, and its layers placed at start-up */
static void inkc_fit_start(Model *m);
static void inkc_place_init(Model *m, int *dropped);
static void ink_dho_place(Model *m, int layer_begin, int layer_end) {
const Cfg *c = &m->c; int D = c->hidden, n = 0;
if (g_inkc_fit.L) {
/* the chain's fit: its layers whole, each one's host copies given back once all of
* it is on the device (inkc_place_init), then lm_head when the tail went up too */
inkc_place_init(m, &n);
if (g_inkc_fit.n == g_inkc_fit.L && g_inkc_fit.tail) ink_dho_drop(&m->lm_head, 1, 0, D, c->unpad_vocab, &n);
} else {
ink_dho_drop(&m->lm_head, 1, 0, D, c->unpad_vocab, &n);
for (int i = layer_begin; i < layer_end; i++) ink_dho_drop_layer(m, i, &n);
}
coli_vk_dense_host_placed("inkling", "the embedding and audio tables (row lookups), the routers, norms, convolution taps, bias banks"
#ifdef HAVE_BF16_DOT
", bf16 matrices (this CPU's bf16 dot rounds activations, the device would not)"
#endif
);
}
static void ink_vk_init_model(Model *m, int layer_begin, int layer_end) {
int sparse = 0, other_gpu = 0;
for (int i = layer_begin; i < layer_end; i++) sparse += m->c.sparse[i];
#ifdef COLI_CUDA
other_gpu |= g_cuda;
#endif
#ifdef COLI_METAL
other_gpu |= g_metal;
#endif
g_vk_tier_try = vkt_wanted() && m->c.n_experts > 0 && sparse > 0 && !other_gpu &&
layer_begin == 0 && layer_end == m->c.n_layers;
if (!g_vk_ready) g_vk_ready = coli_vk_init_env_tier("inkling", g_vk_tier_try);
if (!g_vk_ready) { g_vk_tier_try = 0; return; }
/* COLI_VK_DENSE_HOST: with the dense part on the device (the per-matrix path, or the
* chain: asked here silently, inkc_start says it after the tier), every matrix the
* shaders read is marked and goes up now, its host copy given back */
int chain = !other_gpu && layer_begin == 0 && layer_end == m->c.n_layers &&
coli_vk_chain_decide(NULL, g_vk_tier_try, INKLING_CHAIN_IGPU) != COLI_VK_CHAIN_OFF;
/* the chain's fit before any upload and before the expert cache is sized: the first n
* layers on the device, the others (and lm_head, unless the tail went up) on the CPU
* with their host copies, the per-matrix path leaving them there */
if (chain) inkc_fit_start(m);
int end = g_inkc_fit.L ? g_inkc_fit.n : layer_end, head = g_inkc_fit.L ? g_inkc_fit.tail : 1;
int on_device = (coli_vk_dense() || chain) && !other_gpu && !(g_inkc_fit.L && !g_inkc_fit.n);
if (coli_vk_dense_host_decide("inkling", on_device, ink_dho_bytes(m, layer_begin, end, head))) {
g_ink_dho = 1; g_ink_dho_model = m;
if (g_inkc_fit.L) coli_vk_dense_host_layers(g_inkc_fit.n, g_inkc_fit.L);
ink_vk_mark_dense(m, layer_begin, end, head);
ink_dho_place(m, layer_begin, end);
return;
}
if (g_inkc_fit.L && vkc_fit_partial(&g_inkc_fit)) {
/* a partial chain: its layers up now, before the expert cache and the tier size
* themselves; the per-matrix path shares their copies where it runs */
if (coli_vk_dense()) ink_vk_mark_dense(m, 0, end, 0);
inkc_place_init(m, NULL);
return;
}
if (!coli_vk_dense()) return; /* nothing marked: the CPU computes the dense matrices */
ink_vk_mark_dense(m, layer_begin, layer_end, 1);
}
#endif
static void model_init_range(Model *m, const char *snap, int cap, int bits,
int layer_begin, int layer_end,
int load_boundaries, int allocate_state,
int init_telemetry) {
memset(m, 0, sizeof(*m));
m->quant_bits = bits;
load_cfg(&m->c, snap);
st_init(&m->S, snap);
/* densa pre-quantizzata, se il container c'e' (INK_DENSE_Q4=0 la ignora).
* Sta in una SOTTOCARTELLA apposta: nella dir dello snapshot i nomi tensore
* andrebbero in collisione con i bf16 originali, che restano intatti. */
{ char qd[2100]; snprintf(qd, sizeof(qd), "%s/dense-int4g64", snap);
const char *off = getenv("INK_DENSE_Q4");
struct stat qs;
if (!(off && *off == '0') && stat(qd, &qs) == 0 && S_ISDIR(qs.st_mode)) {
st_init(&m->Sq, qd);
if (m->Sq.n > 0) { m->has_q = 1;
fprintf(stderr, "[dense] container int4-gs64: %s (%d tensori)\n", qd, m->Sq.n); }
}
}
Cfg *c = &m->c;
if (layer_end == 0) layer_end = c->n_layers;
if (layer_begin < 0 || layer_begin >= layer_end ||
layer_end > c->n_layers) {
fprintf(stderr, "invalid Inkling layer range [%d,%d) for %d layers\n",
layer_begin, layer_end, c->n_layers);
exit(1);
}
int D = c->hidden, K = c->conv_k;
double t0 = now_s();
#ifdef COLI_CUDA
if (!getenv("NOGPU")) {
int dev = getenv("GPU_DEV") ? atoi(getenv("GPU_DEV")) : 0;
if (ink_cuda_init(dev) == 0) {
g_cuda = 1;
fprintf(stderr, "[cuda] device %d ready, %.1f GB free — bf16 residents to VRAM\n",
dev, ink_cuda_free_bytes()/1e9);
} else fprintf(stderr, "[cuda] init failed, running on CPU\n");
}
#endif
#ifdef COLI_METAL
{ const char *me = getenv("COLI_METAL");
if (me && *me == '1' && !getenv("NOGPU")) {
/* Residency set ON by default for inkling: without it every MoE
* block pays per-buffer useResource churn — measured 0.17 vs 1.76
* tok/s decode on Inkling-Small. COLI_METAL_RESSET=0 opts out. */
setenv("COLI_METAL_RESSET", "1", 0);
if (coli_metal_init()) {
g_metal = 1;
fprintf(stderr, "[metal] ready — batched expert MoE on the Apple GPU\n");
} else fprintf(stderr, "[metal] init failed, running on CPU\n");
}
}
#endif
if (load_boundaries) {
m->embed = load_w(m, "model.embed_tokens.weight", 0);
{ const char *en = embed_norm_name(&m->S); m->embed_norm = en ? load_t(m, en) : NULL; }
m->final_norm = load_t(m, "model.norm.weight");
m->lm_head = load_w(m, "lm_head.weight", 1);
}
/* Inkling's audio "tower" is one embedding table + one RMSNorm. The int4
* containers are text-only, so these usually arrive via an audio.safetensors
* sidecar dropped in the snapshot dir (st_init indexes every *.safetensors).
* Absent tensors = text-only engine, exactly as before. */
if (load_boundaries && st_has(&m->S, "model.audio.encoder.weight")) {
/* SEC (GHSA-w696): mel_bins/mel_vocab come straight from config.json with
* no bounds. audio_embed_row indexes the table at (b*mel_vocab+v)*D with
* b<mel_bins, v<mel_vocab — so the table must have exactly
* mel_bins*mel_vocab rows or that index runs off the heap (bidirectional
* OOB read, config-controlled). Reconcile the real element count against
* the geometry before using it. */
st_tensor *aet = st_find(&m->S, "model.audio.encoder.weight");
if (c->mel_bins < 1 || c->mel_vocab < 1 || D < 1 ||
(int64_t)c->mel_bins * c->mel_vocab > INT64_MAX / D ||
!aet || aet->numel != (int64_t)c->mel_bins * c->mel_vocab * D) {
fprintf(stderr, "[audio] rejected: encoder table has %lld elements, "
"config geometry is %d bins x %d levels x D=%d\n",
aet ? (long long)aet->numel : -1, c->mel_bins, c->mel_vocab, D);
exit(1);
}
m->audio_enc = load_w(m, "model.audio.encoder.weight", 0);
m->audio_norm = load_t(m, "model.audio.final_norm.weight");
fprintf(stderr, "[audio] DMel encoder loaded (%d bins x %d levels -> D=%d)\n",
c->mel_bins, c->mel_vocab, D);
}
m->L = calloc(c->n_layers, sizeof(Layer));
for (int j = 0; j < 4; j++)
m->cs[j] = calloc(c->n_layers, sizeof(float*));
char nm[320];
for (int i = layer_begin; i < layer_end; i++) {
Layer *l = &m->L[i];
#define LD(field, suffix) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_t(m,nm)
#define LDW(field, suffix) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,i); l->field = load_w(m,nm,1)
LD(in_ln, "input_layernorm.weight");
LD(post_ln,"post_attention_layernorm.weight");
LDW(q, "self_attn.q_proj.weight"); LDW(k, "self_attn.k_proj.weight");
LDW(v, "self_attn.v_proj.weight"); LDW(r, "self_attn.r_proj.weight");
LDW(o, "self_attn.o_proj.weight");
LD(qn,"self_attn.q_norm.weight"); LD(kn,"self_attn.k_norm.weight");
LD(relp, "self_attn.rel_logits_proj.proj");
LD(k_cw, "self_attn.k_sconv.conv1d.weight");
LD(v_cw, "self_attn.v_sconv.conv1d.weight");
LD(a_cw, "attn_sconv.conv1d.weight");
LD(m_cw, "mlp_sconv.conv1d.weight");
if (!c->sparse[i]) {
LDW(dg, "mlp.gate_proj.weight"); LDW(du, "mlp.up_proj.weight"); LDW(dd, "mlp.down_proj.weight");
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.global_scale",i); l->dgs = load_scalar(m,nm,1.f);
} else {
LD(router, "mlp.gate.weight");
LD(rbias, "mlp.gate.e_score_correction_bias");
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.gate.global_scale",i); l->rgs = load_scalar(m,nm,1.f);
LDW(sh_g, "mlp.shared_experts.gate_proj");
LDW(sh_u, "mlp.shared_experts.up_proj");
LDW(sh_d, "mlp.shared_experts.down_proj");
#ifdef COLI_METAL
/* Move the bf16 shared-expert weights into one page-aligned slab
* (gates, then ups, then downs) and repoint the Wt handles into
* it: the CPU path reads the same bytes it always did, and the
* slab registers once so the GPU fmt=5 path can resolve it. */
if (g_metal && l->sh_g.h && l->sh_u.h && l->sh_d.h) {
int64_t I = c->moe_inter, ns = c->n_shared;
size_t one = (size_t)ns*I*D*2, pg = 16384;
size_t len = (3*one + pg - 1) / pg * pg;
void *sl = NULL;
if (!posix_memalign(&sl, pg, len)) {
memcpy((char*)sl, l->sh_g.h, one);
memcpy((char*)sl + one, l->sh_u.h, one);
memcpy((char*)sl + 2*one, l->sh_d.h, one);
free(l->sh_g.h); free(l->sh_u.h); free(l->sh_d.h);
l->sh_g.h = (uint16_t*)sl;
l->sh_u.h = (uint16_t*)((char*)sl + one);
l->sh_d.h = (uint16_t*)((char*)sl + 2*one);
coli_metal_register(sl, len);
}
}
#endif
}
#undef LD
#undef LDW
/* conv states: raw inputs of the previous K-1 steps, zero-init */
int kvdim = L_KV(c,i) * L_HD(c,i);
for (int j = 0; j < 4; j++) if (allocate_state) {
int C = (j < 2) ? kvdim : D;
m->cs[j][i] = calloc((int64_t)C * (K-1), sizeof(float));
}
}
/* container detection: converted snapshots store experts as U8 + .qs.
* rb13/rb2 = bytes per packed row (D/2|D and I/2|I for int4|int8) */
int64_t I = c->moe_inter, E = c->n_experts;
for (int i = layer_begin; i < layer_end; i++) if (c->sparse[i]) {
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.gate_up_proj",i);
st_tensor *t = st_find(&m->S, nm);
if (t && t->dtype == 3) {
m->xq = 1;
m->rb13 = t->nbytes / (E * 2*I);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.down_proj",i);
st_tensor *t2 = st_find(&m->S, nm);
m->rb2 = t2->nbytes / (E * (int64_t)D);
if (m->rb13 != D && m->rb13*2 != D) { fprintf(stderr,"unsupported container row size %lld\n",(long long)m->rb13); exit(1); }
}
break;
}
int nsp = 0; for (int i = layer_begin; i < layer_end; i++) nsp += c->sparse[i];
int64_t slotb = m->xq ? m->rb13*2*I + m->rb2*D + (2*I+D)*4
: m->quant_bits ? 3*I*D + (2*I+D)*4 : 3*I*D*4;
double avail_auto = cap <= 0 ? mem_avail_bytes() : 0;
#ifdef COLI_VULKAN
/* the device opens here, before the expert cache is sized: with the dense matrices on
* the device only (COLI_VK_DENSE_HOST) their host copies are gone before the auto cap
* measures what RAM is left (on a GPU that shares the RAM the device copy holds it) */
double t_vk = now_s();
ink_vk_init_model(m, layer_begin, layer_end);
t0 += now_s() - t_vk; /* the device's start is not the weights' load (dense_load_s), as before */
if (cap <= 0 && g_ink_dho) avail_auto = mem_avail_bytes();
#endif
if (cap <= 0) { /* auto: fit the LRU in available RAM, 20% + 4 GB headroom */
double avail = avail_auto;
cap = avail > 0 ? (int)((avail*0.80 - 4e9) / ((double)slotb * (nsp ? nsp : 1))) : 16;
if (cap < 4) cap = 4;
if (cap > c->n_experts) cap = c->n_experts;
fprintf(stderr, "[cap auto] %d experts/layer (%.1f GB cache budget)\n",
cap, (double)cap*slotb*nsp/1e9);
#ifdef COLI_VULKAN
if (g_ink_dho)
fprintf(stderr, "[cap auto] dense weights on the device only: RAM measured after %.3f GB of host copies "
"went (%.3f GB available)\n", coli_vk_dense_host_dropped_bytes() / 1e9, avail / 1e9);
#endif
}
m->cache = calloc(c->n_layers, sizeof(LCache));
for (int i = layer_begin; i < layer_end; i++) {
LCache *lc = &m->cache[i];
lc->cap = cap;
if (c->sparse[i]) lc->slots = calloc(cap, sizeof(Slot));
if (c->sparse[i]) {
lc->slot_by_expert = malloc((size_t)E * sizeof(int));
if (!lc->slot_by_expert) { fprintf(stderr,"OOM expert cache index\n"); exit(1); }
for (int e = 0; e < E; e++) lc->slot_by_expert[e] = -1;
}
}
/* container mode: slot storage is one page-aligned slab per sparse layer
* (weights) plus one for the row scales, with every slot's pointers carved
* out up front. Slots then recycle their region on eviction — no per-slot
* malloc, and (under Metal) each slab registers ONCE for zero-copy GPU
* resolve instead of churning register/unregister on every fill. */
if (m->xq) {
int64_t st13 = m->rb13*2*I, st2 = m->rb2*D;
size_t pg = 16384;
size_t wlen = ((size_t)cap*(st13+st2) + pg - 1) / pg * pg;
size_t slen = ((size_t)cap*(2*I+D)*4 + pg - 1) / pg * pg;
for (int i = layer_begin; i < layer_end; i++) {
if (!c->sparse[i]) continue;
void *wsl = NULL, *ssl = NULL;
if (posix_memalign(&wsl, pg, wlen) || posix_memalign(&ssl, pg, slen)) {
fprintf(stderr, "OOM expert slab layer %d (%zu MB)\n", i, (wlen+slen)>>20); exit(1); }
for (int k = 0; k < cap; k++) {
Slot *s = &m->cache[i].slots[k];
s->p13 = (uint8_t*)wsl + (int64_t)k*(st13+st2);
s->p2 = s->p13 + st13;
s->s13 = (float*)ssl + (int64_t)k*(2*I+D);
s->s2 = s->s13 + 2*I;
}
#ifdef COLI_METAL
if (g_metal) { coli_metal_register(wsl, wlen); coli_metal_register(ssl, slen); }
#endif
}
}
/* usage counters; seeded from a previous run's history when present */
if (init_telemetry) {
rt_init("inkling", c->n_layers, E);
for (int i = 0; i < c->n_layers; i++) if (!c->sparse[i]) rt_drop_row(i);
rt_drop_row(c->n_layers); /* inkling has no MTP row */
m->eusage = rt_counts_all(); /* alias: the bump sites stay as they are */
}
m->dense_load_s = now_s() - t0;
}
static void model_init(Model *m, const char *snap, int cap, int bits) {
model_init_range(m, snap, cap, bits, 0, 0, 1, 1, 1);
}
static double mem_avail_bytes(void) {
/* Shared probe: Linux MemAvailable, macOS free+inactive+purgeable,
* Windows ullAvailPhys/commit. The #else here used to return 0, so
* Windows auto-cap was always 16 experts/layer and never warned. */
double gb = compat_mem_available_gb();
if (gb > 0.0) return gb * 1e9;
static int noted = 0;
if (!noted) {
noted = 1;
fprintf(stderr, "[inkling] could not measure available RAM on this platform; "
"auto cache falls back to 16 experts/layer. Pass --cap to set it.\n");
}
return 0;
}
/* ---------- routed-expert slots: serial bookkeeping, parallel fills ---------- */
#ifdef COLI_CACHE_INDEX_TEST
static uint64_t g_slot_index_probes;
#define SLOT_INDEX_PROBE() (g_slot_index_probes++)
#else
#define SLOT_INDEX_PROBE() ((void)0)
#endif
/* Cache identity is mutated only by slot_acquire(), which keeps this index in
* lockstep with the slot. Fills change bytes/filled but never identity. The
* validation makes a stale/corrupt entry a miss instead of serving another
* expert's weights; production slot_find then touches the LRU clock. */
static Slot *slot_indexed(Model *m, int layer, int eid) {
LCache *lc = &m->cache[layer];
if (eid < 0 || eid >= m->c.n_experts || !lc->slot_by_expert) return NULL;
SLOT_INDEX_PROBE();
int i = lc->slot_by_expert[eid];
if (i < 0 || i >= lc->n || lc->slots[i].eid != eid) return NULL;
return &lc->slots[i];
}
/* One byte per expert: routed in this turn or not. The dashboard's Brain tab
* reads it as the HITS bitmap after every turn (serve_hits), and it is cleared
* there. Lives outside INKLING_NO_MAIN because the routing site that marks it
* is compiled into the segment adapter object too. */
static void ehit_mark(Model *m, int layer, int eid) {
Cfg *c = &m->c;
if (!m->ehit) {
m->ehit = calloc((size_t)c->n_layers, sizeof(uint8_t *));
for (int i = 0; i < c->n_layers; i++) m->ehit[i] = calloc((size_t)c->n_experts, 1);
}
if (layer >= 0 && layer < c->n_layers && eid >= 0 && eid < c->n_experts) m->ehit[layer][eid] = 1;
}
static Slot *slot_find(Model *m, int layer, int eid) {
Slot *s = slot_indexed(m, layer, eid);
if (s) s->used = ++m->clock;
return s;
}
/* allocate a slot (or evict the LRU non-pinned one); serial callers only */
static Slot *slot_acquire(Model *m, int layer, int eid) {
LCache *lc = &m->cache[layer]; Cfg *c = &m->c;
if (eid < 0 || eid >= c->n_experts || !lc->slot_by_expert) {
fprintf(stderr,"layer %d: invalid expert id %d for cache index\n",layer,eid); exit(1); }
int64_t D = c->hidden, I = c->moe_inter, n13 = 2*I*D, n2 = D*I;
Slot *s;
if (lc->n < lc->cap) {
s = &lc->slots[lc->n++];
if (m->xq) { /* slab-carved at model_init; nothing to allocate */ }
else if (m->quant_bits) { s->q13 = malloc(n13); s->q2 = malloc(n2);
s->s13 = falloc(2*I); s->s2 = falloc(D);
if (!s->q13 || !s->q2) { fprintf(stderr,"OOM expert slot\n"); exit(1); } }
else { s->f13 = falloc(n13); s->f2 = falloc(n2); }
} else {
int lru = -1, dev = -1; /* an expert the Vulkan tier holds goes first (vkt_ram_first) */
for (int i = 0; i < lc->n; i++) {
if (lc->slots[i].pinned) continue;
if (lc->slots[i].eid >= 0 && vkt_ram_first(layer, lc->slots[i].eid)) {
if (dev < 0 || lc->slots[i].used < lc->slots[dev].used) dev = i;
continue;
}
if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
}
if (dev >= 0) { lru = dev; vkt_ram_gave(); }
if (lru < 0) { fprintf(stderr, "layer %d: cache cap %d entirely pinned\n", layer, lc->cap); exit(1); }
s = &lc->slots[lru];
}
int si = (int)(s - lc->slots);
if (s->eid >= 0 && s->eid < c->n_experts &&
lc->slot_by_expert[s->eid] == si)
lc->slot_by_expert[s->eid] = -1;
s->eid = eid; s->used = ++m->clock; s->filled = 0; s->pinned = 0;
lc->slot_by_expert[eid] = si;
return s;
}
/* pure I/O (+ optional requant): safe to run in parallel across slots */
static void slot_fill(Model *m, int layer, Slot *s) {
Cfg *c = &m->c;
int64_t D = c->hidden, I = c->moe_inter, n13 = 2*I*D, n2 = D*I;
int64_t eid = s->eid;
char nm[320], qs[340];
if (m->xq) {
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.gate_up_proj",layer);
read_u8_slice(&m->S, nm, s->p13, eid*2*I*m->rb13, 2*I*m->rb13);
snprintf(qs,sizeof(qs),"%s.qs",nm);
read_f32_slice(&m->S, qs, s->s13, eid*2*I, 2*I);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.down_proj",layer);
read_u8_slice(&m->S, nm, s->p2, eid*D*m->rb2, D*m->rb2);
snprintf(qs,sizeof(qs),"%s.qs",nm);
read_f32_slice(&m->S, qs, s->s2, eid*D, D);
} else if (m->quant_bits) {
float *tmp = falloc(n13 > n2 ? n13 : n2);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.gate_up_proj",layer);
read_f32_slice(&m->S, nm, tmp, eid*n13, n13);
quantize_rows(tmp, s->q13, s->s13, 2*I, D, m->quant_bits);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.down_proj",layer);
read_f32_slice(&m->S, nm, tmp, eid*n2, n2);
quantize_rows(tmp, s->q2, s->s2, D, I, m->quant_bits);
free(tmp);
} else {
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.gate_up_proj",layer);
read_f32_slice(&m->S, nm, s->f13, eid*n13, n13);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.down_proj",layer);
read_f32_slice(&m->S, nm, s->f2, eid*n2, n2);
}
s->filled = 1;
}
/* pin the top-N experts per sparse layer from a usage-history file (colibri
* .coli_usage convention: one uint32 count per expert per layer). Pins are
* regular cache slots flagged non-evictable, filled in parallel at startup.
* Toggles: PIN=off (or PIN=0) skips cache warming entirely (no seeding, no
* pins, cold LRU start); PIN_N=0 seeds the ranking from the history but pins
* nothing; PIN=<path> uses an alternate history file; PIN_N=<n> pin depth. */
static void pins_load(Model *m, const char *snap) {
Cfg *c = &m->c; int E = c->n_experts;
char up[2048];
const char *env = getenv("PIN");
if (env && (!strcmp(env, "off") || !strcmp(env, "0"))) {
fprintf(stderr, "[pin] cache warming disabled (PIN=%s)\n", env);
return;
}
if (env) snprintf(up, sizeof(up), "%s", env);
else snprintf(up, sizeof(up), "%s/.coli_usage", snap);
/* Reads both layouts: the IKU1 block every previous inkling wrote, and the shared
* text format. Dimension and identity checks moved into the reader, which refuses
* by name instead of with a generic 'ignoring'. Counts land straight in eusage. */
if (rt_load(up) <= 0) return;
int cap = m->cache[0].cap;
/* default: pin half the cap. Measured on the 975B: cap/4 (19/layer) gave
* 83.6% hit / 0.32 tok/s; 40/layer gave 95.6% / 0.80 tok/s — decode fills
* run at queue depth ~1, so every pinned expert removes a ~35ms stall. */
m->npin = getenv("PIN_N") ? atoi(getenv("PIN_N")) : cap/2;
if (m->npin > cap - 8) m->npin = cap - 8;
if (m->npin < 0) m->npin = 0;
Slot **ps = malloc((size_t)c->n_layers * m->npin * sizeof(Slot*));
int *pl = malloc((size_t)c->n_layers * m->npin * sizeof(int));
int np = 0;
for (int i = 0; i < c->n_layers; i++) {
const uint32_t *tmp = rt_counts(i); /* seeded by rt_load above */
if (!tmp || !c->sparse[i] || !m->npin) continue;
for (int r = 0; r < m->npin; r++) { /* top-N selection */
int best = -1; uint32_t bv = 0;
for (int e = 0; e < E; e++) {
int taken = slot_indexed(m, i, e) != NULL;
if (!taken && tmp[e] >= bv && tmp[e] > 0) { bv = tmp[e]; best = e; }
}
if (best < 0) break;
Slot *s = slot_acquire(m, i, best);
s->pinned = 1;
ps[np] = s; pl[np] = i; np++;
}
}
if (np) {
double t0 = now_s();
#pragma omp parallel for schedule(dynamic,1)
for (int j = 0; j < np; j++) slot_fill(m, pl[j], ps[j]);
fprintf(stderr, "[pin] %d experts pinned (%d/layer) from %s in %.1fs\n",
np, m->npin, up, now_s()-t0);
}
free(ps); free(pl);
}
/* usage snapshot: rewritten after every generation run (same contract as
* glm's .coli_usage — copy it aside if you need a stable ranking).
* USAGE_SAVE=0 skips the rewrite (e.g. benchmark loops that would skew the
* ranking); PIN=off also implies no save (that run never seeded counts). */
static int usage_save(Model *m, const char *snap) {
(void)m; /* the counters live in route_trace.h now */
char up[2048];
const char *env = getenv("PIN");
/* USAGE_SAVE=0 is honoured inside rt_save itself (#1039) */
if (env && (!strcmp(env, "off") || !strcmp(env, "0"))) return 0;
if (env) snprintf(up, sizeof(up), "%s", env);
else snprintf(up, sizeof(up), "%s/.coli_usage", snap);
/* One format for every engine now: sparse text with the dimension and identity
* header. The IKU1 block this replaced is still readable on load, so a history
* written by an older inkling keeps working and is rewritten in the new form. */
return rt_save(up, 1);
}
/* KV rows actually kept per layer: sliding layers only ever attend to the last
* `window` positions (t0 clamp in attention), so their cache is a ring of
* `window` rows instead of max_t — at 32k context that is a ~64x cut on the
* 5-of-6 sliding layers. Global layers keep the full max_t. Must be computed
* from the SAME max_t the buffers were allocated with (m->max_t). */
static inline int kv_ring_rows(const Cfg *c, int li, int max_t) {
return (c->local[li] && c->window > 0 && c->window < max_t) ? c->window : max_t;
}
/* A short convolution over S rows: in a multiplexed step each row is the next input
* of its own conversation's bank (sconv_apply over that one row), else the rows are a
* sequence through the Model's bank. */
static void ink_sconv(Model *m, float *seq, int S, int C, const float *w, int bank, int li) {
if (!m->mux_rows) { sconv_apply(seq, S, C, w, m->cs[bank][li], m->c.conv_k); return; }
for (int s = 0; s < S; s++)
sconv_apply(seq + (int64_t)s*C, 1, C, w, m->mux_rows[s].seq->cs[bank][li], m->c.conv_k);
}
/* ---------- attention (GQA + sliding/global + relative bias + K/V sconv) ---------- */
static void attention(Model *m, Layer *l, int li, float *x, int S, int pos0, float *out) {
Cfg *c = &m->c;
int D = c->hidden, H = L_HEADS(c,li), KV = L_KV(c,li), hd = L_HD(c,li), ext = L_EXT(c,li);
int local = c->local[li];
/* the ring made an over-run silent (t%win wraps instead of writing OOB), so
* fail fast here: every caller sizes the cache via kv_alloc before stepping */
const InkRow *mr = m->mux_rows; /* a multiplexed step: each row's own conversation and position */
if (mr) { for (int s = 0; s < S; s++) if (mr[s].pos >= m->max_t) { fprintf(stderr, "attention: pos %d exceeds kv alloc %d\n", mr[s].pos, m->max_t); exit(1); } }
else if (pos0 + S > m->max_t) { fprintf(stderr, "attention: pos %d+%d exceeds kv alloc %d\n", pos0, S, m->max_t); exit(1); }
int qdim = H*hd, kvdim = KV*hd, group = H/KV;
float *q = falloc((int64_t)S*qdim);
float *k = falloc((int64_t)S*kvdim);
float *vv = falloc((int64_t)S*kvdim);
float *rr = falloc((int64_t)S*H*c->d_rel);
matmul_w(q, x, l->q, S, D, qdim);
matmul_w(k, x, l->k, S, D, kvdim);
matmul_w(vv, x, l->v, S, D, kvdim);
matmul_w(rr, x, l->r, S, D, H*c->d_rel);
/* short convs on K and V (sequence-wise, over the raw projections) */
ink_sconv(m, k, S, kvdim, l->k_cw, 0, li);
ink_sconv(m, vv, S, kvdim, l->v_cw, 1, li);
/* per-head q/k rmsnorm (scaling below is 1/hd, not 1/sqrt(hd), because of this) */
for (int s = 0; s < S; s++) {
for (int h = 0; h < H; h++) rmsnorm_row(q + (int64_t)s*qdim + h*hd, q + (int64_t)s*qdim + h*hd, l->qn, hd, c->eps);
for (int h = 0; h < KV; h++) rmsnorm_row(k + (int64_t)s*kvdim + h*hd, k + (int64_t)s*kvdim + h*hd, l->kn, hd, c->eps);
}
/* Rows for positions inside this batch are read from the k/vv scratch, not
* the cache: with a ring, appending the whole batch up front could overwrite
* history rows that earlier queries in the batch still need. The scratch
* holds exactly what the cache would (post-sconv, post-rmsnorm), so the
* arithmetic is unchanged; the cache is appended after scoring. */
int win = kv_ring_rows(c, li, m->max_t);
float scale = 1.f / (float)hd;
float *ctx = falloc((int64_t)S*qdim);
#pragma omp parallel
{
float *rl = malloc(ext * sizeof(float));
float *sc = malloc((size_t)m->max_t * sizeof(float));
#pragma omp for collapse(2) schedule(static)
for (int h = 0; h < H; h++) {
for (int s = 0; s < S; s++) {
int qpos = mr ? mr[s].pos : pos0 + s;
int sp0 = mr ? qpos : pos0; /* the scratch's first position: the row's own in a multiplexed step */
int t0 = local && qpos - c->window + 1 > 0 ? qpos - c->window + 1 : 0;
int tb = sp0 > t0 ? sp0 : t0; /* first row served by the scratch */
float **Kc = mr ? mr[s].seq->K : m->K, **Vc = mr ? mr[s].seq->V : m->V;
int64_t srow = mr ? (int64_t)s*kvdim : 0;
/* mix the relative-bias bank for this (token, head): rl[dist] */
const float *rv = rr + (int64_t)s*H*c->d_rel + h*c->d_rel;
for (int e = 0; e < ext; e++) {
float acc = 0.f;
for (int d = 0; d < c->d_rel; d++) acc += rv[d] * l->relp[(int64_t)d*ext + e];
rl[e] = acc;
}
/* tau: log-length scaling on global layers (f32, per query pos) */
float tau = 1.f;
if (!local && c->log_floor > 0) {
double en = (double)(qpos + 1) / c->log_floor;
if (en > 1.0) tau = 1.f + c->log_alpha * (float)log(en);
}
const float *qv = q + (int64_t)s*qdim + h*hd;
const float *Kh = Kc[li] + ((int64_t)(h/group)*win)*hd;
const float *Kb = k + srow + (int64_t)(h/group)*hd;
for (int t = t0; t <= qpos; t++) {
const float *kv = t < tb ? Kh + (int64_t)(t % win)*hd
: Kb + (int64_t)(t - sp0)*kvdim;
float acc = dot_f32_lanes(qv, kv, hd);
int dist = qpos - t;
sc[t - t0] = tau * (acc*scale + (dist < ext ? rl[dist] : 0.f));
}
int n = qpos - t0 + 1;
softmax_row(sc, n);
float *cx = ctx + (int64_t)s*qdim + h*hd;
for (int d = 0; d < hd; d++) cx[d] = 0.f;
const float *Vh = Vc[li] + ((int64_t)(h/group)*win)*hd;
const float *Vb = vv + srow + (int64_t)(h/group)*hd;
for (int t = t0; t <= qpos; t++) {
const float *vrow = t < tb ? Vh + (int64_t)(t % win)*hd
: Vb + (int64_t)(t - sp0)*kvdim;
float a = sc[t - t0];
for (int d = 0; d < hd; d++) cx[d] += a * vrow[d];
}
}
}
free(rl); free(sc);
}
/* append K,V to the cache (ring on sliding layers); rows the ring would
* overwrite within this same batch are skipped, they can never be read */
int s0 = !mr && S - win > 0 ? S - win : 0; /* a multiplexed step: every row to its own cache */
for (int s = s0; s < S; s++) for (int h = 0; h < KV; h++) {
int t = mr ? mr[s].pos : pos0 + s;
float *Kd = mr ? mr[s].seq->K[li] : m->K[li], *Vd = mr ? mr[s].seq->V[li] : m->V[li];
memcpy(Kd + ((int64_t)h*win + t % win)*hd, k + (int64_t)s*kvdim + h*hd, hd*sizeof(float));
memcpy(Vd + ((int64_t)h*win + t % win)*hd, vv + (int64_t)s*kvdim + h*hd, hd*sizeof(float));
}
matmul_w(out, ctx, l->o, S, qdim, D);
free(q); free(k); free(vv); free(rr); free(ctx);
}
/* ---------- dense MLP ---------- */
static void dense_mlp(Model *m, Layer *l, float *x, int S, float *out) {
Cfg *c = &m->c; int D = c->hidden, I = c->dense_inter;
float *g = falloc((int64_t)S*I), *u = falloc((int64_t)S*I);
matmul_w(g, x, l->dg, S, D, I);
matmul_w(u, x, l->du, S, D, I);
for (int64_t i = 0; i < (int64_t)S*I; i++) g[i] = siluf(g[i]) * u[i];
matmul_w(out, g, l->dd, S, I, D);
for (int64_t i = 0; i < (int64_t)S*D; i++) out[i] *= l->dgs;
free(g); free(u);
}
/* ---------- MoE: sigmoid router + bias top-k, joint routed+shared weights ----------
* Three passes per layer call: (1) route every position and acquire slots,
* (2) fill ALL missing experts in one parallel burst (the NVMe wants queue
* depth — during prefill this batches the whole sequence's misses), then
* (3) compute. */
/* shared experts for all S positions: gamma inside (before down_proj is
* linear, so applied at the end). Factored out so the Metal path can run it
* on the CPU while the last routed-expert round is in flight on the GPU.
*
* Prefill batches positions so each shared matrix is traversed once per chunk,
* rather than once per token. Decode (S=1) deliberately stays on the original
* scalar path: its scratch remains tiny and the compiler sees a one-row GEMV.
* Bound the batch scratch to 64 MiB so a long prompt cannot turn this speedup
* into a memory spike. INK_SHARED_BATCH=0 restores the scalar path; a positive
* value sets a smaller maximum row count for deterministic A/B tests. */
static int shared_batch_rows(int S, int D, int I) {
if (S <= 1) return 1;
const char *env = getenv("INK_SHARED_BATCH");
if (env) {
int requested = atoi(env);
if (requested <= 0) return 1;
if (requested < S) S = requested;
}
int64_t row_bytes = ((int64_t)2*I + D) * (int64_t)sizeof(float);
int64_t bounded = (64LL << 20) / (row_bytes > 0 ? row_bytes : 1);
if (bounded < 1) bounded = 1;
if (S > bounded) S = (int)bounded;
return S;
}
static void shared_experts_cpu(Model *m, Layer *l, const float *x, int S,
float *out, const float *wgt,
float *g, float *u, float *hh) {
Cfg *c = &m->c;
int D = c->hidden, K = c->topk, I = c->moe_inter, ns = c->n_shared;
if (ns <= 0 || S <= 0) return;
int B = shared_batch_rows(S, D, I);
if (B == 1) {
for (int s = 0; s < S; s++) {
const float *xs = x + (int64_t)s*D;
float *os = out + (int64_t)s*D;
const float *w = wgt + (int64_t)s*(K+ns);
for (int j = 0; j < ns; j++) {
matmul_w(g, xs, wt_off_i(l->sh_g, (int64_t)j*I*D, D), 1, D, I);
matmul_w(u, xs, wt_off_i(l->sh_u, (int64_t)j*I*D, D), 1, D, I);
for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];
matmul_w(hh, g, wt_off_i(l->sh_d, (int64_t)j*D*I, I), 1, I, D);
for (int d = 0; d < D; d++) os[d] += w[K+j] * hh[d];
}
}
return;
}
float *bg = falloc((int64_t)2*B*I), *bu = bg + (int64_t)B*I;
float *bh = falloc((int64_t)B*D);
for (int base = 0; base < S; base += B) {
int rows = S - base < B ? S - base : B;
for (int j = 0; j < ns; j++) {
matmul_w(bg, x + (int64_t)base*D,
wt_off_i(l->sh_g, (int64_t)j*I*D, D), rows, D, I);
matmul_w(bu, x + (int64_t)base*D,
wt_off_i(l->sh_u, (int64_t)j*I*D, D), rows, D, I);
for (int64_t q = 0; q < (int64_t)rows*I; q++) bg[q] = siluf(bg[q]) * bu[q];
matmul_w(bh, bg, wt_off_i(l->sh_d, (int64_t)j*D*I, I), rows, I, D);
for (int s = 0; s < rows; s++) {
float *os = out + (int64_t)(base+s)*D;
const float *w = wgt + (int64_t)(base+s)*(K+ns);
const float *hs = bh + (int64_t)s*D;
for (int d = 0; d < D; d++) os[d] += w[K+j] * hs[d];
}
}
}
free(bg); free(bh);
}
#ifdef COLI_VULKAN
/* ---- the Vulkan routed-expert tier (vk_tier.c) ---------------------------------
* With the tier on, one MoE layer runs per block of rows: the block's resident
* experts go to the device as one batch (vkt_issue), the CPU computes the other
* (row, rank) pairs into rows of their own -- the same cache rounds and kernels as
* moe() -- and the shared experts into a buffer of their own while the batch runs.
* Then every rank of every row joins `out` in rank order, the device's and the CPU's
* alike, and the shared experts after them, as moe() adds them; so the order of the
* sum never depends on which experts were resident. Every expert the CPU computed
* passes its RAM bytes to the tier (vkt_note), which may promote it. */
#define INK_VK_ROWS 64
/* How the slots hold an expert: the container's packed int4 or int8 rows with f32
* row scales (gate and up in one fused [2I, D] tensor: up starts I rows in), the
* runtime-quantized int8 rows (int4 values when bits <= 4), or f32 at bits=0.
* moe() reads down with gate/up's kernel, so the two share one kind. 0: none. */
static int ink_vk_fmt_of(const Model *m, VktFmt *f) {
int D = m->c.hidden, I = m->c.moe_inter;
if (m->xq) {
int q4 = m->rb13 * 2 == D;
if (q4 ? m->rb2 * 2 != I : m->rb2 != I) return 0;
*f = (VktFmt){q4 ? VKT_SRC_I4U_PAIRS_ROW : VKT_SRC_I8_ROW, 0};
} else if (m->quant_bits) *f = (VktFmt){m->quant_bits <= 4 ? VKT_SRC_I8_AS_I4_ROW : VKT_SRC_I8_ROW, 0};
else *f = (VktFmt){VKT_SRC_F32, 0};
return 1;
}
static VktExpertSrc ink_vk_slot_src(const Model *m, const Slot *e) {
int64_t D = m->c.hidden, I = m->c.moe_inter;
if (m->xq) return (VktExpertSrc){e->p13, e->p13 + I * m->rb13, e->p2, e->s13, e->s13 + I, e->s2};
if (m->quant_bits) return (VktExpertSrc){e->q13, e->q13 + I * D, e->q2, e->s13, e->s13 + I, e->s2};
return (VktExpertSrc){e->f13, e->f13 + I * D, e->f2, NULL, NULL, NULL};
}
/* Is the expert in this layer's RAM cache now (the tier's balance asks)? */
static int ink_vk_in_ram(void *ctx, int layer, int e) {
Slot *s = slot_indexed((Model *)ctx, layer, e);
return s && s->filled;
}
/* Bytes the dense matrices marked for the device will take there (the budget leaves
* them room), from the geometry: int8 1 byte, int4-g64 0.5625, f32 4, bf16 2. */
static size_t inkc_fit_later(const Model *m); /* inkling_chain.h */
static size_t ink_vk_dense_bytes(const Model *m) {
const Cfg *c = &m->c;
double b = 0;
if (g_inkc_fit.L && vkc_fit_partial(&g_inkc_fit)) /* a partial chain: its layers placed already */
return inkc_fit_later(m);
if (g_ink_dho) return 0; /* placed already (ink_dho_place): the free memory the tier reads counts them */
#define INK_VKB(w, n) do { const Wt *w_ = (w); int f_ = w_->vk ? ink_vk_fmt(w_) : 0; \
b += (double)(n) * (f_ == 1 ? 1.0 : f_ == 4 ? 0.5625 : f_ == 10 ? 4.0 : f_ == 11 ? 2.0 : 0.0); } while (0)
INK_VKB(&m->lm_head, (double)c->vocab * c->hidden);
for (int i = 0; i < c->n_layers; i++) {
const Layer *l = &m->L[i];
double D = c->hidden, q = (double)L_HEADS(c, i) * L_HD(c, i), kv = (double)L_KV(c, i) * L_HD(c, i);
INK_VKB(&l->q, D * q); INK_VKB(&l->k, D * kv); INK_VKB(&l->v, D * kv);
INK_VKB(&l->r, D * L_HEADS(c, i) * c->d_rel); INK_VKB(&l->o, q * D);
INK_VKB(&l->dg, D * c->dense_inter); INK_VKB(&l->du, D * c->dense_inter); INK_VKB(&l->dd, D * c->dense_inter);
double sh = (double)c->n_shared * c->moe_inter * D;
INK_VKB(&l->sh_g, sh); INK_VKB(&l->sh_u, sh); INK_VKB(&l->sh_d, sh);
}
#undef INK_VKB
return (size_t)b;
}
/* A slot of its own for the warm start's reads (slot_fill's buffers), and its free. */
static int ink_vk_tmp_slot(const Model *m, Slot *s, int eid) {
int64_t D = m->c.hidden, I = m->c.moe_inter;
memset(s, 0, sizeof *s);
s->eid = eid;
if (m->xq) { s->p13 = malloc((size_t)(m->rb13 * 2 * I)); s->p2 = malloc((size_t)(m->rb2 * D)); }
else if (m->quant_bits) { s->q13 = malloc((size_t)(2 * I * D)); s->q2 = malloc((size_t)(D * I)); }
else { s->f13 = malloc((size_t)(2 * I * D) * sizeof(float)); s->f2 = malloc((size_t)(D * I) * sizeof(float)); }
s->s13 = malloc((size_t)(2 * I + D) * sizeof(float));
if (s->s13) s->s2 = s->s13 + 2 * I;
return s->s13 && (m->xq ? s->p13 && s->p2 : m->quant_bits ? s->q13 && s->q2 : s->f13 && s->f2);
}
static void ink_vk_tmp_free(Slot *s) {
free(s->p13); free(s->p2); free(s->q13); free(s->q2); free(s->f13); free(s->f2); free(s->s13);
}
/* COLI_VULKAN=1: describe the experts to the tier, after the weights and the history
* (pins_load reads it in the generate and serve modes; the oracle reads none), and
* warm it from that history: the hottest experts first, read from disk in parallel. */
static int ink_vk_load(void *ctx, int layer, int e, VktExpertSrc *src, void **h);
static void ink_vk_unhold(void *ctx, void *h);
static int ink_vk_load_batch(void *ctx, int layer, const int *e, int n, VktExpertSrc *srcs, void **h);
static void ink_vk_tier_start(Model *m) {
Cfg *c = &m->c;
if (!g_vk_ready) return;
#ifdef COLI_CUDA
if (g_cuda) {
if (vkt_wanted()) fprintf(stderr, "[VK] tier inkling: the CUDA backend is on and wins; the Vulkan tier stays off\n");
return;
}
#endif
#ifdef COLI_METAL
if (g_metal) {
if (vkt_wanted()) fprintf(stderr, "[VK] tier inkling: the Metal expert path is on and wins; the Vulkan tier stays off\n");
return;
}
#endif
VktFmt f;
if (g_vk_tier_try && !ink_vk_fmt_of(m, &f))
fprintf(stderr, "[VK] tier inkling: container rows of %lld/%lld bytes are no expert format the tier reads; the experts stay on the CPU\n",
(long long)m->rb13, (long long)m->rb2);
else if (g_vk_tier_try) {
int nsp = 0;
for (int i = 0; i < c->n_layers; i++) nsp += c->sparse[i];
int64_t D = c->hidden, I = c->moe_inter;
int64_t slotb = m->xq ? m->rb13*2*I + m->rb2*D + (2*I+D)*4
: m->quant_bits ? 3*I*D + (2*I+D)*4 : 3*I*D*4;
VktConfig vc = {.engine = "inkling", .layers = c->n_layers, .experts = c->n_experts,
.hidden = c->hidden, .inter = c->moe_inter, .topk = c->topk,
.gate_up = f, .down = f, .act = VKT_ACT_SWIGLU,
.max_rows = INK_VK_ROWS * c->topk,
.ram_reserve = (size_t)slotb * (size_t)m->cache[0].cap * (size_t)nsp,
.dense_bytes = ink_vk_dense_bytes(m),
.in_ram = ink_vk_in_ram, .ram_ctx = m,
.load = ink_vk_load, .release = ink_vk_unhold, .load_ctx = m, .load_batch = ink_vk_load_batch};
atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if (vkt_init(&vc, m->eusage)) {
atexit(vkt_shutdown);
int all = c->n_layers * c->n_experts;
int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));
const char *warm = getenv("COLI_VK_TIER_WARM"); /* 0: no warm start, the tier fills as experts pass by */
int n = pl && pe && !(warm && *warm == '0') ? vkt_plan(pl, pe, all) : 0;
if (n > 0) {
double t0 = now_s();
#pragma omp parallel for schedule(dynamic, 4)
for (int i = 0; i < n; i++) {
Slot tmp;
if (ink_vk_tmp_slot(m, &tmp, pe[i])) {
slot_fill(m, pl[i], &tmp);
VktExpertSrc src = ink_vk_slot_src(m, &tmp);
vkt_put(pl[i], pe[i], &src);
} else vkt_put(pl[i], pe[i], NULL); /* no memory: the planned slot is released */
ink_vk_tmp_free(&tmp);
}
vkt_put_done();
fprintf(stderr, "[VK] tier inkling: warm start, %d experts from the history in %.1fs\n", n, now_s() - t0);
}
free(pl); free(pe);
}
}
if (!vkt_ready() && !coli_vk_dense() && coli_vk_dense_decide("inkling", 0, 1)) { /* no tier after all */
g_vk_tier_try = 0;
/* a partial chain's layers are up already (the chain's copies): the rest stays on the CPU */
if (!vkc_fit_partial(&g_inkc_fit)) ink_vk_mark_dense(m, 0, c->n_layers, 1);
}
g_vk_tier_try = vkt_ready();
}
/* The tier's streaming (a big prompt chunk's cold experts on the device): an expert's
* bytes through the layer cache as moe()'s rounds get them; a group at once, filled in
* parallel, up to the cache's capacity (a slot stays the expert's until the next
* acquire). Hits and misses count as the CPU path counts them. */
static Slot *ink_vk_get(Model *m, int layer, int e, Slot **fill, int *nfill) {
Slot *s = slot_find(m, layer, e);
if (s) m->hits++;
else { m->miss++; s = slot_acquire(m, layer, e); fill[(*nfill)++] = s; }
return s;
}
static int ink_vk_load_batch(void *ctx, int layer, const int *e, int n, VktExpertSrc *srcs, void **h) {
Model *m = ctx;
int cap = m->cache[layer].cap;
if (n > cap) n = cap;
if (n < 1 || n > 64) return 0;
Slot *use[64], *fill[64]; int nfill = 0;
for (int i = 0; i < n; i++) use[i] = ink_vk_get(m, layer, e[i], fill, &nfill);
if (nfill) {
double tf = now_s();
#pragma omp parallel for schedule(dynamic,1)
for (int j = 0; j < nfill; j++) slot_fill(m, layer, fill[j]);
m->t_fill += now_s() - tf;
}
for (int i = 0; i < n; i++) {
if (use[i]->eid != e[i]) return i; /* the cache served another: the rest one by one */
srcs[i] = ink_vk_slot_src(m, use[i]); h[i] = use[i];
}
return n;
}
static int ink_vk_load(void *ctx, int layer, int e, VktExpertSrc *src, void **h) {
Model *m = ctx;
Slot *fill[1]; int nfill = 0;
Slot *s = ink_vk_get(m, layer, e, fill, &nfill);
if (nfill) { double tf = now_s(); slot_fill(m, layer, s); m->t_fill += now_s() - tf; }
if (s->eid != e) return 0;
*src = ink_vk_slot_src(m, s); *h = s;
return 1;
}
static void ink_vk_unhold(void *ctx, void *h) { (void)ctx; (void)h; }
static void ink_vk_tier_report(const Model *m, const char *scope) {
vkt_report(scope, m->hits, m->miss); /* nothing when the tier never started */
}
/* The CPU's pairs of a block (want[i] set) into ctb[i]: moe()'s rounds of at most
* `cap` pairs -- acquire, fill in parallel, check, compute -- with its kernels. The
* pairs' history and HITS were counted by the caller. */
static void ink_vk_cpu_pairs(Model *m, int layer, const float *x, int n, const int *ib,
const uint8_t *want, float *ctb, float *g) {
Cfg *c = &m->c;
int D = c->hidden, K = c->topk, I = c->moe_inter;
float *u = g + I;
int cap = m->cache[layer].cap; if (cap < 1) cap = 1;
int q4 = m->xq && m->rb13*2 == D;
Slot **use = malloc((size_t)cap * sizeof(Slot *)), **fill = malloc((size_t)cap * sizeof(Slot *));
int *pi = malloc((size_t)cap * sizeof(int));
if (!use || !fill || !pi) { fprintf(stderr, "OOM ink_vk_cpu_pairs\n"); exit(1); }
for (int i = 0; i < n; ) {
int nr = 0, nfill = 0;
for (; i < n && nr < cap; i++) {
if (!want[i] || ib[i] < 0) continue;
Slot *e = slot_find(m, layer, ib[i]);
if (e) m->hits++;
else { m->miss++; e = slot_acquire(m, layer, ib[i]); fill[nfill++] = e; }
use[nr] = e; pi[nr] = i; nr++;
}
if (nfill) {
double tf = now_s();
#pragma omp parallel for schedule(dynamic,1)
for (int j = 0; j < nfill; j++) slot_fill(m, layer, fill[j]);
m->t_fill += now_s() - tf;
}
for (int r = 0; r < nr; r++) {
if (use[r]->eid != ib[pi[r]]) {
fprintf(stderr, "layer %d: cache served expert %d for requested expert %d\n",
layer, use[r]->eid, ib[pi[r]]);
exit(1);
}
}
for (int r = 0; r < nr; r++) {
Slot *e = use[r];
const float *xs = x + (int64_t)(pi[r] / K) * D;
float *hh = ctb + (int64_t)pi[r] * D;
if (m->xq) {
if (q4) {
matmul_q4(g, xs, e->p13, e->s13, D, 2*I); /* gate rows then up rows */
for (int k = 0; k < I; k++) g[k] = siluf(g[k]) * u[k];
matmul_q4(hh, g, e->p2, e->s2, I, D);
} else {
matmul_q(g, xs, (int8_t*)e->p13, e->s13, D, 2*I);
for (int k = 0; k < I; k++) g[k] = siluf(g[k]) * u[k];
matmul_q(hh, g, (int8_t*)e->p2, e->s2, I, D);
}
} else if (m->quant_bits) {
matmul_q(g, xs, e->q13, e->s13, D, 2*I);
for (int k = 0; k < I; k++) g[k] = siluf(g[k]) * u[k];
matmul_q(hh, g, e->q2, e->s2, I, D);
} else {
matmul(g, xs, e->f13, 1, D, 2*I);
for (int k = 0; k < I; k++) g[k] = siluf(g[k]) * u[k];
matmul(hh, g, e->f2, 1, I, D);
}
VktExpertSrc vs = ink_vk_slot_src(m, e);
vkt_note(layer, e->eid, &vs);
}
}
free(use); free(fill); free(pi);
}
/* moe() with the tier on, after its routing pass: idx/keff/wgt as moe() made them;
* routed_only leaves the shared experts out (the dense chain runs them). */
static void ink_vk_moe(Model *m, Layer *l, int layer, const float *x, int S, float *out,
const int *idx, const int *keff, const float *wgt, int routed_only) {
Cfg *c = &m->c;
int D = c->hidden, K = c->topk, I = c->moe_inter, ns = c->n_shared;
int B = vkt_step_rows(S, INK_VK_ROWS); /* a whole prompt chunk when the tier streams */
float *ctb = falloc((int64_t)B * K * D), *shb = ns > 0 && !routed_only ? falloc((int64_t)B * D) : NULL;
float *g = falloc(2*I), *u = g + I, *hh = falloc(D);
int *ib = malloc((size_t)B * K * sizeof(int));
uint8_t *taken = malloc((size_t)B * K), *want = malloc((size_t)B * K);
const float **dev = malloc((size_t)B * K * sizeof(*dev));
if (!ib || !taken || !want || !dev) { fprintf(stderr, "OOM ink_vk_moe\n"); exit(1); }
for (int s0 = 0; s0 < S; s0 += B) {
int rows = S - s0 < B ? S - s0 : B, n = rows * K;
const float *xb = x + (int64_t)s0 * D;
double te = now_s(), f0 = m->t_fill, sh = 0;
for (int i = 0; i < n; i++) { /* TOPP's trimmed ranks: -1, unused */
int s = s0 + i / K, kk = i % K;
ib[i] = kk < keff[s] ? idx[(int64_t)s * K + kk] : -1;
if (ib[i] < 0) continue;
if (m->eusage && m->eusage[layer]) m->eusage[layer][ib[i]]++;
ehit_mark(m, layer, ib[i]);
}
int ndev = vkt_issue(layer, xb, rows, K, ib, taken);
for (int i = 0; i < n; i++) want[i] = ib[i] >= 0 && !taken[i];
ink_vk_cpu_pairs(m, layer, xb, n, ib, want, ctb, g);
if (shb) {
double ts = now_s();
memset(shb, 0, (size_t)rows * D * sizeof(float));
shared_experts_cpu(m, l, xb, rows, shb, wgt + (int64_t)s0 * (K + ns), g, u, hh);
sh = now_s() - ts; m->t_shared += sh;
}
if (ndev && !vkt_join(dev)) { /* the batch failed (the tier stops): those pairs here */
ink_vk_cpu_pairs(m, layer, xb, n, ib, taken, ctb, g);
memset(taken, 0, (size_t)n);
}
for (int s = 0; s < rows; s++) {
float *os = out + (int64_t)(s0 + s) * D;
const float *w = wgt + (int64_t)(s0 + s) * (K + ns);
for (int kk = 0; kk < K; kk++) {
int i = s * K + kk;
if (ib[i] < 0) continue;
const float *cp = taken[i] ? dev[i] : ctb + (int64_t)i * D;
for (int d = 0; d < D; d++) os[d] += w[kk] * cp[d];
}
if (shb) { const float *sp = shb + (int64_t)s * D; for (int d = 0; d < D; d++) os[d] += sp[d]; }
}
m->t_expert += now_s() - te - sh - (m->t_fill - f0);
}
free(ctb); free(shb); free(g); free(hh); free(ib); free(taken); free(want); free(dev);
}
#endif
/* moe() with two doors for the dense chain (inkling_chain.h): lg_in, the router's
* logits [S][E+ns] the device computed (NULL: computed here); shw, when not NULL, asks
* for the routed experts only, and receives the shared experts' combine weights,
* shw[j*S + s] (the device runs the shared experts and adds them in moe()'s order).
* Under Metal the shared experts' GPU block ignores shw: the chain does not run there. */
static void moe_ex(Model *m, Layer *l, int layer, float *x, int S, float *out, const float *lg_in, float *shw) {
Cfg *c = &m->c;
int D = c->hidden, E = c->n_experts, K = c->topk, I = c->moe_inter, ns = c->n_shared;
int ET = E + ns;
float *logits = lg_in ? (float *)lg_in : falloc((int64_t)S*ET);
if (!lg_in) matmul(logits, x, l->router, S, D, ET);
memset(out, 0, (int64_t)S*D*sizeof(float));
int *idx = malloc((size_t)S*K*sizeof(int));
int *keff = malloc((size_t)S*sizeof(int)); /* routed effettivi per token (TOPP) */
float *wgt = malloc((size_t)S*(K+ns)*sizeof(float));
Slot **use = malloc((size_t)S*K*sizeof(Slot*));
Slot **fill = malloc((size_t)S*K*sizeof(Slot*));
int *fl = malloc((size_t)S*K*sizeof(int));
int nfill = 0;
/* pass 1: routing + slot bookkeeping (serial) */
for (int s = 0; s < S; s++) {
float *lg = logits + (int64_t)s*ET;
int *si = idx + (int64_t)s*K;
/* selection: sigmoid(routed) + correction bias, top-K */
for (int kk = 0; kk < K; kk++) {
int best = -1; float bv = -1e30f;
for (int e = 0; e < E; e++) {
int taken = 0; for (int j = 0; j < kk; j++) if (si[j]==e){taken=1;break;}
float ch = sigmoidf(lg[e]) + l->rbias[e];
if (!taken && ch > bv) { bv = ch; best = e; }
}
/* SEC: all-NaN logits leave best at -1, and lg[-1] / eusage[-1] are
* next. See rt_router_pick in route_trace.h. */
si[kk] = rt_router_pick(best, kk, E, layer);
}
/* combine weights: sigmoids of the raw logits of (topK routed + shared),
* normalized to sum 1 over all K+ns, x route_scale x gate.global_scale */
float *w = wgt + (int64_t)s*(K+ns); float sum = 0.f;
for (int kk = 0; kk < K; kk++) { w[kk] = sigmoidf(lg[si[kk]]); sum += w[kk]; }
for (int j = 0; j < ns; j++) { w[K+j] = sigmoidf(lg[E+j]); sum += w[K+j]; }
for (int kk = 0; kk < K+ns; kk++) w[kk] *= c->route_scale * l->rgs / sum;
/* TOPP: tieni i routed fino al peso cumulato p, scarta la coda. Stessa
* semantica di colibri.c (g_topp) e kimi_k3.c (K3_TOPP): NON rinormalizza,
* il peso scartato semplicemente non contribuisce. E' una leva di QUALITA'
* — cambia il calcolo — quindi opt-in e annunciata all'avvio.
* Su questo motore vale piu' che sugli altri: Inkling e' limitato dal disco
* (topk=6 x 28 MB di esperto x 66 layer ~ 11 GB letti per token), e ogni
* esperto scartato e' un esperto NON letto.
* L'ordine di si[] segue sigmoid(logit)+bias mentre il peso e' sigmoid(logit)
* senza bias, quindi i pesi non sono garantiti decrescenti: si riordina la
* coppia (peso, id) prima di tagliare, come fa colibri.c. */
keff[s] = K;
m->ereq += K;
if (g_topp > 0.f && g_topp < 1.f) {
for (int a = 1; a < K; a++) { int ii = si[a]; float ww = w[a]; int b = a-1;
while (b >= 0 && w[b] < ww) { w[b+1] = w[b]; si[b+1] = si[b]; b--; }
w[b+1] = ww; si[b+1] = ii; }
float tot = 1e-20f; for (int kk = 0; kk < K; kk++) tot += w[kk];
float cum = 0;
for (int kk = 0; kk < K; kk++) { cum += w[kk];
if (cum >= g_topp * tot) { keff[s] = kk+1; break; } }
}
m->euse += keff[s];
}
if (shw) for (int s = 0; s < S; s++) for (int j = 0; j < ns; j++) shw[(int64_t)j*S + s] = wgt[(int64_t)s*(K+ns) + K + j];
#ifdef COLI_VULKAN
if (vkt_ready()) { /* the Vulkan expert tier takes the layer from here (ink_vk_moe) */
ink_vk_moe(m, l, layer, x, S, out, idx, keff, wgt, shw != NULL);
if (!lg_in) free(logits);
free(idx); free(keff); free(wgt); free(use); free(fill); free(fl);
return;
}
#endif
/* Il ciclo sotto ACQUISISCE uno slot per ogni coppia (token, esperto) e ne tiene
* il puntatore fino al calcolo. slot_acquire evince l'LRU quando la cache e'
* piena — anche uno slot gia' consegnato in QUESTA chiamata: il puntatore resta
* valido ma lo slot ora contiene un ALTRO esperto, e il modello calcola con i
* pesi sbagliati. In silenzio: niente crash, solo output incoerente. Acquisire
* tutte le S*K coppie in una volta pretendeva quindi una cache capace di tenere
* tutti gli esperti distinti del batch (18 token x topk 6 = fino a 108 slot per
* layer), e sotto quella soglia il modello sembrava rotto.
* Fix: si lavora a GIRI di al piu' `cap` coppie — acquisisci, riempi, calcola e
* accumula — cosi' nessuno slot puo' essere evinto mentre serve. L'output MoE e'
* una somma pesata, quindi accumulare a giri da' lo stesso risultato di una
* passata sola, e la cache puo' scendere fino a 1 slot per layer (piu' letture
* da disco, ma memoria proporzionale a `cap` invece che al batch). */
int cap = m->cache[layer].cap; if (cap < 1) cap = 1;
float *g = falloc(2*I), *u = g + I, *hh = falloc(D);
int q4 = m->xq && m->rb13*2 == D; /* packed int4 vs int8 container */
int shared_done = 0; /* set when overlapped with the last GPU round */
int64_t npair = (int64_t)S*K;
#ifdef COLI_METAL
/* per-round scratch for the batched GPU submit: pairs grouped by expert,
* activations packed in group order. Allocated once per moe() call. */
float *mxg = NULL, *mrw = NULL; const void **mgp = NULL, **mup = NULL, **mdp = NULL;
const float **mgs = NULL, **mus = NULL, **mds = NULL; Slot **mslot = NULL;
int *mxoff = NULL, *mnr = NULL, *mrows = NULL, *mgi = NULL, *mfp = NULL;
/* GPU for decode too, now that the residency set is on by default: with
* it a decode block runs in ~3ms and measures 1.76 tok/s vs 0.84 on CPU
* (Inkling-Small, M-series 128 GB). Without the set the same block paid
* ~135ms of useResource churn and CPU won — INK_METAL_MIN_S=2 restores
* the prefill-only gate if that regime ever returns. */
int metal_min_s = getenv("INK_METAL_MIN_S") ? atoi(getenv("INK_METAL_MIN_S")) : 1;
if (g_metal && m->xq && S >= metal_min_s) {
mxg = falloc((int64_t)cap*D); mrw = falloc(cap);
mgp = malloc(cap*sizeof(void*)); mup = malloc(cap*sizeof(void*)); mdp = malloc(cap*sizeof(void*));
mgs = malloc(cap*sizeof(float*)); mus = malloc(cap*sizeof(float*)); mds = malloc(cap*sizeof(float*));
mslot = malloc(cap*sizeof(Slot*));
mxoff = malloc((cap+1)*sizeof(int)); mnr = malloc(cap*sizeof(int));
mrows = malloc(cap*sizeof(int)); mgi = malloc(cap*sizeof(int)); mfp = malloc(cap*sizeof(int));
}
/* Shared experts as ONE bf16 (fmt=5) block, submitted BEFORE the routed
* rounds so it rides the GPU while the CPU routes, fills, and packs.
* Every token uses every shared expert, so group j is simply all S rows
* with weight w[K+j]. Falls back to the CPU loop if begin refuses. */
ColiMetalMoeHandle *sh_h = NULL;
float *sxg = NULL, *srw = NULL;
if (mxg && ns > 0 && ns <= 8 && l->sh_g.h && l->sh_u.h && l->sh_d.h &&
!(getenv("INK_METAL_SHARED") && *getenv("INK_METAL_SHARED") == '0')) {
double ts = now_s();
sxg = falloc((int64_t)ns*S*D); srw = falloc((int64_t)ns*S);
const void *sgp[8], *sup[8], *sdp[8];
const float *sscale[8];
int sxoff[9], snr[8];
int *srows = malloc((size_t)ns*S*sizeof(int));
for (int j = 0; j < ns; j++) {
sgp[j] = l->sh_g.h + (int64_t)j*I*D;
sup[j] = l->sh_u.h + (int64_t)j*I*D;
sdp[j] = l->sh_d.h + (int64_t)j*D*I;
sscale[j] = (const float*)sgp[j]; /* fmt=5 never reads scales */
sxoff[j] = j*S; snr[j] = S;
for (int s = 0; s < S; s++) {
memcpy(sxg + ((int64_t)j*S + s)*D, x + (int64_t)s*D, (size_t)D*sizeof(float));
srows[j*S + s] = s;
srw[j*S + s] = wgt[(int64_t)s*(K+ns) + K + j];
}
}
sxoff[ns] = ns*S;
sh_h = coli_metal_moe_block_begin(ns, D, I, 5, 0, sgp, sup, sdp,
sscale, sscale, sscale,
sxg, sxoff, snr, srows, srw);
free(srows);
m->t_shared += now_s() - ts;
}
#endif
for (int64_t base = 0; base < npair; base += cap) {
int64_t end = base + cap < npair ? base + cap : npair;
nfill = 0;
for (int64_t t = base; t < end; t++) { /* acquisizione del giro */
int s = (int)(t / K), kk = (int)(t % K);
if (kk >= keff[s]) { use[t - base] = NULL; continue; } /* scartato da TOPP */
int eid = idx[(int64_t)s*K + kk];
if (m->eusage && m->eusage[layer]) m->eusage[layer][eid]++;
ehit_mark(m, layer, eid);
Slot *e = slot_find(m, layer, eid);
if (e) m->hits++;
else {
m->miss++;
e = slot_acquire(m, layer, eid);
fill[nfill] = e; fl[nfill] = layer; nfill++;
}
use[t - base] = e;
}
if (nfill) { /* riempimento in parallelo */
double tf = now_s();
#pragma omp parallel for schedule(dynamic,1)
for (int j = 0; j < nfill; j++) slot_fill(m, fl[j], fill[j]);
m->t_fill += now_s() - tf;
}
/* Validate before the CPU/Metal split: either backend must refuse a
* cache slot whose weights belong to a different routed expert. */
for (int64_t t = base; t < end; t++) {
Slot *e = use[t - base];
if (!e) continue; /* scartato da TOPP */
int s = (int)(t / K), kk = (int)(t % K);
if (e->eid != idx[(int64_t)s*K + kk]) {
fprintf(stderr, "layer %d: cache served expert %d for requested expert %d\n",
layer, e->eid, idx[(int64_t)s*K + kk]);
exit(1);
}
}
double te = now_s();
#ifdef COLI_METAL
if (mxg) {
/* group this round's (token, expert) pairs by expert and submit ONE
* command buffer; the kernel scatter-adds rw*hh into out with the
* same accumulate semantics as the CPU loop below. 0 -> CPU. */
int nb = 0;
for (int64_t t = base; t < end; t++) {
Slot *e = use[t - base];
if (!e) { mgi[t - base] = -1; continue; }
int gi = -1;
for (int j = 0; j < nb; j++) if (mslot[j] == e) { gi = j; break; }
if (gi < 0) { gi = nb++; mslot[gi] = e; mnr[gi] = 0; }
mgi[t - base] = gi; mnr[gi]++;
}
if (nb) {
mxoff[0] = 0;
for (int j = 0; j < nb; j++) mxoff[j+1] = mxoff[j] + mnr[j];
memcpy(mfp, mxoff, nb*sizeof(int));
for (int64_t t = base; t < end; t++) {
int gi = mgi[t - base];
if (gi < 0) continue;
int s = (int)(t / K), kk = (int)(t % K);
int r = mfp[gi]++;
memcpy(mxg + (int64_t)r*D, x + (int64_t)s*D, (size_t)D*sizeof(float));
mrows[r] = s;
mrw[r] = wgt[(int64_t)s*(K+ns) + kk];
}
for (int j = 0; j < nb; j++) {
Slot *e = mslot[j];
mgp[j] = e->p13; mup[j] = e->p13 + (int64_t)I*m->rb13; mdp[j] = e->p2;
mgs[j] = e->s13; mus[j] = e->s13 + I; mds[j] = e->s2;
}
/* Last round: submit async and run the shared experts on the
* CPU while the GPU computes the routed ones — they are
* independent (both read x, both accumulate into out, and the
* GPU only touches out in _end's scatter-add, after the wait).
* On a GPU fault _end returns 0 and the CPU loop below redoes
* the round; shared_done stays set either way. */
if (base + cap >= npair && !sh_h) {
ColiMetalMoeHandle *h = coli_metal_moe_block_begin(
nb, D, I, q4 ? 2 : 1, 0, mgp, mup, mdp, mgs, mus, mds,
mxg, mxoff, mnr, mrows, mrw);
if (h) {
double ts = now_s();
shared_experts_cpu(m, l, x, S, out, wgt, g, u, hh);
double sh = now_s() - ts;
m->t_shared += sh;
shared_done = 1;
int ok = coli_metal_moe_block_end(h, out);
m->t_expert += (now_s() - te) - sh;
if (ok) continue; /* round done on the GPU */
te = now_s(); /* fault: CPU redo below */
}
}
if (coli_metal_moe_block(nb, D, I, q4 ? 2 : 1, 0, mgp, mup, mdp, mgs, mus, mds,
mxg, mxoff, mnr, mrows, mrw, out, S)) {
m->t_expert += now_s() - te;
continue; /* round done on the GPU */
}
} else { m->t_expert += now_s() - te; continue; }
}
#endif
for (int64_t t = base; t < end; t++) { /* calcolo + accumulo */
int s = (int)(t / K), kk = (int)(t % K);
Slot *e = use[t - base];
if (!e) continue; /* scartato da TOPP */
const float *xs = x + (int64_t)s*D;
float *os = out + (int64_t)s*D;
float *w = wgt + (int64_t)s*(K+ns);
if (m->xq) {
if (q4) {
matmul_q4(g, xs, e->p13, e->s13, D, 2*I); /* gate rows then up rows */
for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];
matmul_q4(hh, g, e->p2, e->s2, I, D);
} else {
matmul_q(g, xs, (int8_t*)e->p13, e->s13, D, 2*I);
for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];
matmul_q(hh, g, (int8_t*)e->p2, e->s2, I, D);
}
} else if (m->quant_bits) {
matmul_q(g, xs, e->q13, e->s13, D, 2*I);
for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];
matmul_q(hh, g, e->q2, e->s2, I, D);
} else {
matmul(g, xs, e->f13, 1, D, 2*I);
for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];
matmul(hh, g, e->f2, 1, I, D);
}
for (int d = 0; d < D; d++) os[d] += w[kk] * hh[d];
}
m->t_expert += now_s() - te;
}
#ifdef COLI_METAL
/* GPU shared block: wait + scatter-add. A fault falls through to CPU. */
if (sh_h) {
double ts = now_s();
if (coli_metal_moe_block_end(sh_h, out)) shared_done = 1;
m->t_shared += now_s() - ts;
}
#endif
/* shared experts: una volta per token, fuori dai giri (non usano la cache).
* Under Metal these run on the GPU as an fmt=5 block overlapped with the
* routed rounds (or on the CPU during the last GPU round); either path
* sets shared_done. */
if (!shared_done && !shw) {
double ts = now_s();
shared_experts_cpu(m, l, x, S, out, wgt, g, u, hh);
m->t_shared += now_s() - ts;
}
if (!lg_in) free(logits);
free(idx); free(keff); free(wgt); free(use); free(fill); free(fl);
free(g); free(hh); /* u aliases g+I */
#ifdef COLI_METAL
if (mxg) {
free(mxg); free(mrw); free(mgp); free(mup); free(mdp);
free(mgs); free(mus); free(mds); free(mslot);
free(mxoff); free(mnr); free(mrows); free(mgi); free(mfp);
}
free(sxg); free(srw);
#endif
}
static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
moe_ex(m, l, layer, x, S, out, NULL, NULL);
}
/* ---------- DMel audio embedding ----------
* One frame = mel_bins u8 levels in [0, mel_vocab). Its decoder embedding is
* sum_b E[b*mel_vocab + v_b], RMSNorm'd with the audio tower norm (eps is a
* literal 1e-6 in the HF audio tower, independent of the text rms_norm_eps).
* This row REPLACES the <|audio|> placeholder's text embedding — embed_norm
* does not apply, matching masked_scatter in modeling_inkling.py. */
static void audio_embed_row(Model *m, const uint8_t *frame, float *out, float *tmp) {
Cfg *c = &m->c; int D = c->hidden;
memset(out, 0, (size_t)D * sizeof(float));
for (int b = 0; b < c->mel_bins; b++) {
int v = frame[b] < c->mel_vocab ? frame[b] : c->mel_vocab - 1;
wt_row_f32(m->audio_enc, (int64_t)(b*c->mel_vocab + v)*D, tmp, D);
for (int d = 0; d < D; d++) out[d] += tmp[d];
}
rmsnorm_row(out, out, m->audio_norm, D, 1e-6f);
}
/* count <|audio|> placeholders in a token sequence (0 if no audio tower) */
static int audio_tok_count(Model *m, const int *ids, int n) {
if (!m->audio_norm) return 0;
int k = 0;
for (int i = 0; i < n; i++) k += (ids[i] == m->c.audio_tok);
return k;
}
static void inkling_layers_forward_range(Model *m, float *x, int S, int pos0,
int layer_begin, int layer_end) {
Cfg *c = &m->c; int D = c->hidden;
float *nrm = falloc((int64_t)S*D), *tmp = falloc((int64_t)S*D);
for (int i = layer_begin; i < layer_end; i++) {
Layer *l = &m->L[i];
for (int s = 0; s < S; s++)
rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D,
l->in_ln, D, c->eps);
double ta = now_s();
attention(m, l, i, nrm, S, pos0, tmp);
m->t_attn += now_s() - ta;
ink_sconv(m, tmp, S, D, l->a_cw, 2, i);
for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j];
for (int s = 0; s < S; s++)
rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D,
l->post_ln, D, c->eps);
if (c->sparse[i]) moe(m, l, i, nrm, S, tmp);
else dense_mlp(m, l, nrm, S, tmp);
ink_sconv(m, tmp, S, D, l->m_cw, 3, i);
for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j];
}
free(nrm); free(tmp);
}
#ifdef COLI_VULKAN
#include "inkling_chain.h" /* COLI_VK_CHAIN: every layer's dense chain on the device */
#endif
/* ---------- one forward pass over S new tokens ----------
* Returns malloc'd logits of the last token (unpadded vocab). If tf_out is
* non-NULL also writes the per-position argmax (teacher-forcing check).
* dmel: u8 [naud, mel_bins] frames consumed left-to-right by the <|audio|>
* placeholder positions in ids (prefill only; decode steps pass NULL). */
/* Canale logprobs: coda numerica per token, lettura del prefill, fotografia
* dello stato. Attenzione: questo motore NON e ad attenzione pura. Oltre alle
* righe K/V, che sono indicizzate per posizione e quindi si riavvolgono da
* sole, porta quattro stati di convoluzione corta per strato (cs[0..3], gli
* ingressi grezzi degli ultimi conv_k-1 passi). Quelli non si riavvolgono: se
* non li si fotografa, l'opzione successiva parte con la coda dell'opzione
* precedente dentro la convoluzione e i logprob sono sbagliati in modo
* silenzioso. Sono piccoli (tre passi per canale), quindi la fotografia costa
* poco. */
static int g_echo_k = 0;
static const char *g_echo_id = NULL;
static ColiPinPool g_pins; /* piu scatti annidati, vedi pin_pool.h */
static const float *g_pin_logit = NULL;
static int g_pin_use_logit = 0;
static Tok *g_echo_tok = NULL;
/* Lo stato che questo motore deve fotografare oltre alle righe K/V: i quattro
* banchi di convoluzione corta per strato. Piccoli (conv_k-1 passi per canale)
* ma indispensabili: senza, l'alternativa successiva parte con la coda della
* precedente dentro la convoluzione. */
typedef struct { float **cs[4]; int n_layers; } InkPinState;
static void ink_echo(const char *id, int pos, int token, const float *lo, int V, int k){
char tail[1024]; coli_logprob_tail(tail, sizeof tail, lo, V, token, k);
char piece[512]; int n = g_echo_tok ? tok_decode(g_echo_tok, &token, 1, piece, (int)sizeof piece) : 0;
if (n < 0) n = 0;
printf("ECHO %s %d %d%s\n", id, n, pos, tail);
if (n > 0) fwrite(piece, 1, (size_t)n, stdout);
fputc('\n', stdout); fflush(stdout);
}
/* larghezza in float di uno stato di convoluzione, per strato e per banco */
static int64_t ink_cs_cells(Cfg *c, int bank, int layer){
int kvdim = L_KV(c,layer) * L_HD(c,layer);
int C = (bank < 2) ? kvdim : c->hidden;
return (int64_t)C * (c->conv_k - 1);
}
static void ink_pin_state_free(void *v){
InkPinState *st = (InkPinState *)v;
if (!st) return;
for (int j = 0; j < 4; j++){
if (!st->cs[j]) continue;
for (int i = 0; i < st->n_layers; i++) free(st->cs[j][i]);
free(st->cs[j]);
}
free(st);
}
static InkPinState *ink_pin_state_save(Model *m, InkPinState *reuse){
Cfg *c = &m->c;
InkPinState *st = reuse;
if (st && st->n_layers != c->n_layers) { ink_pin_state_free(st); st = NULL; }
if (!st){
st = (InkPinState *)calloc(1, sizeof(*st));
if (!st) return NULL;
st->n_layers = c->n_layers;
for (int j = 0; j < 4; j++){
st->cs[j] = (float**)calloc((size_t)c->n_layers, sizeof(float*));
if (!st->cs[j]){ ink_pin_state_free(st); return NULL; }
for (int i = 0; i < c->n_layers; i++){
st->cs[j][i] = (float*)malloc((size_t)ink_cs_cells(c,j,i) * sizeof(float));
if (!st->cs[j][i]){ ink_pin_state_free(st); return NULL; }
}
}
}
for (int j = 0; j < 4; j++)
for (int i = 0; i < c->n_layers; i++)
memcpy(st->cs[j][i], m->cs[j][i],
(size_t)ink_cs_cells(c,j,i) * sizeof(float));
return st;
}
static void ink_pin_state_restore(Model *m, const InkPinState *st){
Cfg *c = &m->c;
if (!st) return;
for (int j = 0; j < 4; j++)
for (int i = 0; i < c->n_layers; i++)
memcpy(m->cs[j][i], st->cs[j][i],
(size_t)ink_cs_cells(c,j,i) * sizeof(float));
#ifdef COLI_VULKAN
inkc_host_wrote(m, 0); /* the dense chain's copy goes up again before its next step */
#endif
}
static float *step_mm(Model *m, const int *ids, int S, int pos0, int *tf_out,
const uint8_t *dmel, int naud) {
Cfg *c = &m->c; int D = c->hidden;
float *x = falloc((int64_t)S*D);
float *arow = (dmel && naud > 0) ? falloc(D) : NULL;
int aidx = 0;
for (int s = 0; s < S; s++) {
if (arow && m->audio_norm && ids[s] == c->audio_tok && aidx < naud) {
audio_embed_row(m, dmel + (int64_t)aidx*c->mel_bins, x + (int64_t)s*D, arow);
aidx++;
continue;
}
wt_row_f32(m->embed, (int64_t)ids[s]*D, x + (int64_t)s*D, D);
if (m->embed_norm) rmsnorm_row(x + (int64_t)s*D, x + (int64_t)s*D, m->embed_norm, D, c->eps);
}
free(arow);
#ifdef COLI_VULKAN
/* COLI_VK_CHAIN: the layers, the final norm and lm_head on the device; x comes
* back only when the per-position heads below read every row */
float *chain_logit = NULL;
int layer0 = 0; /* the first layer the CPU runs */
if (g_vk_chain) {
chain_logit = falloc(c->unpad_vocab);
int took = inkc_forward(m, x, S, pos0, (g_echo_k > 0 && g_echo_id && S > 1) || tf_out != NULL, chain_logit);
if (took != 1) { free(chain_logit); chain_logit = NULL; }
if (!took) inkc_cpu_step(m, pos0, S);
else if (took == 2) layer0 = inkc_ran(); /* a partial chain: x is the residual after the last layer it ran */
}
if (!chain_logit)
inkling_layers_forward_range(m, x, S, pos0, layer0, c->n_layers);
#else
inkling_layers_forward_range(m, x, S, pos0, 0, c->n_layers);
#endif
m->kv_len = pos0 + S;
/* record what was just fed, at the positions it went to (kv_prefix.h).
* Audio taints the record: every frame carries the same token id while the
* mel payload differs, so ids alone cannot tell two clips apart. */
kv_prefix_record(&m->kvp, ids, pos0, S);
if (dmel && naud > 0) kv_prefix_taint(&m->kvp);
float *last = falloc(D);
float *logit = falloc(c->unpad_vocab);
/* Lettura del prefill: un passaggio di lm_head per posizione, pagato solo
* da chi ha chiesto il canale. La posizione p predice il token p+1; il
* primo token fresco e predetto dalla fotografia. */
/* The per-position heads (the logprobs channel, teacher forcing) run over a
* block of positions at a time: one S-row lm_head call (a device GEMM) instead
* of one per position. Every kernel matmul_w uses computes a row as the one-row
* call does, so the logits are the same bits. */
int hb = (int)((64LL << 20) / ((int64_t)c->unpad_vocab * (int64_t)sizeof(float)));
hb = hb < 1 ? 1 : hb > 64 ? 64 : hb;
if ((g_echo_k > 0 && g_echo_id && S > 0) || tf_out) {
float *normed = falloc((int64_t)hb * D), *logits = falloc((int64_t)hb * c->unpad_vocab);
if (g_echo_k > 0 && g_echo_id && S > 0) {
if (g_pin_use_logit && g_pin_logit)
ink_echo(g_echo_id, pos0, ids[0], g_pin_logit, c->unpad_vocab, g_echo_k);
for (int base = 0; base + 1 < S; base += hb) {
int rows = S - 1 - base < hb ? S - 1 - base : hb;
for (int r = 0; r < rows; r++) {
float *nr = normed + (int64_t)r*D;
rmsnorm_row(nr, x + (int64_t)(base + r)*D, m->final_norm, D, c->eps);
for (int d = 0; d < D; d++) nr[d] /= c->mup;
}
matmul_w(logits, normed, m->lm_head, rows, D, c->unpad_vocab);
for (int r = 0; r < rows; r++) {
int p = base + r;
ink_echo(g_echo_id, pos0 + p + 1, ids[p+1], logits + (int64_t)r * c->unpad_vocab,
c->unpad_vocab, g_echo_k);
}
}
}
if (tf_out) {
for (int base = 0; base < S; base += hb) {
int rows = S - base < hb ? S - base : hb;
for (int r = 0; r < rows; r++) {
float *nr = normed + (int64_t)r*D;
rmsnorm_row(nr, x + (int64_t)(base + r)*D, m->final_norm, D, c->eps);
for (int d = 0; d < D; d++) nr[d] /= c->mup;
}
matmul_w(logits, normed, m->lm_head, rows, D, c->unpad_vocab);
for (int r = 0; r < rows; r++) {
const float *lr = logits + (int64_t)r * c->unpad_vocab;
int best = 0; for (int i = 1; i < c->unpad_vocab; i++) if (lr[i] > lr[best]) best = i;
tf_out[pos0 + base + r] = best;
}
}
}
free(normed); free(logits);
}
#ifdef COLI_VULKAN
if (chain_logit) { free(x); free(last); return chain_logit; } /* the chain's last frame ran the head */
#endif
rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps);
for (int d = 0; d < D; d++) last[d] /= c->mup;
matmul_w(logit, last, m->lm_head, 1, D, c->unpad_vocab);
free(x); free(last);
return logit;
}
static float *step(Model *m, const int *ids, int S, int pos0, int *tf_out) {
return step_mm(m, ids, S, pos0, tf_out, NULL, 0);
}
/* ---- several conversations at once (KV_SLOTS>1, serve_mux) ------------------- */
static void ink_seq_swap(Model *m, InkSeq *q) {
float **K = m->K, **V = m->V; int len = m->kv_len; kv_prefix p = m->kvp;
m->K = q->K; m->V = q->V; m->kv_len = q->kv_len; m->kvp = q->kvp;
q->K = K; q->V = V; q->kv_len = len; q->kvp = p;
for (int j = 0; j < 4; j++) { float **cs = m->cs[j]; m->cs[j] = q->cs[j]; q->cs[j] = cs; }
}
/* A conversation's state of its own, the shape the Model's has at its max_t. */
static void ink_seq_alloc(Model *m, InkSeq *q) {
Cfg *c = &m->c;
memset(q, 0, sizeof *q);
q->K = calloc(c->n_layers, sizeof(float*)); q->V = calloc(c->n_layers, sizeof(float*));
for (int j = 0; j < 4; j++) q->cs[j] = calloc(c->n_layers, sizeof(float*));
for (int i = 0; i < c->n_layers; i++) {
int kv = L_KV(c,i), hd = L_HD(c,i);
int64_t rows = kv_ring_rows(c, i, m->max_t);
q->K[i] = falloc((int64_t)kv * rows * hd);
q->V[i] = falloc((int64_t)kv * rows * hd);
for (int j = 0; j < 4; j++)
if (m->cs[j][i]) q->cs[j][i] = calloc((size_t)ink_cs_cells(c, j, i), sizeof(float));
}
kv_prefix_alloc(&q->kvp, m->max_t);
}
/* One decode step of several conversations: row s is the token ids[s] at
* rows[s].pos of the conversation rows[s].seq, every conversation parked. The
* matrices, the routed experts and lm_head run once over the S rows; the attention
* and the short convolutions read and write each row's own state (m->mux_rows). The
* CPU kernels give a row the same bits whatever S is, so each conversation gets the
* logits it would alone. S rows of logits. */
static float *ink_step_rows(Model *m, const InkRow *rows, const int *ids, int S) {
Cfg *c = &m->c; int D = c->hidden;
float *x = falloc((int64_t)S*D);
for (int s = 0; s < S; s++) {
wt_row_f32(m->embed, (int64_t)ids[s]*D, x + (int64_t)s*D, D);
if (m->embed_norm) rmsnorm_row(x + (int64_t)s*D, x + (int64_t)s*D, m->embed_norm, D, c->eps);
}
m->mux_rows = rows;
inkling_layers_forward_range(m, x, S, 0, 0, c->n_layers);
m->mux_rows = NULL;
for (int s = 0; s < S; s++) {
rows[s].seq->kv_len = rows[s].pos + 1;
kv_prefix_record(&rows[s].seq->kvp, ids + s, rows[s].pos, 1);
}
float *normed = falloc((int64_t)S*D), *logit = falloc((int64_t)S * c->unpad_vocab);
for (int s = 0; s < S; s++) {
float *nr = normed + (int64_t)s*D;
rmsnorm_row(nr, x + (int64_t)s*D, m->final_norm, D, c->eps);
for (int d = 0; d < D; d++) nr[d] /= c->mup;
}
matmul_w(logit, normed, m->lm_head, S, D, c->unpad_vocab);
free(x); free(normed);
return logit;
}
static void state_reset(Model *m) {
Cfg *c = &m->c;
m->kv_len = 0;
kv_prefix_clear(&m->kvp);
for (int i = 0; i < c->n_layers; i++) {
int kvdim = L_KV(c,i) * L_HD(c,i);
for (int j = 0; j < 4; j++)
memset(m->cs[j][i], 0, (int64_t)((j < 2) ? kvdim : c->hidden) * (c->conv_k-1) * sizeof(float));
}
#ifdef COLI_VULKAN
inkc_host_wrote(m, 1); /* zeros: the dense chain fills its copy with zeros */
#endif
}
static void kv_alloc(Model *m, int max_t) {
Cfg *c = &m->c;
if (m->K && max_t <= m->max_t) return; /* reuse across prompts when big enough */
/* GROW, DO NOT RESTART.
*
* This used to free the buffers and allocate fresh ones, which discarded
* every position already computed. That was invisible while each turn
* re-prefilled anyway — but it defeats KV prefix reuse in exactly the case
* reuse exists for: a conversation whose prompt is longer every turn asks
* for a larger max_t every turn, so the state was thrown away just before
* the point of using it.
*
* K/V are laid out [kv_head][rows][hd] with rows = kv_ring_rows(), so a
* larger max_t changes the stride wherever rows follows max_t: those
* contents cannot be realloc'd, they have to be re-laid-out head by head.
* That copy costs a memcpy of what is already computed, which is nothing
* beside re-running the prefill that produced it.
*
* Sliding layers whose ring is already window rows are the exception in
* BOTH directions: their size does not depend on max_t (nothing to grow),
* and a ring that has wrapped is not linear (position t lives at row
* t % window), so the linear copy below would silently rotate it. Steal
* the buffer instead — the slot map is unchanged, so it stays valid.
* Every layer that does reach the copy IS linear: rows != old_rows only
* happens when old_rows == old_max (a wrapped ring keeps rows == window
* forever), and keep <= old_max <= rows there, so the copy fits. */
float **oldK = m->K, **oldV = m->V;
int old_max = m->max_t;
int keep = (m->K && m->kv_len > 0 && m->kv_len <= max_t) ? m->kv_len : 0;
m->max_t = max_t;
m->K = calloc(c->n_layers, sizeof(float*)); m->V = calloc(c->n_layers, sizeof(float*));
int64_t ring = 0, full = 0;
for (int i = 0; i < c->n_layers; i++) {
int kv = L_KV(c,i), hd = L_HD(c,i);
int64_t rows = kv_ring_rows(c, i, max_t);
int64_t old_rows = oldK ? kv_ring_rows(c, i, old_max) : 0;
if (oldK && rows == old_rows) {
m->K[i] = oldK[i]; m->V[i] = oldV[i];
oldK[i] = NULL; oldV[i] = NULL;
} else {
m->K[i] = falloc((int64_t)kv * rows * hd);
m->V[i] = falloc((int64_t)kv * rows * hd);
for (int h = 0; h < kv && keep; h++) {
memcpy(m->K[i] + (int64_t)h * rows * hd,
oldK[i] + (int64_t)h * old_max * hd, (size_t)keep * hd * sizeof(float));
memcpy(m->V[i] + (int64_t)h * rows * hd,
oldV[i] + (int64_t)h * old_max * hd, (size_t)keep * hd * sizeof(float));
}
}
ring += 2 * (int64_t)kv * rows * hd * (int64_t)sizeof(float);
full += 2 * (int64_t)kv * max_t * hd * (int64_t)sizeof(float);
}
if (oldK) for (int i = 0; i < c->n_layers; i++) { free(oldK[i]); free(oldV[i]); }
free(oldK); free(oldV);
if (ring < full)
fprintf(stderr, "[kv] %.1f MiB (ring buffers on sliding layers; full cache would be %.1f MiB)\n",
ring/1048576.0, full/1048576.0);
/* the record describes those same positions, so it survives with them --
* unless its own allocation fails, in which case reuse simply stops. */
if (kv_prefix_grow(&m->kvp, max_t, keep)) m->kv_len = keep;
else m->kv_len = 0;
}
/* greedy generation, olmoe.c-style */
static void generate(Model *m, const int *prompt, int np, int n_new, int *out,
const uint8_t *dmel, int naud) {
for (int i = 0; i < np; i++) out[i] = prompt[i];
/* DUMP=<path>: every forward's logits (unpadded vocab raw float32 each, in order),
* for a comparison of two runs (tests/vulkan_engines.sh compares the dense chain's) */
const char *dp = getenv("DUMP");
FILE *df = dp && *dp ? fopen(dp, "wb") : NULL;
float *logit = step_mm(m, prompt, np, 0, NULL, dmel, naud);
int len = np;
Cfg *c = &m->c;
for (int s = 0; s < n_new; s++) {
if (df) fwrite(logit, sizeof(float), (size_t)c->unpad_vocab, df);
int best = 0; float bv = logit[0];
for (int i = 1; i < c->unpad_vocab; i++) if (logit[i] > bv) { bv = logit[i]; best = i; }
free(logit);
out[len++] = best;
if (s == n_new - 1) break;
int one = best;
logit = step(m, &one, 1, len - 1, NULL);
}
if (df) fclose(df);
}
/* ---------- interactive prompt mode: greedy, streaming, stop on eos ---------- */
static void generate_stream(Model *m, Tok *T, const char *prompt, int n_new,
const uint8_t *dmel, int naud) {
Cfg *c = &m->c;
int cap = (int)strlen(prompt) + 16;
int *ids = malloc(cap * sizeof(int));
int np = tok_encode(T, prompt, (int)strlen(prompt), ids, cap);
if (np <= 0) { fprintf(stderr, "empty prompt after tokenization\n"); return; }
if (audio_tok_count(m, ids, np) != naud) {
fprintf(stderr, "audio frames (%d) do not match <|audio|> placeholders (%d)%s\n",
naud, audio_tok_count(m, ids, np),
m->audio_norm ? "" : " — snapshot has no audio tensors (audio.safetensors)");
free(ids); return;
}
kv_alloc(m, np + n_new + 8);
printf("[%d prompt tokens%s] %s", np, naud ? " incl. audio" : "", naud ? "" : prompt);
fflush(stdout);
double t0 = now_s(), t1 = 0;
float *logit = step_mm(m, ids, np, 0, NULL, dmel, naud);
int len = np;
char buf[512];
for (int s = 0; s < n_new; s++) {
int best = 0; float bv = logit[0];
for (int i = 1; i < c->unpad_vocab; i++) if (logit[i] > bv) { bv = logit[i]; best = i; }
free(logit);
if (s == 0) t1 = now_s();
if (best == c->eos) { printf("\n[eos after %d tokens]", s); break; }
int nb = tok_decode(T, &best, 1, buf, sizeof(buf)-1);
buf[nb] = 0; fputs(buf, stdout); fflush(stdout);
int one = best;
len++;
if (s == n_new - 1) break;
logit = step(m, &one, 1, len - 1, NULL);
}
double dt = now_s() - t1;
int gen = len - np;
printf("\n[prefill %.1fs | %d tokens in %.1fs = %.2f tok/s | RSS %.1f GB]\n",
t1 - t0, gen, dt, gen > 1 ? (gen-1)/dt : 0.0, rss_gb());
/* One line, every engine, one format: `coli tune` sweeps scheduling knobs and
* needs tokens-and-elapsed to compare candidates. Before this only colibri
* emitted a parseable throughput line (REPLAY decode), so the tuner was
* GLM-only and bannered the right model while launching the wrong engine
* (#898). Printed to stdout, which is what autotune captures.
* Tokens and seconds, not tok/s: the ratio is derived by the caller at full
* precision (#852 -- two decimals of tok/s is one significant digit at the
* rates this engine runs at). */
printf("TUNE decode: %d tokens in %.3fs\n", gen > 1 ? gen - 1 : gen, dt);
double wall = now_s() - t0;
#ifdef COLI_METAL
if (g_metal) {
uint64_t mok = 0, mfb = 0, mex = 0;
coli_metal_moe_counts(&mok, &mfb, &mex);
printf("[metal] %llu MoE blocks on GPU, %llu CPU fallbacks, %llu experts\n",
(unsigned long long)mok, (unsigned long long)mfb, (unsigned long long)mex);
}
#endif
printf("[phases] fill %.1fs | expert-mm %.1fs | shared %.1fs | attn %.1fs | other %.1fs\n",
m->t_fill, m->t_expert, m->t_shared, m->t_attn,
wall - m->t_fill - m->t_expert - m->t_shared - m->t_attn);
#ifdef COLI_VULKAN
fflush(stdout); ink_vk_report(); ink_vk_tier_report(m, "run"); inkc_report(m);
#endif
free(ids);
}
/* ---------- serve mode: openai_server.py engine protocol ----------
* stdin: SUBMIT <id> <slot> <len> <max_tokens> <temp> <top_p>\n<payload>\n
* CANCEL <id>\n
* stdout: READY sentinel once loaded, then per request a stream of
* DATA <id> <size>\n<bytes>\n frames and a final
* DONE <id> STAT <tok> <tps> <hit%> <rss> <prompt_tok> <len_limited>\n
* Byte-identical to colibri.c's serve protocol so the shared openai_server.py
* gateway drives inkling unchanged (v1: one request at a time; the KV slot arg
* is accepted but every request re-prefills). */
static uint64_t g_rng = 0x9E3779B97F4A7C15ull;
static double rng_next(void) {
g_rng ^= g_rng << 13; g_rng ^= g_rng >> 7; g_rng ^= g_rng << 17;
return (double)(g_rng >> 11) / 9007199254740992.0;
}
/* temperature + top-p nucleus sampling; temp<=0 = greedy (the oracle path) */
typedef struct { float p; int i; } PI;
static int pi_desc(const void *a, const void *b) {
float d = ((const PI*)b)->p - ((const PI*)a)->p;
return d > 0 ? 1 : d < 0 ? -1 : 0;
}
static int sample_logits(const float *logit, int n, float temp, float top_p) {
int best = 0;
for (int i = 1; i < n; i++) if (logit[i] > logit[best]) best = i;
if (temp <= 0.f) return best;
PI *c = malloc((size_t)n * sizeof(PI));
double sum = 0;
for (int i = 0; i < n; i++) {
c[i].p = expf((logit[i] - logit[best]) / temp);
c[i].i = i; sum += c[i].p;
}
qsort(c, n, sizeof(PI), pi_desc);
double cut = (top_p > 0.f && top_p < 1.f) ? top_p * sum : sum;
double acc = 0; int k = 0;
while (k < n && acc < cut) acc += c[k++].p;
double r = rng_next() * acc, run = 0;
int pick = c[0].i;
for (int i = 0; i < k; i++) { run += c[i].p; if (run >= r) { pick = c[i].i; break; } }
free(c);
return pick;
}
/* light repeat guard: recently emitted tokens get their logit divided by pen>1 */
static void apply_rep_penalty(float *logit, int n, const int *hist, int nhist, float pen) {
if (pen <= 1.f) return;
for (int i = 0; i < nhist; i++) {
int t = hist[i];
if (t < 0 || t >= n) continue;
logit[t] = logit[t] > 0 ? logit[t] / pen : logit[t] * pen;
}
}
/* Refuse only a prompt that does not fit the served KV bound (CTX_MAX,
* default 8192). max_tokens is a ceiling: coli chat's interactive default
* (16384) used to 400 every turn because 2 + 16384 > 8192. The refusal is
* the frame the gateway turns into a 400 context_length_exceeded (#506,
* #1381); free text here reached the client as a 500. One request is served
* at a time, so the returned buffer is only read before the next call. */
static int ink_ctx_max(void) {
const char *cm = getenv("CTX_MAX");
return cm ? atoi(cm) : 8192;
}
static const char *prompt_reject(int np, int want) {
static char message[96];
int ctx_max = ink_ctx_max();
if (coli_serve_budget(np, want, ctx_max, 0) >= 0) return NULL;
snprintf(message, sizeof(message),
"CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
np, want, ctx_max);
return message;
}
typedef struct { char id[64]; int max_tok; float temp, top_p; char *payload; int plen;
uint8_t *audio; int alen;
int logprobs, pin; } SReq; /* raw DMel bytes after the payload; SUBMIT logprobs=k / pin=1 */
#define SRV_QMAX 16
static SReq g_q[SRV_QMAX]; static int g_qn = 0;
static const ColiServeWireProfile inkling_wire = {
.max_header_bytes = 511,
.max_payload_bytes = 1u << 22,
.max_extension_bytes = 1u << 26,
.max_tokens = 1 << 20,
.require_exact_lf = 1,
.require_finite_sampling = 0,
.allow_extension_bytes = 1,
.allow_prefix_hint = 0,
};
static void serve_request_dispose(SReq *request) {
if (!request) return;
free(request->payload);
memset(request, 0, sizeof(*request));
}
static int stdin_readable(void) {
/* Windows non ha fd_set/select in questa forma: la build falliva del tutto.
* La versione portabile (con i fix #139/#195) vive in compat.h, incluso via st.h. */
return coli_stdin_readable();
}
/* read one control line (+ payload for SUBMIT). cur_id: request in flight;
* returns 1 if that request was cancelled, 0 otherwise, -1 on stdin EOF. */
static int serve_read_cmd(FILE *input, FILE *output, const char *cur_id) {
ColiServeCommand command;
ColiServeReadResult result=coli_serve_read_command(input,&inkling_wire,&command);
if(result==COLI_SERVE_READ_EOF) return -1;
if(result==COLI_SERVE_READ_BAD_FRAME) return -2;
if(result==COLI_SERVE_READ_NOMEM){
coli_serve_write_error(output,command.id,"out of memory"); return -2;
}
if(result==COLI_SERVE_READ_BAD_REQUEST&&
command.kind==COLI_SERVE_COMMAND_SUBMIT){
/* 6th field (optional, backward compatible): DMel byte count appended
* verbatim after the text payload — frames x mel_bins u8 levels */
/* SEC: max_tok was the one field nobody validated, and it is the one
* the context check is built on. prompt_reject() asks
* `np + want > ctx_max`; a negative `want` makes that sum smaller than
* np, so the check passes for any prompt length. kv_alloc() is then
* sized on the same np + max_tok + 8 and comes out shorter than the
* prompt, and prefill writes past the end of the K/V cache -- a heap
* overflow whose length and contents both follow from the request.
*
* The official gateway forces a positive integer, so this is not
* reachable through openai_server.py. The SERVE protocol is public and
* anything bridging it (socat, a custom gateway, a sidecar) exposes it
* directly, so the check belongs here, next to the one plen already
* has, rather than in one of its callers. */
coli_serve_write_error(output,command.id,"bad submit header"); return -2;
}
if(result!=COLI_SERVE_READ_OK) return 0;
if(command.kind==COLI_SERVE_COMMAND_CANCEL){
int matched=cur_id&&!strcmp(command.id,cur_id);
coli_serve_command_dispose(&command); return matched;
}
if(command.kind==COLI_SERVE_COMMAND_STOP){
coli_serve_command_dispose(&command); return 0;
}
if(command.kind==COLI_SERVE_COMMAND_SUBMIT){
if (g_qn < SRV_QMAX) {
SReq *q = &g_q[g_qn++];
snprintf(q->id, sizeof(q->id), "%s", command.id);
q->max_tok=command.max_tokens; q->temp=command.temperature;
q->logprobs=command.logprobs; q->pin=command.pin;
q->top_p=command.top_p; q->plen=(int)command.payload_bytes;
q->alen=(int)command.extension_bytes;
q->audio=coli_serve_command_extension(&command);
q->payload=(char*)coli_serve_command_take_payload(&command);
} else coli_serve_write_error(output,command.id,"queue full");
}
coli_serve_command_dispose(&command);
return 0;
}
/* HITS rows cols hex: which experts this turn routed, one bit each over the
* sparse layers (same rows and columns as EMAP), packed 8 per hex pair. Same
* line colibri.c emits; the Brain tab lights up from it. A turn that routed
* nothing still reports a bitmap, all zero, so the tab shows THIS turn. */
static void serve_hits(Model *m) {
Cfg *c = &m->c; int E = c->n_experts;
if (!m->ehit) ehit_mark(m, -1, -1);
int nsp = 0;
for (int i = 0; i < c->n_layers; i++) if (c->sparse[i]) nsp++;
int nb = (nsp * E + 7) / 8;
uint8_t *bm = calloc((size_t)nb, 1); int bit = 0;
for (int i = 0; i < c->n_layers; i++) {
if (!c->sparse[i]) continue;
for (int e = 0; e < E; e++, bit++)
if (m->ehit[i][e]) { bm[bit >> 3] |= (uint8_t)(1 << (bit & 7)); m->ehit[i][e] = 0; }
}
char *hex = malloc((size_t)nb * 2 + 1); int w = 0;
for (int b = 0; b < nb; b++) { hex[w++] = "0123456789abcdef"[bm[b] >> 4]; hex[w++] = "0123456789abcdef"[bm[b] & 15]; }
hex[w] = 0;
printf("HITS %d %d %s\n", nsp, E, hex);
fflush(stdout); free(hex); free(bm);
}
/* When a request's prefill began, and the counters then: DONE and PROF report its share. */
typedef struct { double t0, f0, e0, s0, a0; uint64_t h0, m0; } InkReqClock;
/* A request's prompt into the state the Model holds: its tokens and audio, the
* budget, the prefix reuse and the photos, the prefill and its read-out. 1 with the
* prompt's ids and the logits after it; 0 when the request ended here, its ERROR
* written. serve_one and serve_mux start every request here. */
static int ink_serve_start(Model *m, Tok *T, SReq *q, int **ids_out, int *np_out, float **logit_out,
InkReqClock *clk) {
Cfg *c = &m->c;
int cap = q->plen + 16;
int *ids = malloc((size_t)cap * sizeof(int));
int np = tok_encode(T, q->payload, q->plen, ids, cap);
if (np <= 0) { coli_serve_write_error(stdout,q->id,"empty prompt"); free(ids); return 0; }
int ctx_max = ink_ctx_max();
int budget = coli_serve_budget(np, q->max_tok, ctx_max, q->logprobs > 0);
if (budget < 0) {
const char *bad = prompt_reject(np, q->max_tok);
coli_serve_write_error(stdout,q->id,bad ? bad : "CONTEXT_EXCEEDED");
free(ids); return 0;
}
if (budget < q->max_tok) {
fprintf(stderr, "[serve] max_tokens %d clamped to %d (context %d - prompt %d); "
"raise CTX_MAX for longer answers\n",
q->max_tok, budget, ctx_max, np);
q->max_tok = budget;
}
/* audio: every <|audio|> placeholder must have exactly one DMel frame */
int naud = q->alen / m->c.mel_bins;
if (q->alen % m->c.mel_bins != 0 || audio_tok_count(m, ids, np) != naud) {
char message[256];
snprintf(message,sizeof(message),
"audio frames (%d) do not match <|audio|> placeholders (%d)%s",
naud,audio_tok_count(m,ids,np),
m->audio_norm?"":" — snapshot has no audio tensors");
coli_serve_write_error(stdout,q->id,message); free(ids); return 0;
}
/* KV PREFIX REUSE (#639 for GLM; this engine re-prefilled every turn).
* A chat client resends the whole transcript each turn, so turn N used to
* re-process turns 1..N-1 from scratch — the cost of a message grew with
* the conversation, and every replayed position pulled its experts off
* disk again. When this prompt begins with the sequence the state already
* holds, that state IS the state at that position: keep it and prefill
* only the tail.
*
* Requirements, all necessary:
* - kv_alloc must not have grown (it frees the K/V fed[] describes), so
* the reuse decision is taken AFTER it
* - at least one new token, since the state cannot be rewound
* - no audio on either side: every audio frame carries the same token id,
* so ids alone cannot tell two different clips apart
* Either the reused positions are token-identical or nothing is reused;
* the emitted tokens are unchanged in both cases. */
kv_alloc(m, np + q->max_tok + 8);
/* naud>0 taints this turn before the comparison, not after: a request that
* brings its own audio must not match a text-only state either. */
if (naud > 0) kv_prefix_taint(&m->kvp);
int reuse = kv_prefix_reuse(&m->kvp, ids, np);
/* La fotografia si prova sempre, non solo quando il riuso vivo fallisce:
* altrimenti la prima opzione trova ancora lo stato del prompt, passa dal
* riuso normale e il suo primo token resta senza predittore. */
g_pin_use_logit = 0; g_pin_logit = NULL;
if (naud == 0) {
/* Il piu profondo degli scatti valido per questo prompt. Uno scatto le
* cui righe K/V non ci sono piu si butta e si riprova col precedente,
* invece di rinunciare e rifare tutto da zero. */
int s = coli_pin_best(&g_pins, ids, np);
while (s >= 0) {
ColiPin *k = &g_pins.slot[s];
if (kv_prefix_holds(&m->kvp, k->ids, k->len)) {
ink_pin_state_restore(m, (const InkPinState *)k->state);
kv_prefix_clear(&m->kvp);
kv_prefix_record(&m->kvp, k->ids, 0, k->len);
m->kv_len = k->len;
reuse = k->len;
g_pin_logit = k->logit; g_pin_use_logit = k->logit != NULL;
coli_pin_touch(&g_pins, s);
break;
}
k->len = 0;
s = coli_pin_best(&g_pins, ids, np);
}
}
g_echo_k = q->logprobs; g_echo_id = q->id; g_echo_tok = T;
if (getenv("INK_PREFIX_LOG")) {
/* Report the decision either way, with the reason when it is no. "It
* did not get faster" is otherwise indistinguishable from "reuse is not
* wired up", both for a user and for the CI gate. */
if (reuse)
fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",
reuse, np, 100.0 * reuse / np);
else
fprintf(stderr, "[PREFIX] no reuse: held=%d cap=%d prompt=%d%s%s\n",
m->kvp.len, m->kvp.cap, np,
m->kvp.tainted ? " tainted" : "",
(m->kvp.len > 0 && m->kvp.len < np) ? " (diverged)" : "");
fflush(stderr);
}
if (!reuse) state_reset(m);
clk->t0 = now_s();
clk->h0 = m->hits; clk->m0 = m->miss;
/* per-turn phase snapshot for the PROF line (timers accumulate globally) */
clk->f0 = m->t_fill; clk->e0 = m->t_expert; clk->s0 = m->t_shared; clk->a0 = m->t_attn;
/* `reuse` is the ABSOLUTE position of the first fresh token: attention and
* the KV slots are position-indexed, so this has to be the real offset. */
float *logit = step_mm(m, ids + reuse, np - reuse, reuse, NULL, q->audio, naud);
if (q->pin && logit && naud == 0) {
coli_pin_pool_init(&g_pins, c->unpad_vocab);
ColiPin *k = coli_pin_store(&g_pins, ids, np, logit);
if (k) {
InkPinState *st = ink_pin_state_save(m, (InkPinState *)k->state);
if (st) { k->state = st; fprintf(stderr, "[PIN] scatto a %d token\n", np); }
else { k->len = 0; } /* senza stato lo scatto e' una bugia */
fflush(stderr);
}
}
g_echo_k = 0; g_echo_id = NULL; /* la lettura riguarda il prefill, non la decodifica */
*ids_out = ids; *np_out = np; *logit_out = logit;
return 1;
}
static int serve_one(Model *m, Tok *T, SReq *q) {
Cfg *c = &m->c;
int *ids = NULL, np = 0; float *logit = NULL; InkReqClock clk;
if (!ink_serve_start(m, T, q, &ids, &np, &logit, &clk)) return 0;
double t0 = clk.t0;
uint64_t h0 = clk.h0, m0 = clk.m0;
double f0 = clk.f0, e0 = clk.e0, s0 = clk.s0, a0 = clk.a0;
int forwards = 1; /* il prefill e' il primo forward */
int len = np, gen = 0, limited = 1, cancelled = 0;
char buf[512];
/* repetition-penalty history: prompt tail + emitted tokens, ring of 128 */
float rep = getenv("REP_PEN") ? atof(getenv("REP_PEN")) : 1.1f;
int hist[128], nhist = 0;
for (int i = (np > 128 ? np - 128 : 0); i < np; i++) hist[nhist++] = ids[i];
for (int s = 0; s < q->max_tok && !cancelled; s++) {
apply_rep_penalty(logit, c->unpad_vocab, hist, nhist, rep);
int tk = sample_logits(logit, c->unpad_vocab, q->temp, q->top_p);
char lptail[1024]; lptail[0] = 0;
if (q->logprobs > 0) coli_logprob_tail(lptail, sizeof lptail, logit, c->unpad_vocab, tk, q->logprobs);
free(logit); logit = NULL;
if (tk == c->eos) { limited = 0; break; }
if (nhist < 128) hist[nhist++] = tk;
else { memmove(hist, hist+1, 127*sizeof(int)); hist[127] = tk; }
int nb = tok_decode(T, &tk, 1, buf, sizeof(buf)-1);
if (q->logprobs > 0) coli_serve_write_data_lp(stdout,q->id,buf,(size_t)nb,lptail);
else coli_serve_write_data(stdout,q->id,buf,(size_t)nb);
gen++; len++;
while (stdin_readable()) {
int r = serve_read_cmd(stdin, stdout, q->id);
if (r < 0) { free(ids); return -1; }
if (r > 0) { cancelled = 1; limited = 0; }
}
if (cancelled || s == q->max_tok - 1) break;
logit = step(m, &tk, 1, len - 1, NULL); forwards++;
}
free(logit);
double dt = now_s() - t0;
double tot = (double)(m->hits - h0 + m->miss - m0);
ColiServeDone done={gen,dt>0?gen/dt:0.0,
tot?100.0*(m->hits-h0)/tot:0.0,rss_gb(),np,limited};
char done_line[256];
int done_bytes=coli_serve_format_done(done_line,sizeof(done_line),q->id,&done);
if(done_bytes>0) fwrite(done_line,1,(size_t)done_bytes,stdout);
/* PROF: per-turn phase timings for the dashboard (gateway schema — we map
* expert_wait -> shared-expert compute, lm_head folded into 0). */
printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n", dt, np, gen,
m->t_fill - f0, m->t_shared - s0, m->t_expert - e0, m->t_attn - a0, 0.0, forwards);
fflush(stdout);
#ifdef COLI_VULKAN
ink_vk_report(); ink_vk_tier_report(m, "turn"); inkc_report(m);
#endif
serve_hits(m);
free(ids);
return 0;
}
/* ---------- dashboard protocol (HWINFO / TIERS / EMAP) ----------
* Same stdout lines colibri.c emits for the web dashboard; the gateway parses
* them and the Brain/Profiling pages render live expert-tier state. */
static void serve_hwinfo(Model *m) {
char cpu[256] = ""; int cores = 0; double rt = 0, ra = 0;
FILE *ci = fopen("/proc/cpuinfo", "r");
if (ci) { char ln[256];
while (fgets(ln, sizeof(ln), ci)) if (!strncmp(ln, "model name", 10)) {
char *p = strchr(ln, ':'); if (p) { p++; while (*p == ' ') p++;
int n = (int)strlen(p); if (n > 0 && p[n-1] == '\n') p[--n] = 0;
snprintf(cpu, sizeof(cpu), "%s", p); } break; }
fclose(ci); }
#ifdef _SC_NPROCESSORS_ONLN
cores = (int)sysconf(_SC_NPROCESSORS_ONLN);
#endif
FILE *mi = fopen("/proc/meminfo", "r");
if (mi) { char ln[256]; double v = 0;
while (fgets(ln, sizeof(ln), mi)) {
if (sscanf(ln, "MemTotal: %lf", &v) == 1) rt = v/1e6;
if (sscanf(ln, "MemAvailable: %lf", &v) == 1) ra = v/1e6;
} fclose(mi); }
if (rt <= 0.0 || ra <= 0.0) {
double t2 = 0, a2 = 0;
compat_meminfo_gb(&t2, &a2);
if (rt <= 0.0) rt = t2;
if (ra <= 0.0) ra = a2;
}
#ifdef _WIN32
if (cores <= 0) {
SYSTEM_INFO si;
GetSystemInfo(&si);
cores = (int)si.dwNumberOfProcessors;
}
#endif
#ifdef __APPLE__
if (!cpu[0]) {
size_t sl = sizeof(cpu);
if (sysctlbyname("machdep.cpu.brand_string", cpu, &sl, NULL, 0)) cpu[0] = 0;
}
#endif
int ngpu = 0; double vram = 0;
const char *gpu = "";
#ifdef COLI_CUDA
if (g_cuda) { ngpu = 1; vram = ink_cuda_free_bytes()/1e9; gpu = "CUDA device"; }
#endif
(void)m;
printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", cores, rt, ra, ngpu, vram, cpu[0]?cpu:"unknown", gpu);
fflush(stdout);
}
static void serve_tiers_emap(Model *m) {
Cfg *c = &m->c; int E = c->n_experts;
int nsp = 0, filled = 0;
for (int i = 0; i < c->n_layers; i++) if (c->sparse[i]) { nsp++; filled += m->cache[i].n; }
int64_t I = c->moe_inter, D = c->hidden;
int64_t slotb = m->xq ? m->rb13*2*I + m->rb2*D + (2*I+D)*4
: m->quant_bits ? 3*I*D + (2*I+D)*4 : 3*I*D*4;
printf("TIERS 0 %d %d 0.00 %.2f\n", filled, nsp*E - filled, filled*(double)slotb/1e9);
/* EMAP: 1 byte/expert hex — tier(2b: 0=disk 1=RAM 2=Vulkan device)<<6 | heat(6b: log2 usage) */
char *hex = malloc((size_t)nsp*E*2 + 1); int w = 0;
for (int i = 0; i < c->n_layers; i++) {
if (!c->sparse[i]) continue;
for (int e = 0; e < E; e++) {
Slot *resident = slot_indexed(m, i, e);
int tier = resident && resident->filled;
#ifdef COLI_VULKAN
if (vkt_resident(i, e)) tier = 2; /* on the Vulkan device */
#endif
uint32_t u = m->eusage[i] ? m->eusage[i][e] : 0;
int heat = 0; while (u) { heat++; u >>= 1; } if (heat > 63) heat = 63;
int b = (tier << 6) | heat;
hex[w++] = "0123456789abcdef"[b >> 4];
hex[w++] = "0123456789abcdef"[b & 15];
}
}
hex[w] = 0;
printf("EMAP %d %d %s\n", nsp, E, hex);
fflush(stdout); free(hex);
}
/* ---- several conversations at once (KV_SLOTS>1) -------------------------------
* The gateway's cache slots, each a conversation with a state of its own (InkSeq).
* A SUBMIT on a free slot starts its request at once through ink_serve_start, on that
* slot's state: its prefix reuse and photos work as a lone serve's. Then every step
* picks the next token of each active request and runs one forward over a row of
* each (ink_step_rows): the matrices and the experts are read once for all of them.
* A request's frames are a lone request's; they interleave by id. As alone, CANCEL
* ends a request with DONE and STOP is not read. */
static InkSeq *g_ink_mux_seq; /* [slots]: the conversations the Model does not hold */
static int g_ink_mux_cur; /* the slot the Model holds, -1 when every one is parked */
typedef struct {
SReq q;
int active, cancel, limited, forwards;
int *ids, np, gen;
float *lo; /* the logits the next pick reads */
int hist[128], nhist; /* the repetition penalty's history, as serve_one keeps it */
InkReqClock clk;
} InkMuxReq;
static void ink_mux_bind(Model *m, int slot) {
if (g_ink_mux_cur == slot) return;
if (g_ink_mux_cur >= 0) ink_seq_swap(m, &g_ink_mux_seq[g_ink_mux_cur]);
if (slot >= 0) ink_seq_swap(m, &g_ink_mux_seq[slot]);
g_ink_mux_cur = slot;
}
/* A request's end, as serve_one ends one. */
static void ink_mux_finish(Model *m, InkMuxReq *r) {
free(r->lo); r->lo = NULL; free(r->ids); r->ids = NULL; r->active = 0;
if (r->cancel) r->limited = 0;
double dt = now_s() - r->clk.t0;
double tot = (double)(m->hits - r->clk.h0 + m->miss - r->clk.m0);
ColiServeDone done = {r->gen, dt > 0 ? r->gen/dt : 0.0,
tot ? 100.0*(m->hits - r->clk.h0)/tot : 0.0, rss_gb(), r->np, r->limited};
char done_line[256];
int done_bytes = coli_serve_format_done(done_line, sizeof(done_line), r->q.id, &done);
if (done_bytes > 0) fwrite(done_line, 1, (size_t)done_bytes, stdout);
printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n", dt, r->np, r->gen,
m->t_fill - r->clk.f0, m->t_shared - r->clk.s0, m->t_expert - r->clk.e0, m->t_attn - r->clk.a0,
0.0, r->forwards);
fflush(stdout);
serve_hits(m);
}
/* The next token of an active request, as serve_one's loop picks and sends it: 1
* with the token when the request goes on, 0 when it ended. */
static int ink_mux_pick(Model *m, Tok *T, InkMuxReq *r, float rep, int *tk_out) {
Cfg *c = &m->c;
if (r->cancel || r->gen >= r->q.max_tok) { ink_mux_finish(m, r); return 0; }
apply_rep_penalty(r->lo, c->unpad_vocab, r->hist, r->nhist, rep);
int tk = sample_logits(r->lo, c->unpad_vocab, r->q.temp, r->q.top_p);
char lptail[1024]; lptail[0] = 0;
if (r->q.logprobs > 0) coli_logprob_tail(lptail, sizeof lptail, r->lo, c->unpad_vocab, tk, r->q.logprobs);
free(r->lo); r->lo = NULL;
if (tk == c->eos) { r->limited = 0; ink_mux_finish(m, r); return 0; }
if (r->nhist < 128) r->hist[r->nhist++] = tk;
else { memmove(r->hist, r->hist + 1, 127*sizeof(int)); r->hist[127] = tk; }
char buf[512];
int nb = tok_decode(T, &tk, 1, buf, sizeof(buf)-1);
if (r->q.logprobs > 0) coli_serve_write_data_lp(stdout, r->q.id, buf, (size_t)nb, lptail);
else coli_serve_write_data(stdout, r->q.id, buf, (size_t)nb);
r->gen++;
if (r->gen >= r->q.max_tok) { ink_mux_finish(m, r); return 0; }
*tk_out = tk; return 1;
}
static void serve_mux(Model *m, Tok *T) {
int n = g_ink_mux_slots, V = m->c.unpad_vocab, input_eof = 0;
InkMuxReq *rq = calloc((size_t)n, sizeof *rq);
InkRow *rows = malloc((size_t)n * sizeof *rows);
int *tok = malloc((size_t)n * sizeof(int)), *who = malloc((size_t)n * sizeof(int));
if (!rq || !rows || !tok || !who) { fprintf(stderr, "[serve] out of memory\n"); exit(1); }
float rep = getenv("REP_PEN") ? atof(getenv("REP_PEN")) : 1.1f;
unsigned long long steps = 0, nrows = 0;
fprintf(stderr, "[inkling] serving %d conversations at once (KV_SLOTS)\n", n);
for (;;) {
int active = 0; for (int i = 0; i < n; i++) active += rq[i].active;
/* idle: wait for a command; decoding: take one only when one is there */
if (!input_eof && (!active || stdin_readable())) {
ColiServeCommand command;
ColiServeReadResult result = coli_serve_read_command(stdin, &inkling_wire, &command);
if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) input_eof = 1;
else if (result == COLI_SERVE_READ_NOMEM) { coli_serve_write_error(stdout, command.id, "out of memory"); input_eof = 1; }
else if (result == COLI_SERVE_READ_BAD_REQUEST) {
if (command.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, command.id, "bad submit header");
coli_serve_command_dispose(&command);
} else if (result == COLI_SERVE_READ_OK) {
if (command.kind == COLI_SERVE_COMMAND_CANCEL) {
for (int i = 0; i < n; i++) if (rq[i].active && !strcmp(rq[i].q.id, command.id)) rq[i].cancel = 1;
} else if (command.kind == COLI_SERVE_COMMAND_SUBMIT) {
if (command.slot < 0 || command.slot >= n) coli_serve_write_error(stdout, command.id, "invalid cache slot");
else if (rq[command.slot].active) coli_serve_write_error(stdout, command.id, "SLOT_BUSY");
else {
InkMuxReq *t = &rq[command.slot];
memset(t, 0, sizeof *t);
snprintf(t->q.id, sizeof(t->q.id), "%s", command.id);
t->q.max_tok = command.max_tokens; t->q.temp = command.temperature;
t->q.logprobs = command.logprobs; t->q.pin = command.pin;
t->q.top_p = command.top_p; t->q.plen = (int)command.payload_bytes;
t->q.alen = (int)command.extension_bytes;
t->q.audio = coli_serve_command_extension(&command);
t->q.payload = (char *)coli_serve_command_take_payload(&command);
ink_mux_bind(m, command.slot);
int ok = ink_serve_start(m, T, &t->q, &t->ids, &t->np, &t->lo, &t->clk);
free(t->q.payload); t->q.payload = NULL; t->q.audio = NULL;
if (ok) {
t->active = 1; t->limited = 1; t->forwards = 1;
for (int i = (t->np > 128 ? t->np - 128 : 0); i < t->np; i++) t->hist[t->nhist++] = t->ids[i];
}
}
}
coli_serve_command_dispose(&command);
}
}
active = 0; for (int i = 0; i < n; i++) active += rq[i].active;
if (!active) { if (input_eof) break; continue; }
int S = 0, ended = 0;
for (int i = 0; i < n; i++) if (rq[i].active) {
int tk;
if (!ink_mux_pick(m, T, &rq[i], rep, &tk)) { ended = 1; continue; }
rows[S] = (InkRow){&g_ink_mux_seq[i], rq[i].np + rq[i].gen - 1}; tok[S] = tk; who[S] = i; S++;
}
if (S) {
ink_mux_bind(m, -1); /* every conversation parked: the rows read theirs */
float *lo = ink_step_rows(m, rows, tok, S);
steps++; nrows += (unsigned long long)S;
for (int s = 0; s < S; s++) {
InkMuxReq *t = &rq[who[s]];
t->lo = falloc(V); memcpy(t->lo, lo + (int64_t)s * V, (size_t)V * sizeof(float));
t->forwards++;
}
free(lo);
}
if (ended) {
#ifdef COLI_VULKAN
ink_vk_report(); ink_vk_tier_report(m, "turn");
#endif
serve_tiers_emap(m);
}
}
fprintf(stderr, "[inkling] KV_SLOTS=%d: %llu decode steps, %llu rows (%.2f a step)\n", n, steps, nrows,
steps ? (double)nrows / (double)steps : 0.0);
ink_mux_bind(m, 0);
free(rq); free(rows); free(tok); free(who);
}
static void serve_loop(Model *m, Tok *T) {
/* Before the sentinel: on Windows a TEXT-mode stdout rewrites the trailing \n
* as \r\n, the gateway never matches it and waits forever (#748). Lives in
* compat.h because colibri.c has had it since #195 and this engine was
* written without it. */
coli_serve_stdio_init();
const char *sd = getenv("SEED");
if (sd) g_rng ^= (uint64_t)strtoull(sd, NULL, 10);
else g_rng ^= (uint64_t)time(NULL) * 2654435761u;
/* the gateway reads a STAT line right after the READY sentinel (colibri
* reports its load stats there) — match the handshake */
coli_serve_write_ready(stdout,rss_gb());
serve_hwinfo(m);
serve_tiers_emap(m);
if (g_ink_mux_slots > 1) { serve_mux(m, T); return; }
for (;;) {
while (!g_qn) if (serve_read_cmd(stdin, stdout, NULL) < 0) return; /* blocks on stdin */
SReq q = g_q[0];
memmove(g_q, g_q+1, (size_t)(--g_qn) * sizeof(SReq));
int fatal=serve_one(m,T,&q);
serve_tiers_emap(m);
serve_request_dispose(&q);
if(fatal<0){ while(g_qn) serve_request_dispose(&g_q[--g_qn]); return; }
}
}
/* ---------- ref_inkling.json harness ---------- */
static int *read_int_array(jval *o, const char *key, int *n_out) {
jval *a = json_get(o, key);
if (!a || a->t != J_ARR) { *n_out = 0; return NULL; }
int *r = malloc(a->len * sizeof(int));
for (int i = 0; i < a->len; i++) r[i] = (int)a->kids[i]->num;
*n_out = a->len; return r;
}
#ifndef INKLING_NO_MAIN
int main(int argc, char **argv) {
/* OpenMP hot-thread tuning, same trick (and rationale) as glm.c: the
* per-expert matmul regions are tiny and back-to-back; the default passive
* wait policy parks the team between regions and re-wake latency dominates.
* libgomp reads OMP_/GOMP_ vars before main(), so seed them and re-exec
* once (COLI_OMP_TUNED guards the exec; COLI_NO_OMP_TUNE=1 disables).
* NOT under CUDA — same exception glm.c makes: a spinning 24-thread team
* starves the CUDA driver during every stream sync. */
#if !defined(COLI_CUDA) && !defined(__APPLE__)
/* NOT on Apple Silicon: the active-spin team steals the shared SoC power
* budget (same mechanism as the M5 Max Metal report) and measured strictly
* worse on Inkling-Small even for pure-CPU decode — 0.50 vs 0.84 tok/s,
* and 2x the prefill. Opt back in with COLI_OMP_TUNED=0 unset + Linux. */
if (!getenv("COLI_OMP_TUNED") && !getenv("COLI_NO_OMP_TUNE")) {
setenv("OMP_WAIT_POLICY","active",0);
setenv("GOMP_SPINCOUNT","200000",0);
setenv("OMP_PROC_BIND","close",0);
setenv("OMP_DYNAMIC","FALSE",0);
setenv("COLI_OMP_TUNED","1",1);
#ifdef __linux__
execv("/proc/self/exe", argv);
perror("[OMP] execv self-reexec failed, running untuned");
#endif
}
#endif /* !COLI_CUDA && !__APPLE__ */
coli_omp_tune_threads("inkling");
const char *snap = getenv("SNAP");
if (!snap) { coli_print_launcher_help("Inkling", "SNAP=<model directory> ./inkling ..."); return 1; }
g_topp = getenv("TOPP") ? (float)atof(getenv("TOPP")) : 0.f;
if (g_topp > 0.f && g_topp < 1.f)
fprintf(stderr, "[TOPP] %.2f: routed experts kept to cumulative weight — "
"fewer experts read per token, but the routing is TRIMMED "
"(quality lever, A/B it before trusting the speed-up)\n", g_topp);
else if (g_topp != 0.f) {
fprintf(stderr, "TOPP must be in (0,1); %.3f ignored\n", g_topp); g_topp = 0.f;
}
/* flags: -p "prompt" [-n N] -> generate mode; positional: [cap] [bits] [ref.json] */
const char *prompt = NULL, *pfile = NULL, *refpath = "ref_inkling.json";
const char *audiopath = NULL;
int cap = -1, bits = 0, n_new = 256, npos = 0, chat = 0;
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "-p") && i+1 < argc) prompt = argv[++i];
else if (!strcmp(argv[i], "-f") && i+1 < argc) pfile = argv[++i];
else if (!strcmp(argv[i], "-n") && i+1 < argc) n_new = coli_arg_int(argv[++i], "-n");
else if (!strcmp(argv[i], "--chat")) chat = 1;
else if (!strcmp(argv[i], "--audio") && i+1 < argc) audiopath = argv[++i];
else if (npos == 0) { cap = coli_arg_int(argv[i], "cache/layer"); npos++; }
else if (npos == 1) { bits = coli_arg_int(argv[i], "expert bits"); npos++; }
else refpath = argv[i];
}
/* --audio <file>: raw u8 DMel frames, [n_frames, mel_bins] row-major —
* what tinkernel-audio / the gateway DSP emit. Implies --chat (the audio
* message needs TMLv0 framing to be in distribution). */
uint8_t *dmel = NULL; long dmel_bytes = 0;
if (audiopath) {
FILE *af = fopen(audiopath, "rb");
if (!af) { perror(audiopath); return 1; }
fseek(af, 0, SEEK_END); dmel_bytes = ftell(af); fseek(af, 0, SEEK_SET);
dmel = malloc((size_t)dmel_bytes);
if (fread(dmel, 1, (size_t)dmel_bytes, af) != (size_t)dmel_bytes) {
fprintf(stderr, "short read on %s\n", audiopath); return 1; }
fclose(af);
chat = 1;
if (!prompt) prompt = "";
}
/* --chat: avvolge il prompt nel template di Inkling (sottoinsieme testuale di
* chat_template.jinja, lo stesso che openai_server.py rende in render_chat_inkling):
* token di ruolo + <|content_text|>, <|end_message|> a chiudere, il livello di
* thinking come messaggio di sistema, e <|message_model|> come prompt di
* generazione. Senza template un modello instruct riceve testo fuori
* distribuzione. THINK=<0..1> alza lo sforzo di ragionamento (default 0). */
char *chat_buf = NULL;
if (chat && prompt && !dmel) {
const char *eff = getenv("THINK") ? getenv("THINK") : "0";
size_t need = strlen(prompt) + strlen(eff) + 256;
chat_buf = malloc(need);
if (!chat_buf) { fprintf(stderr, "OOM chat template\n"); return 1; }
snprintf(chat_buf, need,
"<|message_user|><|content_text|>%s<|end_message|>"
"<|message_system|><|content_text|>Thinking effort level: %s<|end_message|>"
"<|message_model|>", prompt, eff);
prompt = chat_buf;
} else if (chat && dmel) {
/* audio turn, TMLv0 framing (conformant with tml-renderers 0.1.0):
* effort system message, optional text user message, then the audio
* message — <|content_audio_input|>, one <|audio|> per frame,
* <|audio_end|> — and <|message_model|> to hand the turn over.
* Frame count is provisional here (mel_bins is read from config.json
* at model_init); recomputed below once the model is loaded. */
const char *eff = getenv("THINK") ? getenv("THINK") : "0";
long nf_guess = dmel_bytes / 80; /* placeholder emission only */
size_t need = strlen(prompt) + strlen(eff) + (size_t)nf_guess*10 + 320;
chat_buf = malloc(need);
if (!chat_buf) { fprintf(stderr, "OOM chat template\n"); return 1; }
char *w = chat_buf;
w += sprintf(w, "<|message_system|><|content_text|>Thinking effort level: %s<|end_message|>", eff);
if (prompt[0])
w += sprintf(w, "<|message_user|><|content_text|>%s<|end_message|>", prompt);
w += sprintf(w, "<|message_user|><|content_audio_input|>");
for (long i = 0; i < nf_guess; i++) w += sprintf(w, "<|audio|>");
sprintf(w, "<|audio_end|><|end_message|><|message_model|>");
prompt = chat_buf;
}
if (cap < 0) cap = (prompt || pfile) ? 0 : 16; /* generate mode defaults to RAM-sized auto cap */
if (bits && (bits < 2 || bits > 8)) { fprintf(stderr, "quant_bits must be 0 (f32) or 2..8\n"); return 1; }
/* SERVE=1: the openai_server.py gateway drives the engine over stdin/stdout
* (READY handshake, SUBMIT/CANCEL, DATA/DONE frames) — same protocol colibri. */
if (getenv("SERVE") && getenv("SERVE")[0] == '1') {
const char *ks = getenv("KV_SLOTS"); /* the conversations decoded at once */
if (ks && *ks) {
char *end = NULL; long v = strtol(ks, &end, 10);
if (end == ks || *end || v < 1 || v > 16) { fprintf(stderr, "KV_SLOTS must be between 1 and 16\n"); return 2; }
g_ink_mux_slots = (int)v;
}
Model m; model_init(&m, snap, cap, bits);
pins_load(&m, snap);
#ifdef COLI_VULKAN
ink_vk_tier_start(&m); /* after the history: the warm start reads it */
inkc_start(&m); /* COLI_VK_CHAIN, after the tier */
#endif
char tkp[2048]; snprintf(tkp, sizeof(tkp), "%s/tokenizer.json", snap);
Tok T; tok_load(&T, tkp);
if (g_ink_mux_slots > 1) { /* the whole context's state for every slot; slot 0 is the Model's */
kv_alloc(&m, ink_ctx_max() + 8);
g_ink_mux_seq = calloc((size_t)g_ink_mux_slots, sizeof *g_ink_mux_seq);
if (!g_ink_mux_seq) { fprintf(stderr, "[serve] out of memory for %d conversations\n", g_ink_mux_slots); return 1; }
for (int i = 1; i < g_ink_mux_slots; i++) ink_seq_alloc(&m, &g_ink_mux_seq[i]);
g_ink_mux_cur = 0;
}
coli_rt_term_arm(); /* SIGTERM must reach the save below (#1629) */
serve_loop(&m, &T);
usage_save(&m, snap);
return 0;
}
if (prompt || pfile) {
Model m; model_init(&m, snap, cap, bits);
printf("== Inkling C engine, %d layers, experts @ %s, cache %d/layer ==\n",
m.c.n_layers, m.xq ? "container" : bits ? "int" : "f32", m.cache[0].cap);
pins_load(&m, snap);
#ifdef COLI_VULKAN
ink_vk_tier_start(&m);
inkc_start(&m); /* COLI_VK_CHAIN, after the tier */
#endif
char tkp[2048]; snprintf(tkp, sizeof(tkp), "%s/tokenizer.json", snap);
Tok T; tok_load(&T, tkp);
if (prompt) {
int naud = dmel ? (int)(dmel_bytes / m.c.mel_bins) : 0;
if (dmel && dmel_bytes % m.c.mel_bins != 0) {
fprintf(stderr, "%s: %ld bytes is not a multiple of mel_bins=%d\n",
audiopath, dmel_bytes, m.c.mel_bins); return 1; }
generate_stream(&m, &T, prompt, n_new, dmel, naud);
}
else { /* -f: one prompt per line, model loaded once, usage accumulates */
FILE *pf = fopen(pfile, "rb"); if (!pf) { perror(pfile); return 1; }
char ln[8192]; int np = 0;
while (fgets(ln, sizeof(ln), pf)) {
size_t n = strlen(ln); while (n && (ln[n-1]=='\n'||ln[n-1]=='\r')) ln[--n]=0;
if (!n || ln[0]=='#') continue;
printf("\n===== prompt %d =====\n", ++np);
state_reset(&m);
generate_stream(&m, &T, ln, n_new, NULL, 0);
}
fclose(pf);
}
int saved = usage_save(&m, snap);
double tot = m.hits + m.miss;
printf("[cache] hit %.1f%% (%llu hit / %llu load)%s\n",
tot ? 100.0*m.hits/tot : 0.0,
(unsigned long long)m.hits, (unsigned long long)m.miss,
saved ? " | usage history saved" : "");
/* quanto ha tagliato TOPP, in modo che la leva sia misurabile e non creduta */
if (g_topp > 0.f && m.ereq)
printf("[topp] %.2f: %llu/%llu routed used (%.1f%% trimmed, "
"%.2f experts/token avg)\n", g_topp,
(unsigned long long)m.euse, (unsigned long long)m.ereq,
100.0*(double)(m.ereq-m.euse)/(double)m.ereq,
(double)m.euse/((double)m.ereq/(double)m.c.topk));
return 0;
}
FILE *f = fopen(refpath, "rb"); if(!f){perror(refpath);return 1;}
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
char *buf=malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f);
char *arena=NULL; jval *ref = json_parse(buf, &arena);
int np, nfull, ntf, ndm;
int *pids = read_int_array(ref,"prompt_ids",&np);
int *full = read_int_array(ref,"full_ids",&nfull);
int *tfref = read_int_array(ref,"tf_pred",&ntf);
/* optional audio oracle: "dmel" = flattened [n_frames, mel_bins] levels */
int *dmint = read_int_array(ref,"dmel",&ndm);
int ngen = nfull - np;
Model m; model_init(&m, snap, cap, bits);
#ifdef COLI_VULKAN
ink_vk_tier_start(&m); /* the oracle reads no history: the tier fills as experts pass by */
inkc_start(&m); /* COLI_VK_CHAIN, after the tier */
#endif
printf("== Inkling C engine (Stage A), cache = %d experts/layer, experts @ %s ==\n",
cap, m.xq ? "container (int4/int8 + .qs)" : bits ? "int (runtime quant)" : "f32");
printf("cfg: D=%d L=%d V=%d(%d) heads=%d/%d kv=%d/%d hd=%d win=%d d_rel=%d ext=%d E=%d+%d topk=%d\n",
m.c.hidden, m.c.n_layers, m.c.vocab, m.c.unpad_vocab, m.c.n_heads, m.c.swa_heads,
m.c.n_kv, m.c.swa_kv, m.c.head_dim, m.c.window, m.c.d_rel, m.c.rel_extent,
m.c.n_experts, m.c.n_shared, m.c.topk);
printf("resident weights loaded in %.1fs | RSS: %.2f GB\n", m.dense_load_s, rss_gb());
kv_alloc(&m, nfull + 8);
uint8_t *rdmel = NULL; int rnaud = 0;
if (dmint && ndm > 0) {
if (ndm % m.c.mel_bins != 0) { fprintf(stderr, "dmel len %d not a multiple of mel_bins %d\n", ndm, m.c.mel_bins); return 1; }
rnaud = ndm / m.c.mel_bins;
rdmel = malloc((size_t)ndm);
for (int i = 0; i < ndm; i++) rdmel[i] = (uint8_t)dmint[i];
printf("audio oracle: %d DMel frames x %d bins\n", rnaud, m.c.mel_bins);
}
/* pass 1: teacher-forced argmax over the full reference sequence */
if (tfref && ntf == nfull) {
int *tf = malloc(nfull * sizeof(int));
float *lg = step_mm(&m, full, nfull, 0, tf, rdmel, rnaud);
free(lg);
int ok = 0; for (int i = 0; i < nfull; i++) ok += (tf[i] == tfref[i]);
printf("teacher-forced argmax: %d/%d match\n", ok, nfull);
free(tf);
state_reset(&m);
}
/* pass 2: greedy generation, token-for-token vs the oracle */
int *out = malloc(nfull * sizeof(int));
double t = now_s();
generate(&m, pids, np, ngen, out, rdmel, rnaud);
double dt = now_s() - t;
int match = 0;
printf("Reference: "); for (int i=np;i<nfull;i++) printf("%d ", full[i]);
printf("\nC engine : "); for (int i=np;i<nfull;i++) { printf("%d ", out[i]); if (out[i]==full[i]) match++; }
printf("\nMatching tokens: %d/%d\n", match, ngen);
double tot = m.hits + m.miss;
#ifdef COLI_METAL
if (g_metal) {
uint64_t mok = 0, mfb = 0, mex = 0;
coli_metal_moe_counts(&mok, &mfb, &mex);
printf("[metal] %llu MoE blocks on GPU, %llu CPU fallbacks, %llu experts\n",
(unsigned long long)mok, (unsigned long long)mfb, (unsigned long long)mex);
}
#endif
printf("PEAK RSS: %.2f GB | expert cache hit %.1f%% | %.2f tok/s\n",
rss_gb(), tot?100.0*m.hits/tot:0.0, ngen/dt);
#ifdef COLI_VULKAN
fflush(stdout); ink_vk_report(); ink_vk_tier_report(&m, "run"); inkc_report(&m);
#endif
free(buf); free(arena);
return (match == ngen) ? 0 : 1;
}
#endif /* INKLING_NO_MAIN */
#ifdef COLI_SEGMENT_ADAPTER
/* ---------- engine-owned Segment adapter ------------------------------ */
typedef struct {
Model model;
uint32_t layer_begin, layer_end, context_tokens;
pthread_mutex_t run_lock;
} InklingSegmentEngine;
typedef struct {
InklingSegmentEngine *engine;
float **K, **V;
float **cs[4];
uint32_t context_tokens, position;
} InklingSegmentSession;
static void inkling_segment_wt_destroy(Wt *weight) {
if (!weight) return;
#ifdef COLI_VULKAN
ink_vk_release(weight);
free((void *)weight->vk_name);
#endif
free(weight->f); free(weight->h);
free(weight->q4); free(weight->qs);
memset(weight, 0, sizeof(*weight));
}
static void inkling_segment_layer_destroy(Layer *layer) {
if (!layer) return;
free(layer->in_ln); free(layer->post_ln);
inkling_segment_wt_destroy(&layer->q);
inkling_segment_wt_destroy(&layer->k);
inkling_segment_wt_destroy(&layer->v);
inkling_segment_wt_destroy(&layer->r);
inkling_segment_wt_destroy(&layer->o);
free(layer->qn); free(layer->kn); free(layer->relp);
free(layer->k_cw); free(layer->v_cw);
free(layer->a_cw); free(layer->m_cw);
inkling_segment_wt_destroy(&layer->dg);
inkling_segment_wt_destroy(&layer->du);
inkling_segment_wt_destroy(&layer->dd);
free(layer->router); free(layer->rbias);
inkling_segment_wt_destroy(&layer->sh_g);
inkling_segment_wt_destroy(&layer->sh_u);
inkling_segment_wt_destroy(&layer->sh_d);
}
static void inkling_segment_model_destroy(InklingSegmentEngine *engine) {
if (!engine) return;
Model *model = &engine->model;
for (uint32_t layer = engine->layer_begin; layer < engine->layer_end;
layer++) {
inkling_segment_layer_destroy(&model->L[layer]);
LCache *cache = &model->cache[layer];
if (model->xq && cache->cap > 0 && cache->slots) {
free(cache->slots[0].p13);
free(cache->slots[0].s13);
} else if (cache->slots) {
for (int slot = 0; slot < cache->cap; slot++) {
free(cache->slots[slot].q13); free(cache->slots[slot].q2);
free(cache->slots[slot].s13); free(cache->slots[slot].s2);
free(cache->slots[slot].f13); free(cache->slots[slot].f2);
}
}
free(cache->slot_by_expert); free(cache->slots);
}
inkling_segment_wt_destroy(&model->embed);
inkling_segment_wt_destroy(&model->lm_head);
inkling_segment_wt_destroy(&model->audio_enc);
free(model->embed_norm); free(model->final_norm); free(model->audio_norm);
free(model->K); free(model->V);
for (int state = 0; state < 4; state++) free(model->cs[state]);
free(model->cache); free(model->L);
kv_prefix_free(&model->kvp);
st_destroy(&model->Sq); st_destroy(&model->S);
}
static int inkling_segment_engine_open(
void **engine_impl, ColiSegmentCapabilities *capabilities,
const ColiSegmentEngineOptions *options, char *error, size_t error_size) {
if (!engine_impl || !capabilities || !options)
return coli_segment_adapter_error(error, error_size,
"invalid Inkling Segment open");
*engine_impl = NULL;
if (options->backend_mask &&
(options->backend_mask & ~COLI_SEGMENT_CAP_CPU))
return coli_segment_adapter_error(error, error_size,
"Inkling Segment currently supports CPU");
Cfg config;
memset(&config, 0, sizeof(config));
load_cfg(&config, options->model_dir);
if (options->layer_end > (uint32_t)config.n_layers)
return coli_segment_adapter_error(error, error_size,
"Inkling Segment range exceeds model");
int bits = getenv("INK_SEGMENT_BITS")
? atoi(getenv("INK_SEGMENT_BITS")) : 0;
if (bits && (bits < 2 || bits > 8))
return coli_segment_adapter_error(error, error_size,
"INK_SEGMENT_BITS must be 0 or 2..8");
int sparse_layers = 0;
for (uint32_t layer = options->layer_begin; layer < options->layer_end;
layer++) sparse_layers += config.sparse[layer] != 0;
int cap = 0;
if (options->memory_limit_bytes && sparse_layers) {
uint64_t slot_bytes = bits
? 3u * (uint64_t)config.moe_inter * config.hidden +
(uint64_t)(2 * config.moe_inter + config.hidden) * sizeof(float)
: 3u * (uint64_t)config.moe_inter * config.hidden * sizeof(float);
uint64_t slots = slot_bytes
? options->memory_limit_bytes / slot_bytes /
(uint64_t)sparse_layers : 0;
cap = slots > (uint64_t)config.n_experts
? config.n_experts : (int)slots;
if (cap < 1) cap = 1;
}
InklingSegmentEngine *engine = calloc(1, sizeof(*engine));
if (!engine)
return coli_segment_adapter_error(error, error_size,
"out of memory opening Inkling Segment");
engine->layer_begin = options->layer_begin;
engine->layer_end = options->layer_end;
engine->context_tokens = options->context_tokens;
if (pthread_mutex_init(&engine->run_lock, NULL)) {
free(engine);
return coli_segment_adapter_error(error, error_size,
"cannot initialize Inkling Segment lock");
}
model_init_range(&engine->model, options->model_dir, cap, bits,
(int)options->layer_begin, (int)options->layer_end,
0, 0, 0);
for (int state = 0; state < 4; state++) {
free(engine->model.cs[state]);
engine->model.cs[state] = NULL;
}
memset(capabilities, 0, sizeof(*capabilities));
capabilities->struct_size = sizeof(*capabilities);
capabilities->abi_version = COLI_SEGMENT_ABI_VERSION;
capabilities->flags = COLI_SEGMENT_CAP_SNAPSHOT |
COLI_SEGMENT_CAP_RANGE_NATIVE |
COLI_SEGMENT_CAP_MULTI_SESSION |
COLI_SEGMENT_CAP_CPU;
coli_segment_capability_string(capabilities->engine_id,
sizeof(capabilities->engine_id), "inkling");
coli_segment_capability_string(capabilities->state_schema,
sizeof(capabilities->state_schema),
"inkling/kv-ring-conv-f32-v1");
snprintf(capabilities->numeric_class,
sizeof(capabilities->numeric_class),
"inkling/expert-q%d/f32/cpu-v1", bits);
capabilities->state_dtype = COLI_SEGMENT_DTYPE_F32;
capabilities->state_width = (uint32_t)config.hidden;
capabilities->max_batch_rows = 128;
capabilities->max_context_tokens = options->context_tokens;
capabilities->num_layers = (uint32_t)config.n_layers;
*engine_impl = engine;
return 0;
}
static void inkling_segment_engine_destroy(void *engine_impl) {
InklingSegmentEngine *engine = (InklingSegmentEngine *)engine_impl;
if (!engine) return;
inkling_segment_model_destroy(engine);
pthread_mutex_destroy(&engine->run_lock);
free(engine);
}
static void inkling_segment_session_free(InklingSegmentSession *session) {
if (!session) return;
if (session->engine) {
for (uint32_t layer = session->engine->layer_begin;
layer < session->engine->layer_end; layer++) {
free(session->K ? session->K[layer] : NULL);
free(session->V ? session->V[layer] : NULL);
for (int state = 0; state < 4; state++)
free(session->cs[state] ? session->cs[state][layer] : NULL);
}
}
free(session->K); free(session->V);
for (int state = 0; state < 4; state++) free(session->cs[state]);
free(session);
}
static int inkling_segment_session_create(
void *engine_impl, void **session_impl,
const ColiSegmentSessionOptions *options, char *error, size_t error_size) {
InklingSegmentEngine *engine = (InklingSegmentEngine *)engine_impl;
if (!engine || !session_impl || !options)
return coli_segment_adapter_error(error, error_size,
"invalid Inkling Segment session");
*session_impl = NULL;
InklingSegmentSession *session = calloc(1, sizeof(*session));
if (!session)
return coli_segment_adapter_error(error, error_size,
"out of memory creating Inkling session");
session->engine = engine;
session->context_tokens = options->context_tokens;
size_t layers = (size_t)engine->model.c.n_layers;
session->K = calloc(layers, sizeof(*session->K));
session->V = calloc(layers, sizeof(*session->V));
for (int state = 0; state < 4; state++)
session->cs[state] = calloc(layers, sizeof(*session->cs[state]));
if (!session->K || !session->V || !session->cs[0] || !session->cs[1] ||
!session->cs[2] || !session->cs[3]) goto oom;
Cfg *config = &engine->model.c;
uint64_t state_bytes = 0;
for (uint32_t layer = engine->layer_begin; layer < engine->layer_end;
layer++) {
size_t cells;
int kvdim = L_KV(config, layer) * L_HD(config, layer);
int rows = kv_ring_rows(config, (int)layer,
(int)options->context_tokens);
if (coli_segment_size_mul((size_t)L_KV(config, layer),
(size_t)rows, &cells) ||
coli_segment_size_mul(cells, (size_t)L_HD(config, layer),
&cells)) goto oom;
session->K[layer] = calloc(cells, sizeof(float));
session->V[layer] = calloc(cells, sizeof(float));
state_bytes += 2u * (uint64_t)cells * sizeof(float);
for (int state = 0; state < 4; state++) {
int width = state < 2 ? kvdim : config->hidden;
if (coli_segment_size_mul((size_t)width,
(size_t)(config->conv_k - 1),
&cells)) goto oom;
session->cs[state][layer] = calloc(cells, sizeof(float));
state_bytes += (uint64_t)cells * sizeof(float);
}
if (!session->K[layer] || !session->V[layer] ||
!session->cs[0][layer] || !session->cs[1][layer] ||
!session->cs[2][layer] || !session->cs[3][layer]) goto oom;
}
if (options->memory_limit_bytes &&
state_bytes > options->memory_limit_bytes) {
inkling_segment_session_free(session);
return coli_segment_adapter_error(error, error_size,
"Inkling session state exceeds memory limit");
}
*session_impl = session;
return 0;
oom:
inkling_segment_session_free(session);
return coli_segment_adapter_error(error, error_size,
"out of memory allocating Inkling state");
}
static void inkling_segment_session_destroy(void *session_impl) {
inkling_segment_session_free((InklingSegmentSession *)session_impl);
}
static int inkling_segment_session_run(void *session_impl,
const ColiSegmentRunRequest *request,
char *error, size_t error_size) {
InklingSegmentSession *session = (InklingSegmentSession *)session_impl;
if (!session || !request || request->position != session->position)
return coli_segment_adapter_error(
error, error_size, "Inkling Segment requires contiguous positions");
if (request->should_cancel &&
request->should_cancel(request->cancel_user_data))
return coli_segment_adapter_error(error, error_size,
"Inkling Segment run cancelled");
InklingSegmentEngine *engine = session->engine;
if (request->output != request->input)
memcpy(request->output, request->input, request->input_bytes);
pthread_mutex_lock(&engine->run_lock);
Model *model = &engine->model;
model->K = session->K; model->V = session->V;
for (int state = 0; state < 4; state++) model->cs[state] = session->cs[state];
model->max_t = (int)session->context_tokens;
model->kv_len = (int)session->position;
inkling_layers_forward_range(model, (float *)request->output,
(int)request->rows, (int)request->position,
(int)engine->layer_begin,
(int)engine->layer_end);
model->K = NULL; model->V = NULL;
for (int state = 0; state < 4; state++) model->cs[state] = NULL;
model->max_t = 0; model->kv_len = 0;
pthread_mutex_unlock(&engine->run_lock);
session->position += request->rows;
return 0;
}
static int inkling_segment_spans(
InklingSegmentSession *session, uint32_t position,
ColiSegmentStateSpan **spans_output, size_t *count_output,
char *error, size_t error_size) {
Cfg *config = &session->engine->model.c;
size_t capacity = 0;
for (uint32_t layer = session->engine->layer_begin;
layer < session->engine->layer_end; layer++)
capacity += (size_t)(2 * L_KV(config, layer) + 4);
ColiSegmentStateSpan *spans = capacity
? calloc(capacity, sizeof(*spans)) : NULL;
if (capacity && !spans)
return coli_segment_adapter_error(error, error_size,
"out of memory describing Inkling state");
size_t count = 0;
for (uint32_t layer = session->engine->layer_begin;
layer < session->engine->layer_end; layer++) {
int kv = L_KV(config, layer), hd = L_HD(config, layer);
int rows = kv_ring_rows(config, (int)layer,
(int)session->context_tokens);
size_t live_rows = config->local[layer] ? (size_t)rows : position;
size_t stride = (size_t)rows * hd;
size_t row_bytes = live_rows * hd * sizeof(float);
for (int which = 0; which < 2; which++) {
float *state = which ? session->V[layer] : session->K[layer];
for (int head = 0; head < kv; head++)
spans[count++] = (ColiSegmentStateSpan){
state + (size_t)head * stride, row_bytes};
}
int kvdim = kv * hd;
for (int state = 0; state < 4; state++) {
int width = state < 2 ? kvdim : config->hidden;
spans[count++] = (ColiSegmentStateSpan){
session->cs[state][layer],
(size_t)width * (config->conv_k - 1) * sizeof(float)};
}
}
*spans_output = spans; *count_output = count;
return 0;
}
static int inkling_segment_session_snapshot(
void *session_impl, ColiSegmentWriteFn write_fn, void *write_user_data,
char *error, size_t error_size) {
InklingSegmentSession *session = (InklingSegmentSession *)session_impl;
ColiSegmentStateSpan *spans = NULL;
size_t count = 0, payload_bytes;
if (!session || inkling_segment_spans(session, session->position, &spans,
&count, error, error_size))
return -1;
if (coli_segment_spans_size(spans, count, &payload_bytes)) {
free(spans);
return coli_segment_adapter_error(error, error_size,
"Inkling snapshot size overflow");
}
ColiSegmentSnapshotHeader header;
coli_segment_snapshot_header_init(
&header, "inkling", session->engine->layer_begin,
session->engine->layer_end, session->context_tokens, session->position,
payload_bytes, coli_segment_spans_hash(spans, count));
int result = coli_segment_stream_write(
write_fn, write_user_data, &header, sizeof(header), error, error_size);
if (!result)
result = coli_segment_spans_write(spans, count, write_fn,
write_user_data, error, error_size);
free(spans);
return result;
}
static int inkling_segment_session_restore(
void *session_impl, ColiSegmentReadFn read_fn, void *read_user_data,
char *error, size_t error_size) {
InklingSegmentSession *session = (InklingSegmentSession *)session_impl;
ColiSegmentSnapshotHeader header;
if (!session || coli_segment_stream_read(read_fn, read_user_data, &header,
sizeof(header), error, error_size))
return -1;
ColiSegmentStateSpan *spans = NULL;
size_t count = 0, payload_bytes;
if (inkling_segment_spans(session, header.position, &spans, &count,
error, error_size)) return -1;
if (coli_segment_spans_size(spans, count, &payload_bytes) ||
coli_segment_snapshot_header_valid(
&header, "inkling", session->engine->layer_begin,
session->engine->layer_end, session->context_tokens,
payload_bytes, error, error_size)) {
free(spans); return -1;
}
int result = coli_segment_spans_restore(
spans, count, header.payload_hash, read_fn, read_user_data,
error, error_size);
free(spans);
if (!result) session->position = header.position;
return result;
}
static const ColiSegmentAdapter inkling_segment_adapter = {
sizeof(ColiSegmentAdapter), COLI_SEGMENT_ABI_VERSION, "inkling",
inkling_segment_engine_open, inkling_segment_engine_destroy,
inkling_segment_session_create, inkling_segment_session_destroy,
inkling_segment_session_run, inkling_segment_session_snapshot,
inkling_segment_session_restore, {0}
};
int coli_inkling_segment_adapter_register(void) {
return coli_segment_adapter_register(&inkling_segment_adapter);
}
#endif /* COLI_SEGMENT_ADAPTER */
#ifdef COLI_EDGE_ADAPTER
/* ---------- engine-owned model Edge adapter --------------------------- */
typedef struct {
Model model;
Tok tokenizer;
} InklingEdgeEngine;
static void inkling_edge_wt_destroy(Wt *weight) {
if (!weight) return;
#ifdef COLI_VULKAN
ink_vk_release(weight);
free((void *)weight->vk_name);
#endif
free(weight->f); free(weight->h); free(weight->q4); free(weight->qs);
memset(weight, 0, sizeof(*weight));
}
static uint64_t inkling_edge_wt_bytes(const Wt *weight, uint64_t rows,
uint64_t columns) {
if (weight->f) return rows * columns * sizeof(float);
if (weight->h) return rows * columns * sizeof(uint16_t);
if (!weight->q4) return 0;
uint64_t scales = weight->qbits == 4 && weight->gs > 0
? rows * ((columns + (uint64_t)weight->gs - 1u) /
(uint64_t)weight->gs)
: rows;
return (uint64_t)weight->qn + scales * sizeof(float);
}
static void inkling_edge_engine_destroy(void *engine_impl) {
InklingEdgeEngine *engine = (InklingEdgeEngine *)engine_impl;
if (!engine) return;
inkling_edge_wt_destroy(&engine->model.embed);
inkling_edge_wt_destroy(&engine->model.lm_head);
free(engine->model.embed_norm); free(engine->model.final_norm);
st_destroy(&engine->model.Sq); st_destroy(&engine->model.S);
tok_free(&engine->tokenizer);
free(engine);
}
static int inkling_edge_engine_open(
void **engine_impl, ColiEdgeCapabilities *capabilities,
const ColiEdgeEngineOptions *options, char *error, size_t error_size) {
if (!engine_impl || !capabilities || !options)
return coli_edge_adapter_error(error, error_size,
"invalid Inkling Edge open");
*engine_impl = NULL;
if (options->backend_mask &&
(options->backend_mask & ~COLI_EDGE_CAP_CPU))
return coli_edge_adapter_error(error, error_size,
"Inkling Edge supports CPU only");
InklingEdgeEngine *engine = calloc(1, sizeof(*engine));
if (!engine)
return coli_edge_adapter_error(error, error_size,
"out of memory opening Inkling Edge");
Model *model = &engine->model;
load_cfg(&model->c, options->model_dir);
st_init(&model->S, options->model_dir);
char quantized_dir[4096];
snprintf(quantized_dir, sizeof(quantized_dir), "%s/dense-int4g64",
options->model_dir);
const char *disable_quantized = getenv("INK_DENSE_Q4");
struct stat quantized_stat;
if (!(disable_quantized && *disable_quantized == '0') &&
stat(quantized_dir, &quantized_stat) == 0 &&
S_ISDIR(quantized_stat.st_mode)) {
st_init(&model->Sq, quantized_dir);
model->has_q = model->Sq.n > 0;
}
model->embed = load_w(model, "model.embed_tokens.weight", 0);
const char *embed_norm = embed_norm_name(&model->S);
model->embed_norm = embed_norm ? load_t(model, embed_norm) : NULL;
model->final_norm = load_t(model, "model.norm.weight");
model->lm_head = load_w(model, "lm_head.weight", 1);
char tokenizer_path[4096];
snprintf(tokenizer_path, sizeof(tokenizer_path), "%s/tokenizer.json",
options->model_dir);
tok_load(&engine->tokenizer, tokenizer_path);
Cfg *config = &model->c;
uint64_t resident = inkling_edge_wt_bytes(
&model->embed, (uint64_t)config->vocab, (uint64_t)config->hidden);
resident += inkling_edge_wt_bytes(
&model->lm_head, (uint64_t)config->vocab, (uint64_t)config->hidden);
resident += (uint64_t)config->hidden * sizeof(float);
if (model->embed_norm)
resident += (uint64_t)config->hidden * sizeof(float);
if (options->memory_limit_bytes && resident > options->memory_limit_bytes) {
inkling_edge_engine_destroy(engine);
return coli_edge_adapter_error(error, error_size,
"Inkling Edge exceeds memory limit");
}
int bits = getenv("INK_SEGMENT_BITS")
? atoi(getenv("INK_SEGMENT_BITS")) : 0;
memset(capabilities, 0, sizeof(*capabilities));
capabilities->struct_size = sizeof(*capabilities);
capabilities->abi_version = COLI_EDGE_ABI_VERSION;
capabilities->flags = COLI_EDGE_CAP_TOKENIZE |
COLI_EDGE_CAP_DETOKENIZE |
COLI_EDGE_CAP_GREEDY | COLI_EDGE_CAP_LOGITS |
COLI_EDGE_CAP_CPU;
coli_edge_capability_string(capabilities->engine_id,
sizeof(capabilities->engine_id), "inkling");
coli_edge_capability_string(capabilities->state_schema,
sizeof(capabilities->state_schema),
"inkling/kv-ring-conv-f32-v1");
snprintf(capabilities->numeric_class,
sizeof(capabilities->numeric_class),
"inkling/expert-q%d/f32/cpu-v1", bits);
coli_edge_capability_string(capabilities->tokenizer_class,
sizeof(capabilities->tokenizer_class),
"inkling/o200k-byte-bpe-v1");
capabilities->state_dtype = COLI_EDGE_DTYPE_F32;
capabilities->state_width = (uint32_t)config->hidden;
capabilities->vocab_size = (uint32_t)config->unpad_vocab;
capabilities->max_batch_rows = 128;
capabilities->max_context_tokens = UINT32_MAX;
capabilities->num_layers = (uint32_t)config->n_layers;
capabilities->bos_token_id = -1;
capabilities->eos_token_id = config->eos;
capabilities->resident_bytes = resident;
*engine_impl = engine;
return 0;
}
static int inkling_edge_tokenize(
void *engine_impl, const char *text, size_t text_bytes,
int32_t *token_ids, size_t token_capacity, size_t *token_count,
char *error, size_t error_size) {
InklingEdgeEngine *engine = (InklingEdgeEngine *)engine_impl;
return coli_edge_tok_tokenize(&engine->tokenizer, text, text_bytes,
token_ids, token_capacity, token_count,
error, error_size);
}
static int inkling_edge_detokenize(
void *engine_impl, const int32_t *token_ids, size_t token_count,
char *text, size_t text_capacity, size_t *text_bytes,
char *error, size_t error_size) {
InklingEdgeEngine *engine = (InklingEdgeEngine *)engine_impl;
return coli_edge_tok_detokenize(&engine->tokenizer, token_ids, token_count,
text, text_capacity, text_bytes,
error, error_size);
}
static int inkling_edge_embed(void *engine_impl,
const ColiEdgeEmbedRequest *request,
char *error, size_t error_size) {
InklingEdgeEngine *engine = (InklingEdgeEngine *)engine_impl;
Cfg *config = &engine->model.c;
float *output = (float *)request->output;
for (uint32_t row = 0; row < request->rows; row++) {
int token = request->token_ids[row];
if (token < 0 || token >= config->vocab)
return coli_edge_adapter_error(error, error_size,
"Inkling token ID is out of range");
float *state = output + (size_t)row * config->hidden;
wt_row_f32(engine->model.embed, (int64_t)token * config->hidden,
state, config->hidden);
if (engine->model.embed_norm)
rmsnorm_row(state, state, engine->model.embed_norm,
config->hidden, config->eps);
}
return 0;
}
static int inkling_edge_select(void *engine_impl,
const ColiEdgeSelectRequest *request,
char *error, size_t error_size) {
InklingEdgeEngine *engine = (InklingEdgeEngine *)engine_impl;
Cfg *config = &engine->model.c;
float *normalized = falloc(config->hidden);
float *logits = falloc(config->unpad_vocab);
const float *input = (const float *)request->input;
for (uint32_t row = 0; row < request->rows; row++) {
if (request->should_cancel &&
request->should_cancel(request->cancel_user_data)) {
free(logits); free(normalized);
return coli_edge_adapter_error(error, error_size,
"Inkling Edge selection cancelled");
}
rmsnorm_row(normalized, input + (size_t)row * config->hidden,
engine->model.final_norm, config->hidden, config->eps);
for (int item = 0; item < config->hidden; item++)
normalized[item] /= config->mup;
matmul_w(logits, normalized, engine->model.lm_head,
1, config->hidden, config->unpad_vocab);
if (coli_edge_argmax(logits, (uint32_t)config->unpad_vocab,
&request->token_ids[row],
request->scores ? &request->scores[row] : NULL)) {
free(logits); free(normalized);
return coli_edge_adapter_error(error, error_size,
"Inkling Edge head failed");
}
}
free(logits); free(normalized);
return 0;
}
static int inkling_edge_logits(void *engine_impl,
const ColiEdgeLogitsRequest *request,
char *error, size_t error_size) {
InklingEdgeEngine *engine = (InklingEdgeEngine *)engine_impl;
Cfg *config = &engine->model.c;
float *normalized = falloc(config->hidden);
const float *input = (const float *)request->input;
for (uint32_t row = 0; row < request->rows; row++) {
if (request->should_cancel &&
request->should_cancel(request->cancel_user_data)) {
free(normalized);
return coli_edge_adapter_error(error, error_size,
"Inkling Edge logits cancelled");
}
rmsnorm_row(normalized, input + (size_t)row * config->hidden,
engine->model.final_norm, config->hidden, config->eps);
for (int item = 0; item < config->hidden; item++)
normalized[item] /= config->mup;
matmul_w(request->logits + (size_t)row * config->unpad_vocab,
normalized, engine->model.lm_head,
1, config->hidden, config->unpad_vocab);
}
free(normalized);
return 0;
}
static const ColiEdgeAdapter inkling_edge_adapter = {
sizeof(ColiEdgeAdapter), COLI_EDGE_ABI_VERSION, "inkling",
inkling_edge_engine_open, inkling_edge_engine_destroy,
inkling_edge_tokenize, inkling_edge_detokenize,
inkling_edge_embed, inkling_edge_select, inkling_edge_logits, {0}
};
int coli_inkling_edge_adapter_register(void) {
return coli_edge_adapter_register(&inkling_edge_adapter);
}
#endif /* COLI_EDGE_ADAPTER */