Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
/* deepseek_v41.c — DeepSeek-V4.1-Flash on a machine that cannot hold it.
*
* 552B parameters, 510 GB on disk, and the shape of it is why this engine exists:
*
* engram 203 GB two n-gram memories (384M rows x 264 B) read by hash: at most
* (max_ngram-1) * n_heads rows per layer per token, so ~13 KB of
* random reads for 40% of the model. Streaming is not a compromise
* here, it is the only sane way to serve it.
* experts 289 GB 15,360 fp4 experts of 18.8 MB; 6 of 384 routed per layer, so
* 4.5 GB per token, cached in RAM by the same LRU as every other
* family here (GLM-5.2 reads 12.7 GB per token for comparison).
* dense 18 GB attention, gates, norms, hyper-connection mixers: resident.
*
* No conversion step: the released checkpoint is already fp8 (dense, 32x32 ue8m0
* block scales) and fp4 (experts, ue8m0 per 32-column group), and the fp4 layout is
* byte-identical to the mxfp4 colibri already reads for Kimi K3, so quant.h's
* matmul_mxfp4 is the expert kernel. What the container does need is a sidecar the
* engine cannot compute (tools/prepare_dsv41.py): the engram bucket primes (sympy),
* the hash multipliers (numpy PCG64) and the compressed-token map (the tokenizer's
* normalizer). Those are tables, not weights.
*
* Mechanisms, each mirroring a named piece of the vendor's inference/model.py:
* hyper-connections hc_mult parallel residual streams, mixed through a
* doubly-stochastic matrix (Sinkhorn) — hyper_connections.h,
* shared with DeepSeek V4 and GLM-5.3-Flash
* sliding window every layer attends a ring of `window_size` raw KV, MQA:
* one KV vector per position for all heads
* compressed KV kv_source_layers pool `compress_ratio` tokens into one
* latent with a softmax gate and publish it; the layers
* between them read that same cache
* DSA indexer index_source_layers score compressed positions and keep
* index_topk of them, two-level when a candidate source is
* configured
* engram n-gram hash lookups gated into the residual stream
* MoE sqrtsoftplus scores, noaux_tc bias picks but does not
* scale, one shared expert every token pays
*
* The reference this engine is held to is tools/dsv41_ref.py (torch, CPU): the
* vendor's own forward runs on tilelang GPU kernels and cannot be an oracle here.
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <math.h>
#include <errno.h>
#include <time.h>
#include <stdarg.h>
/* Windows has no <sys/resource.h>; compat.h supplies getrusage over
* GetProcessMemoryInfo, and every other engine guards the include the same way. */
#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
#include <sys/resource.h>
#endif
#if defined(COLI_VULKAN) && !defined(_WIN32)
#include <sys/mman.h>
#include <unistd.h>
#endif
#ifdef _OPENMP
#include <omp.h>
#endif
#include "cli_args.h"
#include "compat.h"
#include "json.h"
#include "st.h"
#include "quant.h"
#include "sparse_attn.h"
#include "omp_tune.h"
#include "kv_prefix.h"
#include "pin_pool.h" /* riuso del prefisso tra turni (shared) */
#include <pthread.h> /* ehit_mark publishes the lazy HITS table under a lock */
#if defined(__AVX2__)
#include <immintrin.h>
#endif
#include "hyper_connections.h"
#include "tok.h"
#include "serve_codec.h"
#include "serve_poll.h"
#ifdef COLI_VULKAN
#include "backend_vulkan.h"
#include "vk_tier.h"
#include "route_trace.h" /* the tier's expert history (.coli_usage), kept while it is on */
#include "vk_chain.h" /* vkc_fit: how many layers the dense chain places (a partial chain) */
/* 1 once COLI_VULKAN=1 opened a device, after the weights load (a VK=1 build). */
static int g_vk_ready = 0;
/* The partial chain (deepseek_v41_chain.h, v41c_fit_now): the first g_v41_fit.n layers on
* the device, decided once before anything goes up. g_v41_partial: something the full
* chain would place stays on the CPU (layers, the head), so the per-matrix path uploads
* nothing that is not on the device already. */
static VkcFit g_v41_fit;
static int g_v41_fit_done, g_v41_partial;
static int g_v41_dho_try; /* the fit runs where the trunk may live on the device only (v41_dho_open) */
#endif
#ifndef COLI_VULKAN /* exclusive RAM/VRAM (vk_tier.h) is the Vulkan build's: no device holds an expert */
static inline int vkt_ram_first(int layer, int eid) { (void)layer; (void)eid; return 0; }
static inline void vkt_ram_gave(void) {}
#endif
#define V41_MAX_LAYERS 64
#define V41_MAX_ENGRAM 4
#define V41_MAX_NGRAM 8
#define V41_MAX_EHEADS 16
/* ---------------------------------------------------------------- config ---- */
typedef struct {
int vocab, dim, n_layers, n_heads, head_dim, rope_dim;
int q_lora, o_lora, o_groups;
int moe_inter, n_routed, n_activated, n_shared;
int norm_topk_prob;
float route_scale, gate_temp, swiglu_limit, norm_eps;
int window;
int compress_ratio[V41_MAX_LAYERS];
int kv_source[V41_MAX_LAYERS]; /* 1 = this layer owns a compressed KV cache */
int index_source[V41_MAX_LAYERS]; /* 1 = this layer runs its own indexer */
int index_owner[V41_MAX_LAYERS]; /* which kv_source layer's caches it reads */
float compress_rope_theta, rope_theta, rope_factor;
int original_seq_len, beta_fast, beta_slow;
int index_n_heads, index_head_dim, index_topk;
int candidate_source, candidate_topk_blocks, candidate_block_size;
int hc_mult, hc_iters;
float hc_eps;
int max_positions;
/* engram */
int n_engram, engram_layer[V41_MAX_ENGRAM];
int64_t engram_rows[V41_MAX_ENGRAM];
int engram_max_ngram, engram_heads, engram_head_dim, engram_pad_id;
int engram_cols; /* (max_ngram - 1) * heads */
/* DSpark: the MTP draft head. n_mtp == 0 means the checkpoint carries none. */
int n_mtp, spec_block, spec_noise, spec_targets[8], n_spec_targets;
int markov_rank, spec_routed, spec_activated;
/* vision (VL). vision_layers == 0 means a text-only container. */
int vision_layers, vision_dim, vision_heads, vision_inter, vision_patch;
int vision_ratio, vision_max_tokens, image_token_id;
float vision_rope_theta;
} Cfg;
static double jnum(jval *o, const char *key, double fallback) {
jval *v = o ? json_get(o, key) : NULL;
return (v && v->t == J_NUM) ? v->num : fallback;
}
static int jints(jval *o, const char *key, int *out, int cap) {
jval *v = o ? json_get(o, key) : NULL;
if (!v || v->t != J_ARR) return 0;
int n = v->len < cap ? v->len : cap;
for (int i = 0; i < n; i++) out[i] = (int)v->kids[i]->num;
return n;
}
/* The released config nests everything the text model needs under `text_config`;
* the tiny fixture does the same, so one reader serves both. */
static void cfg_load(Cfg *c, const char *snap) {
char path[1024];
snprintf(path, sizeof(path), "%s/config.json", snap);
FILE *f = fopen(path, "rb");
if (!f) { fprintf(stderr, "[cfg] %s: %s\n", path, strerror(errno)); exit(1); }
fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET);
char *text = malloc((size_t)size + 1);
if (!text || fread(text, 1, (size_t)size, f) != (size_t)size) {
fprintf(stderr, "[cfg] short read on %s\n", path); exit(1); }
text[size] = 0; fclose(f);
char *arena = NULL;
jval *root = json_parse(text, &arena);
if (!root) { fprintf(stderr, "[cfg] %s is not valid JSON\n", path); exit(1); }
jval *t = json_get(root, "text_config");
if (!t) t = root; /* a flat config is accepted: the fields are the same */
memset(c, 0, sizeof(*c));
c->vocab = (int)jnum(t, "vocab_size", 0);
c->dim = (int)jnum(t, "dim", jnum(t, "hidden_size", 0));
c->n_layers = (int)jnum(t, "n_layers", jnum(t, "num_hidden_layers", 0));
c->n_heads = (int)jnum(t, "n_heads", jnum(t, "num_attention_heads", 0));
c->head_dim = (int)jnum(t, "head_dim", 0);
c->rope_dim = (int)jnum(t, "rope_head_dim", jnum(t, "qk_rope_head_dim", 0));
c->q_lora = (int)jnum(t, "q_lora_rank", 0);
c->o_lora = (int)jnum(t, "o_lora_rank", 0);
c->o_groups = (int)jnum(t, "o_groups", 1);
c->moe_inter = (int)jnum(t, "moe_inter_dim", jnum(t, "moe_intermediate_size", 0));
c->n_routed = (int)jnum(t, "n_routed_experts", 0);
c->n_activated= (int)jnum(t, "n_activated_experts", jnum(t, "num_experts_per_tok", 0));
c->n_shared = (int)jnum(t, "n_shared_experts", 1);
c->norm_topk_prob = (int)jnum(t, "norm_topk_prob", 1);
c->route_scale= (float)jnum(t, "route_scale", jnum(t, "routed_scaling_factor", 1.0));
c->gate_temp = (float)jnum(t, "gate_temp", 1.0);
c->swiglu_limit = (float)jnum(t, "swiglu_limit", 0.0);
c->norm_eps = (float)jnum(t, "norm_eps", jnum(t, "rms_norm_eps", 1e-6));
c->window = (int)jnum(t, "window_size", jnum(t, "sliding_window", 128));
c->compress_rope_theta = (float)jnum(t, "compress_rope_theta", 40000.0);
c->rope_theta = (float)jnum(t, "rope_theta", 10000.0);
c->rope_factor= (float)jnum(t, "rope_factor", 1.0);
c->original_seq_len = (int)jnum(t, "original_seq_len", 0);
c->beta_fast = (int)jnum(t, "beta_fast", 32);
c->beta_slow = (int)jnum(t, "beta_slow", 1);
c->index_n_heads = (int)jnum(t, "index_n_heads", 0);
c->index_head_dim= (int)jnum(t, "index_head_dim", 0);
c->index_topk = (int)jnum(t, "index_topk", 0);
c->candidate_source = (int)jnum(t, "candidate_source_layer",
jnum(t, "candidate_source_layer_id", -1));
c->candidate_topk_blocks = (int)jnum(t, "candidate_topk_blocks", 0);
c->candidate_block_size = (int)jnum(t, "candidate_block_size", 0);
c->hc_mult = (int)jnum(t, "hc_mult", 1);
c->hc_iters = (int)jnum(t, "hc_sinkhorn_iters", 20);
c->hc_eps = (float)jnum(t, "hc_eps", 1e-6);
c->max_positions = (int)jnum(t, "max_seq_len", jnum(t, "max_position_embeddings", 4096));
/* CTX caps the context, and with it every buffer sized from it: the compressed
* KV and index keys of the four source layers, and both rope tables. Every
* other engine in the registry reads its own context variable (CTX, GLM53_MAXT,
* CTX_MAX, K3_MAXT, Q36_MAXT, Q38_MAXT); this one read none, so `coli --ctx`
* set CTX for the planner and the engine ignored it. The planner sizes those
* buffers from the context it was asked for, which at the default 4096 is
* 0.02 GiB against the 6.25 GiB the engine allocated for the checkpoint's
* 1,048,576 -- 6.23 GiB of unbudgeted allocation, and a machine planned to the
* number would not have it. Keeping CTX <= max_positions also means the
* checkpoint's own ceiling still wins: this can only ask for less. */
const char *ctx_env = getenv("CTX");
if (ctx_env && *ctx_env) {
int ctx = atoi(ctx_env);
if (ctx >= 2 && ctx < c->max_positions) c->max_positions = ctx;
else if (ctx >= 2)
fprintf(stderr, "[v41] CTX=%d is at or above the checkpoint's own ceiling "
"of %d positions; keeping the ceiling\n", ctx, c->max_positions);
else
fprintf(stderr, "[v41] CTX=%d is below the 2-position minimum; ignored\n", ctx);
}
/* YaRN, as the vendor spells it: rope_scaling.factor when present */
jval *scaling = json_get(t, "rope_scaling");
if (scaling && scaling->t == J_OBJ) {
c->rope_factor = (float)jnum(scaling, "factor", c->rope_factor);
c->beta_fast = (int)jnum(scaling, "beta_fast", c->beta_fast);
c->beta_slow = (int)jnum(scaling, "beta_slow", c->beta_slow);
c->original_seq_len = (int)jnum(scaling, "original_max_position_embeddings",
c->original_seq_len);
}
if (c->n_layers < 1 || c->n_layers > V41_MAX_LAYERS) {
fprintf(stderr, "[cfg] n_layers %d out of range\n", c->n_layers); exit(1); }
/* The rope turns the last rope_head_dim floats of every attention row (head_dim long)
* and of every index query and key (index_head_dim long): it must fit both, in pairs
* (the release: 64 of 512 and of 128). */
if (c->head_dim < 1 || c->rope_dim < 0 || (c->rope_dim & 1) || c->rope_dim > c->head_dim ||
c->index_head_dim < 0 || (c->index_head_dim > 0 && c->rope_dim > c->index_head_dim)) {
fprintf(stderr, "[cfg] rope_head_dim %d does not fit head_dim %d and index_head_dim %d\n",
c->rope_dim, c->head_dim, c->index_head_dim); exit(1); }
int ratios[V41_MAX_LAYERS] = {0};
int nr = jints(t, "compress_ratios", ratios, V41_MAX_LAYERS);
if (nr < c->n_layers) {
fprintf(stderr, "[cfg] compress_ratios has %d entries for %d layers\n",
nr, c->n_layers); exit(1); }
for (int i = 0; i < c->n_layers; i++) c->compress_ratio[i] = ratios[i];
int src[V41_MAX_LAYERS];
int n = jints(t, "kv_source_layers", src, V41_MAX_LAYERS);
if (!n) n = jints(t, "kv_source_layer_ids", src, V41_MAX_LAYERS);
for (int i = 0; i < n; i++) if (src[i] >= 0 && src[i] < c->n_layers) c->kv_source[src[i]] = 1;
n = jints(t, "index_source_layers", src, V41_MAX_LAYERS);
if (!n) n = jints(t, "index_source_layer_ids", src, V41_MAX_LAYERS);
for (int i = 0; i < n; i++) if (src[i] >= 0 && src[i] < c->n_layers) c->index_source[src[i]] = 1;
/* Who owns the caches a layer reads: the nearest kv_source at or before it.
* The vendor keeps this implicit in a module-level object that the source
* writes and everyone after reads; making it explicit here means a layer can
* be served without replaying the ones before it. */
int owner = -1;
for (int i = 0; i < c->n_layers; i++) {
if (c->kv_source[i]) owner = i;
c->index_owner[i] = owner;
}
/* DSpark. The released config spells the activated count two ways depending on
* which file it comes from (config.json says dspark_num_experts_per_tok,
* inference/config.json says dspark_n_activated_experts); read either. */
c->n_mtp = (int)jnum(t, "n_mtp_layers", 0);
c->spec_block = (int)jnum(t, "dspark_block_size", 0);
c->spec_noise = (int)jnum(t, "dspark_noise_token_id", 0);
c->n_spec_targets = jints(t, "dspark_target_layer_ids", c->spec_targets, 8);
c->markov_rank = (int)jnum(t, "dspark_markov_rank", 0);
c->spec_routed = (int)jnum(t, "dspark_n_routed_experts", c->n_routed);
c->spec_activated = (int)jnum(t, "dspark_n_activated_experts",
jnum(t, "dspark_num_experts_per_tok", c->n_activated));
if (c->spec_block <= 0 || c->n_spec_targets <= 0) c->n_mtp = 0;
jval *vision = json_get(root, "vision_config");
if (vision && vision->t == J_OBJ) {
c->vision_layers = (int)jnum(vision, "num_hidden_layers", 0);
c->vision_dim = (int)jnum(vision, "hidden_size", 0);
c->vision_heads = (int)jnum(vision, "num_attention_heads", 0);
c->vision_inter = (int)jnum(vision, "intermediate_size", 0);
c->vision_patch = (int)jnum(vision, "patch_size", 0);
c->vision_ratio = (int)jnum(vision, "downsample_ratio", 1);
c->vision_max_tokens = (int)jnum(vision, "max_image_tokens", 0);
c->vision_rope_theta = (float)jnum(vision, "rope_theta", 10000.0);
}
c->image_token_id = (int)jnum(root, "image_token_id", -1);
json_free(root); free(arena); free(text);
}
/* ------------------------------------------------------- weights in RAM ---- */
/* The dense trunk stays in the checkpoint's own formats: dequantizing fp8 to f32 at
* load would turn 18 GB into 72 GB, which is the whole point of not doing it. Each
* kernel below decodes as it multiplies. */
typedef struct { uint8_t *q; uint8_t *s; int O, I; } W8; /* e4m3 + 32x32 ue8m0 */
typedef struct { uint16_t *w; int O, I; } WB; /* bf16 */
typedef struct { float *w; int64_t n; } WF; /* f32 */
#define FP8_TILE 32
/* Per-turn accounting on stderr. Off by default because `coli chat` shows the
* engine's stderr right next to the answer; V41_STATS=1 turns it on, which is
* what the benchmarks and the server scripts use. */
static int v41_stats(void) {
static int on = -1;
if (on < 0) on = getenv("V41_STATS") ? atoi(getenv("V41_STATS")) : 0;
return on;
}
/* V41_TRACE=1 prints a checksum of the tensors the reference prints too, so a
* divergence is located by diffing two columns rather than by reading two forwards. */
static int g_trace = 0;
static void trace(const char *name, int layer, const float *v, int n) {
if (!g_trace) return;
double sum = 0.0;
for (int i = 0; i < n; i++) sum += v[i];
fprintf(stderr, "T %-10s L%-2d n=%-5d sum=%+.6f [%+.6f %+.6f %+.6f]\n",
name, layer, n, sum, v[0], n > 1 ? v[1] : 0.0f, n > 2 ? v[2] : 0.0f);
}
static double rss_gb(void) {
struct rusage usage;
getrusage(RUSAGE_SELF, &usage);
#ifdef __APPLE__
return usage.ru_maxrss / 1e9;
#else
return usage.ru_maxrss / 1e6;
#endif
}
static double now_s(void) {
struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t);
return t.tv_sec + t.tv_nsec * 1e-9;
}
static void *xmalloc(size_t bytes, const char *what) {
void *p = malloc(bytes ? bytes : 1);
if (!p) { fprintf(stderr, "OOM allocating %s (%zu bytes)\n", what, bytes); exit(1); }
return p;
}
static void *xcalloc(size_t count, size_t size, const char *what) {
void *p = calloc(count ? count : 1, size ? size : 1);
if (!p) { fprintf(stderr, "OOM allocating %s (%zu x %zu bytes)\n", what, count, size); exit(1); }
return p;
}
#ifdef COLI_VULKAN
/* ---- dense weights on the device only (COLI_VK_DENSE_HOST) -------------------------
* With the trunk on the device (the chain, or COLI_VK_DENSE) and this mode on (decided
* by v41_dho_open in model_load, before the layers are read), each trunk matrix (the
* fp8 rows of every W8 of a layer, the compressor's, indexer's and engram's projections;
* not the router, the head, the embedding, the norms, DSpark's stages or the vision
* tower) is read into an anonymous mapping of its own, uploaded as every lookup will ask
* for it, and its pages given back with the address kept: the device copies are found
* by that address (vk_entry), and a matrix the CPU needs again (a lost device, a step
* the device declines) is read back from disk at it (v41_dho_host), and stays. */
typedef struct {
unsigned char *base; size_t bytes;
char name[256];
int kind, O, I; /* 8: the e4m3 rows of a W8 (tiles below), 16: a WB */
const uint8_t *tiles;
int placed, gone;
int layer; /* the layer it belongs to (its name's), -1 none */
} V41Home;
static V41Home *g_v41_home;
static size_t g_v41_home_n, g_v41_home_cap;
static int g_v41_dho, g_v41_map, g_vk_opened;
/* A partial chain: only the first g_v41_dho_layers layers' matrices go to the device only
* (the chain's layers); the others load as without Vulkan. g_v41_load_layer: the layer
* model_load is reading. */
static int g_v41_dho_layers = V41_MAX_LAYERS, g_v41_load_layer = -1;
static shards *g_v41_shards;
static pthread_mutex_t g_v41_home_mx = PTHREAD_MUTEX_INITIALIZER;
static size_t v41_map_len(size_t bytes) {
#ifdef _WIN32
SYSTEM_INFO si; GetSystemInfo(&si);
size_t page = si.dwAllocationGranularity ? si.dwAllocationGranularity : 65536;
#else
long ps = sysconf(_SC_PAGESIZE);
size_t page = ps > 0 ? (size_t)ps : 4096;
#endif
return ((bytes ? bytes : 1) + page - 1) / page * page;
}
/* pages back with the range kept (and unreadable), and back again zeroed */
static int v41_map_drop(void *p, size_t bytes) {
#ifdef _WIN32
return VirtualFree(p, v41_map_len(bytes), MEM_DECOMMIT) ? 0 : -1;
#else
int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED;
#ifdef MAP_NORESERVE
flags |= MAP_NORESERVE;
#endif
return mmap(p, v41_map_len(bytes), PROT_NONE, flags, -1, 0) == MAP_FAILED ? -1 : 0;
#endif
}
static int v41_map_restore(void *p, size_t bytes) {
#ifdef _WIN32
return VirtualAlloc(p, v41_map_len(bytes), MEM_COMMIT, PAGE_READWRITE) == p ? 0 : -1;
#else
return mprotect(p, v41_map_len(bytes), PROT_READ | PROT_WRITE);
#endif
}
static void *v41_home_alloc(size_t bytes, const char *name) {
#ifdef _WIN32
void *p = VirtualAlloc(NULL, v41_map_len(bytes), MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
if (!p) { fprintf(stderr, "OOM mapping %s (%zu bytes)\n", name, bytes); exit(1); }
#else
void *p = mmap(NULL, v41_map_len(bytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) { fprintf(stderr, "OOM mapping %s (%zu bytes)\n", name, bytes); exit(1); }
#endif
if (g_v41_home_n == g_v41_home_cap) {
g_v41_home_cap = g_v41_home_cap ? 2 * g_v41_home_cap : 256;
g_v41_home = realloc(g_v41_home, g_v41_home_cap * sizeof(*g_v41_home));
if (!g_v41_home) { fprintf(stderr, "OOM dense weights table\n"); exit(1); }
}
V41Home *h = &g_v41_home[g_v41_home_n++];
memset(h, 0, sizeof *h);
h->base = p; h->bytes = bytes;
snprintf(h->name, sizeof h->name, "%s", name);
if (sscanf(name, "layers.%d.", &h->layer) != 1) h->layer = -1;
return p;
}
static V41Home *v41_home_of(const void *data) {
uintptr_t q = (uintptr_t)data;
for (size_t i = 0; i < g_v41_home_n; i++)
if (q >= (uintptr_t)g_v41_home[i].base && q - (uintptr_t)g_v41_home[i].base < g_v41_home[i].bytes)
return &g_v41_home[i];
return NULL;
}
/* The CPU is about to read the matrix that holds `data`: read back what the device held
* alone, at its own address. */
static void v41_dho_host(const void *data) {
if (!g_v41_dho || !data) return;
pthread_mutex_lock(&g_v41_home_mx);
V41Home *h = v41_home_of(data);
if (h && h->gone) {
if (v41_map_restore(h->base, h->bytes)) {
fprintf(stderr, "[VK] deepseek_v41: %s could not come back into RAM\n", h->name); exit(1);
}
st_read_raw_cap(g_v41_shards, h->name, h->base, (int64_t)h->bytes, 0);
h->gone = 0;
coli_vk_dense_host_reloaded(h->bytes);
}
pthread_mutex_unlock(&g_v41_home_mx);
}
#endif
static void *v41_weight_alloc(size_t bytes, const char *name) {
#ifdef COLI_VULKAN
if (g_v41_map) return v41_home_alloc(bytes, name);
#endif
return xmalloc(bytes, name);
}
static void w8_load(shards *S, W8 *w, const char *name, int O, int I) {
char scale_name[512];
snprintf(scale_name, sizeof(scale_name), "%.*s.scale",
(int)(strlen(name) - strlen(".weight")), name);
st_tensor *t = st_find(S, name);
if (!t) { fprintf(stderr, "%s: missing\n", name); exit(1); }
if (t->nbytes != (int64_t)O * I) {
fprintf(stderr, "%s: %lld bytes, expected %lld for [%d, %d] fp8\n",
name, (long long)t->nbytes, (long long)O * I, O, I); exit(1); }
int tiles = ((O + FP8_TILE - 1) / FP8_TILE) * ((I + FP8_TILE - 1) / FP8_TILE);
w->O = O; w->I = I;
w->q = v41_weight_alloc((size_t)O * I, name);
w->s = xmalloc((size_t)tiles, scale_name);
st_read_raw_cap(S, name, w->q, (int64_t)O * I, 0);
st_read_raw_cap(S, scale_name, w->s, tiles, 0);
#ifdef COLI_VULKAN
if (g_v41_map) { V41Home *h = &g_v41_home[g_v41_home_n - 1]; h->kind = 8; h->O = O; h->I = I; h->tiles = w->s; }
#endif
}
static void wb_load(shards *S, WB *w, const char *name, int O, int I) {
st_tensor *t = st_find(S, name);
if (!t) { fprintf(stderr, "%s: missing\n", name); exit(1); }
if (t->nbytes != (int64_t)O * I * 2) {
fprintf(stderr, "%s: %lld bytes, expected %lld for [%d, %d] bf16\n",
name, (long long)t->nbytes, (long long)O * I * 2, O, I); exit(1); }
w->O = O; w->I = I;
w->w = v41_weight_alloc((size_t)O * I * 2, name);
st_read_raw_cap(S, name, w->w, (int64_t)O * I * 2, 0);
#ifdef COLI_VULKAN
if (g_v41_map) { V41Home *h = &g_v41_home[g_v41_home_n - 1]; h->kind = 16; h->O = O; h->I = I; }
#endif
}
static void wf_load(shards *S, WF *w, const char *name, int64_t n) {
w->n = n;
w->w = xmalloc((size_t)n * sizeof(float), name);
/* st_read_f32 widens bf16/f16 as well, so a checkpoint that stores one of these
* small tensors in bf16 rather than f32 still loads.
* SEC: capped, because `n` comes from config.json and the element count from the
* file. Uncapped, a tensor longer than `n` was copied over the heap first and
* refused second, by the count check below, after the damage. */
if (st_read_f32_cap(S, name, w->w, n, 0) != n) {
fprintf(stderr, "%s: expected %lld floats\n", name, (long long)n); exit(1); }
}
/* 2^(byte - 127) without a call: ue8m0 is a bare exponent, so the byte lands straight
* in an IEEE exponent field. byte 0 and 255 (denormal / inf) never occur in a real
* checkpoint and decode to +0 and +inf here, matching quant.h's mxfp4 path. */
static inline float ue8m0(uint8_t byte) {
union { uint32_t u; float f; } value;
value.u = (uint32_t)byte << 23;
return value.f;
}
#ifdef COLI_VULKAN
/* The resident trunk on the Vulkan device (COLI_VULKAN=1 in a VK=1 build), in the
* checkpoint's own formats: a W8 goes up as fmt 12, e4m3 with one f32 scale per 32
* inputs, each 32x32 tile's ue8m0 written out for its 32 rows; a WB as fmt 11, bf16.
* The CPU kernels multiply the same f32 activations, with no rounding of their own,
* so the device computes the same products and only sums them in another order.
* The routed experts stay on the CPU: they come from disk into an LRU slot and leave
* it again.
*
* The device copy is found by the weight pointer rather than kept in the matrix:
* wo_a's per-group blocks are W8 views built on the stack at every call, and a field
* there would upload them again each time. Every W8 and WB this engine multiplies
* lives as long as the model, so a pointer names the same bytes for the whole run.
* One command buffer: calls come from the thread that opened the device and never
* from inside an OpenMP region; anything else stays on the CPU. */
typedef struct { const void *data; int fmt, O, I, refused; ColiVkTensor *t; int dho; } VkEntry; /* dho: placed, the device holds it alone */
static VkEntry *g_vk_map;
static size_t g_vk_cap, g_vk_used;
static pthread_t g_vk_thread;
static size_t vk_hash(const void *data, size_t cap) {
uint64_t h = (uint64_t)(uintptr_t)data * 0x9E3779B97F4A7C15ull;
return (size_t)(h >> 20) & (cap - 1);
}
static VkEntry *vk_entry(const void *data, int fmt, int O, int I) {
if ((g_vk_used + 1) * 2 > g_vk_cap) {
size_t cap = g_vk_cap ? g_vk_cap * 2 : 256;
VkEntry *map = calloc(cap, sizeof(*map));
if (!map) return NULL;
for (size_t i = 0; i < g_vk_cap; i++) {
if (!g_vk_map[i].data) continue;
size_t at = vk_hash(g_vk_map[i].data, cap);
while (map[at].data) at = (at + 1) & (cap - 1);
map[at] = g_vk_map[i];
}
free(g_vk_map);
g_vk_map = map; g_vk_cap = cap;
}
size_t at = vk_hash(data, g_vk_cap);
for (;; at = (at + 1) & (g_vk_cap - 1)) {
VkEntry *e = &g_vk_map[at];
if (!e->data) {
*e = (VkEntry){data, fmt, O, I, 0, NULL, 0};
g_vk_used++;
return e;
}
if (e->data == data && e->fmt == fmt && e->O == O && e->I == I) return e;
}
}
/* The entry of a matrix, NULL when it has none (no insertion). */
static VkEntry *vk_find(const void *data, int fmt, int O, int I) {
if (!g_vk_cap) return NULL;
for (size_t at = vk_hash(data, g_vk_cap);; at = (at + 1) & (g_vk_cap - 1)) {
VkEntry *e = &g_vk_map[at];
if (!e->data) return NULL;
if (e->data == data && e->fmt == fmt && e->O == O && e->I == I) return e;
}
}
/* y = W x for `rows` positions on the device; 0 sends the caller to its CPU kernel.
* x and y may be strided (wo_a's blocks read and write inside wider rows): the
* device wants them packed, so they are packed here. */
static int vk_mul(int fmt, const void *data, const uint8_t *tiles, int O, int I,
float *y, int ystride, const float *x, int xstride, int rows) {
if (!g_vk_ready || rows < 1 || !pthread_equal(pthread_self(), g_vk_thread)) return 0;
/* COLI_VK_DENSE=0: the CPU, except for a matrix the device holds alone */
if (!coli_vk_dense()) { VkEntry *f = vk_find(data, fmt, O, I); if (!f || !f->dho) return 0; }
#ifdef _OPENMP
if (omp_in_parallel()) return 0;
#endif
VkEntry *e = vk_entry(data, fmt, O, I);
if (!e || e->refused) return 0;
if (!e->t && g_v41_partial) return 0; /* a partial chain: what is not on the device stays on the CPU */
if (!e->t) v41_dho_host(data); /* a view the placement did not make: its rows back first */
float *scales = NULL;
if (!e->t && fmt == 12) {
int groups = (I + FP8_TILE - 1) / FP8_TILE;
scales = malloc((size_t)O * groups * sizeof(float));
if (!scales) return 0;
for (int o = 0; o < O; o++)
for (int g = 0; g < groups; g++)
scales[(size_t)o * groups + g] = ue8m0(tiles[(size_t)(o / FP8_TILE) * groups + g]);
}
int pack_x = rows > 1 && xstride != I, pack_y = rows > 1 && ystride != O;
float *xp = pack_x ? malloc((size_t)rows * I * sizeof(float)) : NULL;
float *yp = pack_y ? malloc((size_t)rows * O * sizeof(float)) : NULL;
int ok = 0;
if ((!pack_x || xp) && (!pack_y || yp)) {
for (int r = 0; pack_x && r < rows; r++)
memcpy(xp + (size_t)r * I, x + (size_t)r * xstride, (size_t)I * sizeof(float));
int had = e->t != NULL;
ok = coli_vk_matmul(&e->t, pack_y ? yp : y, pack_x ? xp : x, data, scales,
fmt, rows, I, O, fmt == 12 ? FP8_TILE : 0);
if (!ok && !had && !e->t) e->refused = 1; /* no device room: CPU from now on */
for (int r = 0; ok && pack_y && r < rows; r++)
memcpy(y + (size_t)r * ystride, yp + (size_t)r * O, (size_t)O * sizeof(float));
}
free(xp); free(yp); free(scales);
return ok;
}
#endif
/* y[O] = W [O, I] x[I], W in e4m3 with one ue8m0 scale per 32x32 tile.
*
* The dense trunk's matvec, and the reason it is worth vectorising: V4.1's
* attention projects 64 heads of 512, so wq_b alone is 32768 x 1536 per layer
* and forty layers of it are several GFLOP per token. Measured on the released
* checkpoint, the attention block was 41% of a turn's wall clock -- more than
* the expert reads from disk. A scalar byte-at-a-time decode was most of it. */
static void mv8(float *y, const W8 *w, const float *x) {
#ifdef COLI_VULKAN
if (vk_mul(12, w->q, w->s, w->O, w->I, y, w->O, x, w->I, 1)) return;
v41_dho_host(w->q);
#endif
int I = w->I, tiles_i = (I + FP8_TILE - 1) / FP8_TILE;
#pragma omp parallel for schedule(static)
for (int o = 0; o < w->O; o++) {
const uint8_t *row = w->q + (size_t)o * I;
const uint8_t *scale = w->s + (size_t)(o / FP8_TILE) * tiles_i;
float sum = 0.0f;
for (int base = 0; base < I; base += FP8_TILE) {
int width = I - base < FP8_TILE ? I - base : FP8_TILE;
float tile = ue8m0(scale[base / FP8_TILE]);
float part = 0.0f;
#if defined(__AVX2__)
if (width == FP8_TILE) {
__m256 acc = _mm256_setzero_ps();
for (int i = 0; i < FP8_TILE; i += 8)
acc = _mm256_fmadd_ps(e4m3_decode8(row + base + i),
_mm256_loadu_ps(x + base + i), acc);
part = hsum256(acc);
} else
#endif
for (int i = 0; i < width; i++) part += e4m3_decode(row[base + i]) * x[base + i];
sum += part * tile;
}
y[o] = sum;
}
}
/* mv8 and mvb for a block of positions at once.
*
* The matrix is the thing that does not fit in cache -- wq_b is 42 MB on the
* released checkpoint, and so is wo_b -- while a block of positions does. Driving
* the block through one pass of the matrix instead of one pass per position turns
* prefill's twenty-six trips over 42 MB into a single trip, which is memory
* traffic the FMA units were waiting on rather than arithmetic.
*
* Values are bit-identical to the one-at-a-time kernels, not merely close: each
* output still folds the same tiles in the same order, and the decoded weight
* being held across the block instead of re-decoded per position does not change
* what is multiplied. The block is capped so the positions stay in L2 while the
* matrix streams past. */
#define MV_ROWS_MAX 32
static int mv_block_rows(int I) {
int block = (int)((192 * 1024) / ((size_t)I * sizeof(float) + 1));
if (block > MV_ROWS_MAX) block = MV_ROWS_MAX;
return block < 1 ? 1 : block;
}
static void mv8_rows(float *y, int ystride, const W8 *w, const float *x, int xstride, int rows) {
#ifdef COLI_VULKAN
if (vk_mul(12, w->q, w->s, w->O, w->I, y, ystride, x, xstride, rows)) return;
v41_dho_host(w->q);
#endif
int I = w->I, tiles_i = (I + FP8_TILE - 1) / FP8_TILE;
int block = mv_block_rows(I);
for (int r0 = 0; r0 < rows; r0 += block) {
int nr = rows - r0 < block ? rows - r0 : block;
#pragma omp parallel for schedule(static)
for (int o = 0; o < w->O; o++) {
const uint8_t *row = w->q + (size_t)o * I;
const uint8_t *scale = w->s + (size_t)(o / FP8_TILE) * tiles_i;
float sum[MV_ROWS_MAX];
for (int r = 0; r < nr; r++) sum[r] = 0.0f;
for (int base = 0; base < I; base += FP8_TILE) {
int width = I - base < FP8_TILE ? I - base : FP8_TILE;
float tile = ue8m0(scale[base / FP8_TILE]);
#if defined(__AVX2__)
if (width == FP8_TILE) {
__m256 w0 = e4m3_decode8(row + base);
__m256 w1 = e4m3_decode8(row + base + 8);
__m256 w2 = e4m3_decode8(row + base + 16);
__m256 w3 = e4m3_decode8(row + base + 24);
for (int r = 0; r < nr; r++) {
const float *xr = x + (size_t)(r0 + r) * xstride + base;
__m256 acc = _mm256_mul_ps(w0, _mm256_loadu_ps(xr));
acc = _mm256_fmadd_ps(w1, _mm256_loadu_ps(xr + 8), acc);
acc = _mm256_fmadd_ps(w2, _mm256_loadu_ps(xr + 16), acc);
acc = _mm256_fmadd_ps(w3, _mm256_loadu_ps(xr + 24), acc);
sum[r] += hsum256(acc) * tile;
}
continue;
}
#endif
for (int r = 0; r < nr; r++) {
const float *xr = x + (size_t)(r0 + r) * xstride + base;
float part = 0.0f;
for (int i = 0; i < width; i++) part += e4m3_decode(row[base + i]) * xr[i];
sum[r] += part * tile;
}
}
for (int r = 0; r < nr; r++) y[(size_t)(r0 + r) * ystride + o] = sum[r];
}
}
}
static void mvb_cpu(float *y, const WB *w, const float *x) {
int I = w->I;
#pragma omp parallel for schedule(static)
for (int o = 0; o < w->O; o++) {
const uint16_t *row = w->w + (size_t)o * I;
float sum = 0.0f;
int i = 0;
#if defined(__AVX2__)
/* The lm head is [vocab, dim] in bf16: 129280 x 5120 is 662 million
* multiply-adds for one token, and it runs once per position. */
__m256 acc = _mm256_setzero_ps();
for (; i + 8 <= I; i += 8)
acc = _mm256_fmadd_ps(bf16_decode8(row + i), _mm256_loadu_ps(x + i), acc);
sum = hsum256(acc);
#endif
for (; i < I; i++) sum += bf16_to_f32(row[i]) * x[i];
y[o] = sum;
}
}
static void mvb(float *y, const WB *w, const float *x) {
#ifdef COLI_VULKAN
if (vk_mul(11, w->w, NULL, w->O, w->I, y, w->O, x, w->I, 1)) return;
v41_dho_host(w->w);
#endif
mvb_cpu(y, w, x);
}
/* mvb for `rows` positions: one call on the device (an S-row GEMM instead of a GEMV
* per position), the one-position kernel row by row on the CPU, so the CPU bits are
* mvb's. A prefill site that projected token by token calls this once instead. */
static void mvb_rows(float *y, int ystride, const WB *w, const float *x, int xstride, int rows) {
#ifdef COLI_VULKAN
if (vk_mul(11, w->w, NULL, w->O, w->I, y, ystride, x, xstride, rows)) return;
v41_dho_host(w->w);
#endif
for (int r = 0; r < rows; r++) mvb_cpu(y + (size_t)r * ystride, w, x + (size_t)r * xstride);
}
/* ------------------------------------------------------------- engram ------ */
/* One n-gram memory. The table itself never enters RAM: `lookup` reads the row it
* needs (head_dim bytes of e4m3 plus head_dim/32 scale bytes) and keeps it in a small
* LRU, because n-gram traffic is Zipfian -- the common 2-grams repeat constantly. */
typedef struct {
int fd_w, fd_s;
int64_t off_w, off_s, rows;
/* row cache: open addressing on the row id, LRU by a clock */
int cap, mask;
int64_t *key; /* row id, -1 for empty */
uint64_t *used;
float *value; /* cap x head_dim */
uint64_t clock, hits, misses;
} EngramTable;
typedef struct {
int active;
int max_ngram, heads, head_dim, cols, pad_id;
int64_t primes[V41_MAX_ENGRAM][V41_MAX_NGRAM][V41_MAX_EHEADS];
int64_t offsets[V41_MAX_ENGRAM][V41_MAX_NGRAM * V41_MAX_EHEADS];
int64_t multipliers[V41_MAX_ENGRAM][V41_MAX_NGRAM];
int layer_of[V41_MAX_ENGRAM];
int n_layers;
int32_t *token_map;
int token_map_len;
EngramTable table[V41_MAX_ENGRAM];
/* compressed ids of every position seen so far: the n-gram of a position looks
* back over these, and an image position is DEAD (-1) so no n-gram spans it */
int32_t *history;
int history_len, history_cap;
} Engram;
#define ENGRAM_DEAD (-1)
static void engram_load_sidecar(Engram *e, const char *snap) {
char path[1024];
snprintf(path, sizeof(path), "%s/dsv41_engram.json", snap);
FILE *f = fopen(path, "rb");
if (!f) { e->active = 0; return; } /* a text-only container without engram is legal */
fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET);
char *text = xmalloc((size_t)size + 1, "engram sidecar");
if (fread(text, 1, (size_t)size, f) != (size_t)size) {
fprintf(stderr, "[engram] short read on %s\n", path); exit(1); }
text[size] = 0; fclose(f);
char *arena = NULL;
jval *root = json_parse(text, &arena);
if (!root) { fprintf(stderr, "[engram] %s is not valid JSON\n", path); exit(1); }
e->max_ngram = (int)jnum(root, "max_ngram_size", 0);
e->heads = (int)jnum(root, "n_heads", 0);
e->head_dim = (int)jnum(root, "head_dim", 0);
e->pad_id = (int)jnum(root, "pad_id", 0);
e->cols = (e->max_ngram - 1) * e->heads;
if (e->max_ngram > V41_MAX_NGRAM || e->heads > V41_MAX_EHEADS) {
fprintf(stderr, "[engram] layout %dx%d exceeds the compiled maximum\n",
e->max_ngram, e->heads); exit(1); }
jval *ids = json_get(root, "layer_ids");
e->n_layers = ids && ids->t == J_ARR ? ids->len : 0;
if (e->n_layers > V41_MAX_ENGRAM) {
fprintf(stderr, "[engram] %d tables exceed the compiled maximum\n", e->n_layers); exit(1); }
for (int i = 0; i < e->n_layers; i++) e->layer_of[i] = (int)ids->kids[i]->num;
jval *primes = json_get(root, "primes");
jval *offsets = json_get(root, "offsets");
jval *multipliers = json_get(root, "multipliers");
if (!primes || !offsets || !multipliers) {
fprintf(stderr, "[engram] sidecar lacks primes/offsets/multipliers\n"); exit(1); }
/* SEC: the loop below indexes all three by table, up to the length of layer_ids --
* a different number, chosen by the same file. A shorter array was read past its
* end, and a key that is not an array has no kids at all. */
const jval *per_table[] = { primes, offsets, multipliers };
const char *per_table_name[] = { "primes", "offsets", "multipliers" };
for (int k = 0; k < 3; k++)
if (per_table[k]->t != J_ARR || per_table[k]->len < e->n_layers) {
fprintf(stderr, "[engram] %s must be an array with one entry per table (%d)\n",
per_table_name[k], e->n_layers); exit(1); }
for (int layer = 0; layer < e->n_layers; layer++) {
jval *rows = primes->kids[layer];
for (int n = 0; n < rows->len && n < V41_MAX_NGRAM; n++)
for (int h = 0; h < rows->kids[n]->len && h < V41_MAX_EHEADS; h++)
e->primes[layer][n][h] = (int64_t)rows->kids[n]->kids[h]->num;
jval *off = offsets->kids[layer];
for (int i = 0; i < off->len && i < V41_MAX_NGRAM * V41_MAX_EHEADS; i++)
e->offsets[layer][i] = (int64_t)off->kids[i]->num;
jval *mult = multipliers->kids[layer];
for (int i = 0; i < mult->len && i < V41_MAX_NGRAM; i++) {
/* Decimal strings on purpose. The multipliers run to int64 / compressed
* vocabulary, which is past the 53 bits a double keeps exactly; read as a
* number, a rounded multiplier would hash every n-gram into a different --
* and perfectly valid-looking -- row. Refuse rather than round. */
jval *entry = mult->kids[i];
if (entry->t != J_STR) {
fprintf(stderr, "[engram] multipliers must be decimal strings; this "
"sidecar stores them as JSON numbers, which cannot hold "
"them exactly. Regenerate it with tools/prepare_dsv41.py\n");
exit(1);
}
errno = 0;
char *end = NULL;
long long value = strtoll(entry->str, &end, 10);
if (errno || !end || *end) {
fprintf(stderr, "[engram] multiplier %s is not an integer\n", entry->str);
exit(1);
}
e->multipliers[layer][i] = (int64_t)value;
}
}
jval *map = json_get(root, "token_map");
if (!map || map->t != J_ARR) { fprintf(stderr, "[engram] sidecar lacks token_map\n"); exit(1); }
e->token_map_len = map->len;
e->token_map = xmalloc((size_t)map->len * sizeof(int32_t), "engram token map");
for (int i = 0; i < map->len; i++) e->token_map[i] = (int32_t)map->kids[i]->num;
e->active = 1;
json_free(root); free(arena); free(text);
}
static void engram_table_open(EngramTable *t, shards *S, int layer, int head_dim, int cache_rows) {
char name[256];
snprintf(name, sizeof(name), "layers.%d.engram.embed.weight", layer);
st_tensor *w = st_find(S, name);
snprintf(name, sizeof(name), "layers.%d.engram.embed.scale", layer);
st_tensor *s = st_find(S, name);
if (!w || !s) { fprintf(stderr, "[engram] layer %d has no table\n", layer); exit(1); }
/* SEC: engram_row() reads row `id` at id * head_dim bytes into the table and at
* id * head_dim/32 into its scales. head_dim comes from the sidecar and the two
* sizes from the shards, and nothing held them together: a head_dim of 0 divided
* by zero below, one off a multiple of 32 left the tail of every cached row
* unwritten, and a scale tensor shorter than the table was read past its end. */
if (head_dim <= 0 || head_dim % 32) {
fprintf(stderr, "[engram] head_dim %d is not a positive multiple of 32\n", head_dim);
exit(1);
}
if (w->nbytes % head_dim || s->nbytes != w->nbytes / head_dim * (head_dim / 32)) {
fprintf(stderr, "[engram] layer %d: a table of %lld bytes and %lld scale bytes are "
"not whole rows of %d bytes with one scale per 32\n", layer,
(long long)w->nbytes, (long long)s->nbytes, head_dim);
exit(1);
}
t->fd_w = w->fd; t->off_w = w->off;
t->fd_s = s->fd; t->off_s = s->off;
t->rows = w->nbytes / head_dim;
int cap = 1;
while (cap < cache_rows) cap <<= 1;
t->cap = cap; t->mask = cap - 1;
t->key = xmalloc((size_t)cap * sizeof(int64_t), "engram row keys");
t->used = xmalloc((size_t)cap * sizeof(uint64_t), "engram row clocks");
t->value = xmalloc((size_t)cap * head_dim * sizeof(float), "engram row cache");
for (int i = 0; i < cap; i++) { t->key[i] = -1; t->used[i] = 0; }
t->clock = t->hits = t->misses = 0;
}
/* The row for `id`, dequantized. Reads through a direct-mapped-with-probe cache: on a
* miss the row and its scales are one pread each, 264 bytes for the released table. */
static const float *engram_row(EngramTable *t, int64_t id, int head_dim) {
int groups = head_dim / 32;
/* SEC: `id` is a hash bucket plus an offset, both from the sidecar; the row count
* is the shard's. An id past either end read whatever the file holds there, and
* -1 -- the empty-slot key -- "hit" an empty slot and returned a row never read.
* Checked before the probe for that reason. */
if (id < 0 || id >= t->rows) {
fprintf(stderr, "[engram] row %lld is outside the table (%lld rows): "
"dsv41_engram.json does not describe this checkpoint\n",
(long long)id, (long long)t->rows);
exit(1);
}
int slot = (int)((uint64_t)(id * 0x9E3779B97F4A7C15ull) >> 40) & t->mask;
int victim = slot;
for (int probe = 0; probe < 4; probe++) {
int at = (slot + probe) & t->mask;
if (t->key[at] == id) { t->used[at] = ++t->clock; t->hits++;
return t->value + (size_t)at * head_dim; }
if (t->key[at] == -1) { victim = at; goto fill; }
if (t->used[at] < t->used[victim]) victim = at;
}
fill:
t->misses++;
{
uint8_t bytes[512], scales[64];
if (head_dim > (int)sizeof(bytes) || groups > (int)sizeof(scales)) {
fprintf(stderr, "[engram] head_dim %d exceeds the row buffer\n", head_dim); exit(1); }
st_pread_full(t->fd_w, bytes, head_dim, t->off_w + id * head_dim, "engram row");
st_pread_full(t->fd_s, scales, groups, t->off_s + id * groups, "engram scale");
float *out = t->value + (size_t)victim * head_dim;
for (int g = 0; g < groups; g++) {
float scale = ue8m0(scales[g]);
for (int i = 0; i < 32; i++) out[g * 32 + i] = e4m3_decode(bytes[g * 32 + i]) * scale;
}
t->key[victim] = id; t->used[victim] = ++t->clock;
return out;
}
}
/* engram.py NgramHashState: the ids of the n-grams ending at `pos`, for one table.
* `out` receives `cols` row ids. */
static void engram_hash(const Engram *e, int table, int pos, int64_t *out) {
int64_t tokens[V41_MAX_NGRAM];
int blocked = 0;
for (int shift = 0; shift < e->max_ngram; shift++) {
int at = pos - shift;
int32_t source = at >= 0 ? e->history[at] : ENGRAM_DEAD;
if (at < 0 || source == ENGRAM_DEAD) blocked = 1;
tokens[shift] = blocked ? e->pad_id : source;
}
/* XOR the multiplied ids one lookback at a time: after step i the running value is
* the hash of the (i+1)-gram, and each lands in its own prime-sized bucket range.
* int64 throughout -- the multipliers are chosen so id * multiplier cannot wrap. */
int64_t rolling = tokens[0] * e->multipliers[table][0];
int col = 0;
for (int i = 1; i < e->max_ngram; i++) {
rolling ^= tokens[i] * e->multipliers[table][i];
for (int h = 0; h < e->heads; h++, col++) {
int64_t prime = e->primes[table][i - 1][h];
int64_t bucket = prime > 0 ? rolling % prime : 0;
if (bucket < 0) bucket += prime; /* C99 truncates toward zero */
out[col] = bucket + e->offsets[table][col];
}
}
}
/* `image` may be NULL (all text) or one byte per token: an image position is DEAD, so
* no n-gram spans it -- model.py passes ~image_mask as the engram's token_mask for the
* same reason. */
static void engram_push(Engram *e, const int *ids, int n, const uint8_t *image) {
if (e->history_len + n > e->history_cap) {
int cap = e->history_cap ? e->history_cap : 1024;
while (cap < e->history_len + n) cap *= 2;
e->history = realloc(e->history, (size_t)cap * sizeof(int32_t));
if (!e->history) { fprintf(stderr, "OOM growing the engram history\n"); exit(1); }
e->history_cap = cap;
}
for (int i = 0; i < n; i++) {
int id = ids[i];
int32_t compressed = ENGRAM_DEAD;
if (!(image && image[i]) && id >= 0 && id < e->token_map_len)
compressed = e->token_map[id];
e->history[e->history_len++] = compressed;
}
}
/* -------------------------------------------------------------- model ------ */
/* Six tensors make one expert: three packed weight matrices and their three
* ue8m0 scale sidecars. expert_read_list emits them in this order and Slot
* stores one window per part, so both sides can agree through slot_part(). */
#define V41_EXPERT_TENSORS 6
/* One routed expert, resident in a cache slot. The three matrices keep the
* checkpoint's fp4 packing and their ue8m0 group scales: quant.h's matmul_mxfp4
* consumes exactly this, so nothing is unpacked on the way in. */
typedef struct {
int eid;
uint8_t *w1, *w3, *w2; /* packed nibbles */
uint8_t *s1, *s3, *s2; /* ue8m0, one byte per 32-column group */
/* The window each of those six pointers was carved out of. A tensor sits at
* an arbitrary file offset, and the O_DIRECT path reads from the enclosing
* block boundary (st_read_range_rep takes the slack behind the destination
* for exactly this), so the bytes a slot serves begin some way into their
* window and the six pointers above are republished from these once the read
* lands. Only the read path needs the windows. */
uint8_t *base[V41_EXPERT_TENSORS];
uint64_t used;
} Slot;
/* The six buffers of a slot, in the order expert_read_list emits them. */
static uint8_t **slot_part(Slot *s, int part) {
switch (part) {
case 0: return &s->w1;
case 1: return &s->s1;
case 2: return &s->w3;
case 3: return &s->s3;
case 4: return &s->w2;
default: return &s->s2;
}
}
typedef struct { Slot *slot; int n, cap; } LCache;
typedef struct {
W8 wq_a, wq_b, wkv, wo_a, wo_b;
WF q_norm, kv_norm, attn_sink, attn_norm, ffn_norm;
WF hc_attn_fn, hc_ffn_fn, hc_attn_base, hc_ffn_base, hc_attn_scale, hc_ffn_scale;
WB comp_wkv, comp_wgate;
WF comp_norm;
W8 idx_wq_b;
WB idx_wk, idx_wproj;
WF idx_knorm;
WB gate_w;
WF gate_bias;
W8 sh_w1, sh_w3, sh_w2;
/* engram (only on engram layers) */
W8 eng_wkv;
WF eng_q, eng_k;
int engram_index; /* which table, -1 when this layer has none */
/* per-layer state */
float *window; /* [window_size, head_dim] ring of raw KV */
/* Which position each ring slot holds, -1 when it holds nothing yet. A single
* decode step needs only "slot i is position i until the ring wraps", but a
* speculative step writes several positions at once and then keeps only some of
* them: the slots of the rejected drafts still hold their keys, and the only thing
* that makes them unreadable is knowing which position they came from. */
int *window_pos;
float *ckv; /* [max_pos/ratio, head_dim] compressed KV (owners) */
float *ikey; /* [max_pos/ratio, index_head_dim] index keys (owners) */
float *cstate_kv, *cstate_score; /* compressor group still filling */
/* What a speculative step displaced, one entry per row of the batch: the ring slot
* it overwrote and the compressor slot it overwrote. Both are addressed modulo
* something (the window, the compression ratio), so a draft several positions ahead
* lands on a slot a committed position still needs -- the key one window back, or
* the earlier half of a group that has not closed yet. Rejected rows put these
* back. */
float *ring_save, *cstate_save_kv, *cstate_save_score;
int *ring_save_pos, *cstate_save_slot;
} Layer;
/* DSpark: the MTP draft head, under `mtp.*` in the checkpoint.
*
* Three stages of the same block the backbone uses, with three differences: the
* attention never compresses (it is window-only, and the window holds the MAIN
* stream's keys), the expert set is its own and smaller, and the last stage carries
* the heads that turn one hidden state into a block of guesses -- a Markov head that
* biases each position by the token before it, and a confidence head nobody has to
* trust, because the main model verifies every draft before a single one is emitted.
*
* The vendor ships the module and no loop: `forward_spec` exists, nothing calls it.
* The accept/reject loop here is therefore ours, and it is written to be exact rather
* than clever -- a token that comes out of a draft is the token the sequential decode
* would have produced, or it is not emitted. */
typedef struct {
int active; /* the stages loaded */
Layer *stage;
LCache *cache; /* one expert cache per stage, its own expert set */
W8 main_proj;
WF main_norm, norm;
WB markov_embed, markov_out, conf_proj;
uint64_t proposed, accepted, rounds;
/* Drafting is not free: a round reads the stages' experts whether or not anything
* is accepted. When acceptance collapses -- a different domain, a long verbatim
* quote, anything the head has no signal on -- the drafts are pure disk tax, so
* they pause for a while and the guard re-measures. */
int pause, min_accept, max_verify;
uint64_t window_prop, window_acc;
} Spec;
typedef struct {
Cfg c;
shards S;
Engram engram;
Layer *L;
WB embed, head;
WF norm;
LCache *cache;
int ecap; /* expert slots per layer */
uint64_t clock, hits, miss;
double t_disk, t_expert, t_attn, t_engram, t_spec;
uint64_t forwards, expert_bytes;
/* rope tables: [max_pos, rope_dim/2] cos/sin pairs, one per theta in use */
float *rope_window, *rope_compress;
int pos; /* positions consumed so far */
/* candidate mask published by the candidate source layer, one row per query */
uint8_t *candidates;
int candidate_width, candidate_rows;
/* What an index source publishes for the layers after it (model.py's
* shared_attn.topk_idxs): the compressed positions each query keeps, already
* shifted by the window offset. A layer that compresses but does not index reads
* this instead of scoring again -- that is the whole point of index_source_layers
* being a subset. Persisting across forwards is deliberate, as in the vendor. */
int *shared_topk;
int shared_topk_rows, shared_topk_width;
/* The index keys an owner last published, and the layer that owns them.
*
* model.py keeps this in one global slot and republishes it only when a layer
* COMPLETES a compression group (`if self.owns_k and latent is not None`). On a
* decode step where a ratio-2 layer's group is still filling, that layer therefore
* scores its queries against whichever cache was published last -- in the released
* config, layer 20's, whose rows are ratio-1 latents and mean something else.
*
* That is the released behaviour and every published number for this model was
* produced with it, so it is what this engine reproduces by default and what the
* oracle in tools/dsv41_ref.py encodes. V41_INDEX_OWNER=1 selects the reading the
* architecture implies instead -- each layer against its own owner's keys -- which
* is a different model, not a bug fix, and is why it is not the default. */
const float *published_index_k;
int published_index_layer;
/* A speculative step runs several positions in one pass, and the slot above is
* per-STEP state in the vendor: to stay equal to the same positions decoded one at
* a time, each row has to read what was published as of its own position. These
* three hold the schedule -- which layer publishes at each sub-step, computed
* before the layers run because a layer AFTER this one still publishes BEFORE this
* one's later rows -- and the pointer the step began with. */
int *pub_layer;
int pub_rows;
const float *pub_before;
int pub_before_layer;
uint8_t **ehit; /* experts routed this turn (dashboard HITS) */
struct Vision *vision; /* the VL tower, NULL on a text-only container */
Spec spec;
/* the DSpark input the last forward produced: [rows, n_targets * dim] */
float *main_hidden;
int main_hidden_rows;
/* where the last forward started and how many of its rows survived it: a
* speculative step keeps only the prefix that verified, and the next draft has to
* seed the stages with exactly the positions that were committed */
int last_start, last_rows;
/* Whether THIS forward's rows may be rolled back, i.e. whether it is the
* speculative verify batch. The per-layer undo buffers (ring_save,
* cstate_save_*) are sized for one draft block and nothing else, because
* until prefix reuse existed a forward with many rows always started at
* position 0 and took the batched paths that write no undo at all. A
* prefill that resumes mid-sequence is many rows AND past position 0, so
* the guard cannot be `n > 1` any more: it has to be the reason the buffers
* exist. A prefill is never rolled back. */
int rollback_save;
/* What the state describes, so a turn that resends the transcript feeds only
* the new tail. Everything a turn builds here is position-indexed and
* append-only -- the window rings and their position map, the compressor's
* group slots and its partial group, the index keys, the engram history --
* so a state that stopped at position P IS the state at P. The one thing
* that is not described by a token id is an image: its placeholders carry
* the same id whatever the picture was, so consuming one taints the record
* (see kv_prefix.h). */
kv_prefix kvp;
} Model;
/* One conversation's state for a multiplexed serve (KV_SLOTS>1, serve_mux below):
* the per-layer arrays it owns and the cross-layer state attention_run reads from the
* Model (v41_seq_swap trades the two sets). */
typedef struct {
float **window, **ckv, **ikey, **cstate_kv, **cstate_score;
int **window_pos;
int pos;
kv_prefix kvp;
int32_t *history; int history_len, history_cap;
const float *published_index_k; int published_index_layer;
int *shared_topk; int shared_topk_rows, shared_topk_width;
uint8_t *candidates; int candidate_width, candidate_rows;
} V41Seq;
static int g_v41_mux_slots = 1; /* KV_SLOTS: the conversations a serve decodes at once */
/* ------------------------------------------------------------ rope --------- */
static float rope_correction_dim(float rotations, int dim, float base, int max_seq) {
return dim * logf(max_seq / (rotations * 2.0f * (float)M_PI)) / (2.0f * logf(base));
}
/* model.py precompute_freqs_cis, YaRN included, materialized as cos/sin pairs. */
static float *rope_table(int dim, int seqlen, int original_seq_len, float base,
float factor, int beta_fast, int beta_slow) {
int half = dim / 2;
float *freqs = xmalloc((size_t)half * sizeof(float), "rope freqs");
for (int i = 0; i < half; i++)
freqs[i] = 1.0f / powf(base, (float)(2 * i) / (float)dim);
if (original_seq_len > 0 && seqlen > original_seq_len) {
float low = floorf(rope_correction_dim((float)beta_fast, dim, base, original_seq_len));
float high = ceilf(rope_correction_dim((float)beta_slow, dim, base, original_seq_len));
if (low < 0) low = 0;
if (high > dim - 1) high = dim - 1;
if (low == high) high += 0.001f;
for (int i = 0; i < half; i++) {
float ramp = ((float)i - low) / (high - low);
if (ramp < 0) ramp = 0;
if (ramp > 1) ramp = 1;
float smooth = 1.0f - ramp;
freqs[i] = freqs[i] / factor * (1.0f - smooth) + freqs[i] * smooth;
}
}
float *table = xmalloc((size_t)seqlen * half * 2 * sizeof(float), "rope table");
for (int pos = 0; pos < seqlen; pos++)
for (int i = 0; i < half; i++) {
float angle = (float)pos * freqs[i];
table[((size_t)pos * half + i) * 2 + 0] = cosf(angle);
table[((size_t)pos * half + i) * 2 + 1] = sinf(angle);
}
free(freqs);
return table;
}
/* Rotate the trailing `rope_dim` entries of `v` in place, as pairs. `inverse` undoes
* the rotation, which the output projection needs. */
static void rope_apply(float *v, const float *table, int pos, int rope_dim, int inverse) {
int half = rope_dim / 2;
const float *row = table + (size_t)pos * half * 2;
for (int i = 0; i < half; i++) {
float cosine = row[i * 2 + 0], sine = row[i * 2 + 1];
if (inverse) sine = -sine;
float a = v[2 * i], b = v[2 * i + 1];
v[2 * i] = a * cosine - b * sine;
v[2 * i + 1] = a * sine + b * cosine;
}
}
static void rms_into(float *out, const float *x, const float *weight, int n, float eps) {
double square = 0.0;
for (int i = 0; i < n; i++) square += (double)x[i] * x[i];
float inverse = 1.0f / sqrtf((float)(square / n) + eps);
for (int i = 0; i < n; i++) out[i] = weight[i] * x[i] * inverse;
}
/* --------------------------------------------------------- expert cache ---- */
/* ---- which drive answers for an expert (multi-SSD mirror) ----------------
* An expert is what this engine streams: 88.5 GB of them for a 26-token turn on
* the released checkpoint, so where those bytes come from is most of the engine.
* Two things move them that the sibling engines already have and this one did
* not:
*
* - COLI_MODEL_MIRROR=<dir>[;<dir>...] registers read-only copies of the
* checkpoint on other drives. st_mirror_add above accepts a copy only when
* its size and safetensors header are byte-identical to the primary's, so
* every data offset matches by construction and each expert can be served
* by any replica. A partial mirror is legal and a shard it does not carry
* simply stays on the primary.
* - the split between replicas follows COLI_DISK_WEIGHTS, or is measured at
* startup with this engine's own access pattern when that is absent.
*
* Two independent drives answer in parallel; one drive split two ways answers no
* faster than itself, which is why the split is a measurement and not an
* assumption. */
#define V41_MIR_REPS (1 + ST_MAX_MIR)
static int g_mirror = 0; /* 1 = at least one replica accepted */
static int g_mir_nrep = 1; /* replicas incl. the primary */
static int g_mir_cut[V41_MIR_REPS] = {256}; /* cumulative hash cuts of 256 */
static int g_mir_epl = 384; /* experts per layer, for the hash */
static uint64_t g_mir_bytes[V41_MIR_REPS];
static uint64_t g_mir_nread[V41_MIR_REPS];
/* Replica of one expert. Deterministic: the hash depends on nothing but the
* expert's identity, so the same expert always lands on the same drive and a
* drive never caches a copy the next read will not ask it for.
*
* Over the FLAT index, not the two ids apart: deepseek_v4.c records why. An XOR
* of layer and eid spread the full 43x256 grid evenly but clustered the twelve
* or so experts a decode step actually touches onto one replica; multiplying the
* flat index by the golden-ratio constant and taking bits 16..23 inherits the
* uniformity of that index, so hot and cold subsets split alike. */
static inline int expert_route(int layer, int eid) {
if (!g_mirror) return 0;
uint32_t h = (uint32_t)(layer * g_mir_epl + eid) * 2654435761u;
int hv = (int)((h >> 16) & 255), r = 0;
while (hv >= g_mir_cut[r]) r++; /* cut[nrep-1]==256 terminates the scan */
return r;
}
/* Measure one replica's read bandwidth. Uses the engine's own pattern -- the
* depth the expert reader uses, and the O_DIRECT twin when there is one -- over
* `V41_MIR_PROBE_BLOCKS` blocks of V41_MIR_PROBE_BYTES spread across the largest
* shard the replica carries. The probe exists only to weight a split; reading
* through the page cache here would weigh the cache instead of the drive, so it
* says so when it cannot avoid it. */
#define V41_MIR_PROBE_BYTES (19 << 20)
#define V41_MIR_PROBE_BLOCKS 8
static double mirror_probe_bw(shards *S, int rep) {
int big = -1;
int64_t bsz = 0;
for (int i = 0; i < S->nfd; i++) {
int fd = st_fd_rep(S, S->fds[i], rep);
if (fd < 0) continue;
int64_t sz = lseek(fd, 0, SEEK_END);
if (sz > bsz) { bsz = sz; big = i; }
}
const int64_t blk = V41_MIR_PROBE_BYTES;
if (big < 0 || bsz < blk * (V41_MIR_PROBE_BLOCKS + 1)) return 0;
int dfd = st_direct_fd_rep(S, S->fds[big], rep);
int fd = dfd >= 0 ? dfd : st_fd_rep(S, S->fds[big], rep);
if (fd < 0) return 0;
double t0 = now_s();
int64_t total = 0;
#if defined(_OPENMP)
#pragma omp parallel for schedule(dynamic, 1) reduction(+:total)
#endif
for (int i = 0; i < V41_MIR_PROBE_BLOCKS; i++) {
void *buf = NULL;
if (posix_memalign(&buf, ST_DIRECT_ALIGN, (size_t)blk) != 0) continue;
int64_t off = (((bsz - blk) / V41_MIR_PROBE_BLOCKS) * i) & ~(int64_t)(4096 - 1);
ssize_t r = pread(fd, buf, (size_t)blk, off);
if (r > 0) total += r;
compat_aligned_free(buf);
}
double dt = now_s() - t0;
return (dt > 0 && total > 0) ? (double)total / 1e9 / dt : 0;
}
/* Register the replicas and derive the split. Runs after the shard index is
* open and before the first expert read, so nothing is already cached from the
* primary by the time the split exists. */
static void mirror_setup(shards *S, const char *model_dir, int experts_per_layer) {
if (experts_per_layer > 0) g_mir_epl = experts_per_layer;
const char *dirs = getenv("COLI_MODEL_MIRROR");
if (!dirs || !*dirs) dirs = getenv("SNAP_MIRROR");
if (!dirs || !*dirs) return;
st_mirror_reset(S);
int nrep = 1;
char buf[4096];
snprintf(buf, sizeof(buf), "%s", dirs);
for (char *p = buf; p && *p; ) {
char *sep = p;
while (*sep && *sep != ';' && *sep != ',') sep++;
int last = (*sep == 0);
*sep = 0;
while (*p == ' ') p++;
size_t len = strlen(p);
while (len > 0 && p[len - 1] == ' ') p[--len] = 0;
if (*p) {
if (model_dir && !strcmp(model_dir, p))
fprintf(stderr, "[MIRROR] %s equals the model dir — ignored\n", p);
else if (nrep >= V41_MIR_REPS)
fprintf(stderr, "[MIRROR] %s: too many mirrors (max %d) — ignored\n", p, ST_MAX_MIR);
else {
int nf = st_mirror_add(S, p);
if (nf <= 0)
fprintf(stderr, "[MIRROR] %s: no usable shard (missing or divergent "
"copy) — skipped\n", p);
else {
fprintf(stderr, "[MIRROR] %s: %d/%d shards (replica %d)\n",
p, nf, S->nfd, nrep);
nrep++;
}
}
}
p = last ? NULL : sep + 1;
}
if (nrep < 2) {
fprintf(stderr, "[MIRROR] no usable mirror — reading the primary drive only\n");
return;
}
g_mirror = 1; g_mir_nrep = nrep;
double weight[V41_MIR_REPS];
int have = 0;
const char *spec = getenv("COLI_DISK_WEIGHTS");
const char *how = "COLI_DISK_WEIGHTS";
if (spec && *spec) {
char wb[256];
snprintf(wb, sizeof(wb), "%s", spec);
int n = 0, bad = 0;
for (char *tok = strtok(wb, ", "); tok; tok = strtok(NULL, ", ")) {
double v = atof(tok);
if (v <= 0 || n >= V41_MIR_REPS) { bad = 1; break; }
weight[n++] = v;
}
if (!bad && n == nrep) have = 1;
else
fprintf(stderr, "[MIRROR] invalid COLI_DISK_WEIGHTS '%s' (want %d positive "
"comma-separated weights, e.g. 9,3) — probing instead\n", spec, nrep);
}
if (!have) {
how = "measured";
have = 1;
for (int r = 0; r < nrep; r++) {
weight[r] = mirror_probe_bw(S, r);
if (weight[r] <= 0) have = 0;
}
if (have) {
fprintf(stderr, "[MIRROR] probe:");
for (int r = 0; r < nrep; r++)
fprintf(stderr, "%s %s %.2f GB/s", r ? " |" : "",
r ? "mirror" : "primary", weight[r]);
fprintf(stderr, "\n");
} else {
for (int r = 0; r < nrep; r++) weight[r] = 1;
how = "fallback 1:1 (probe failed)";
}
}
double sum = 0;
for (int r = 0; r < nrep; r++) sum += weight[r];
int acc = 0;
double cum = 0;
for (int r = 0; r < nrep; r++) {
cum += weight[r];
int cut = (int)(256.0 * cum / sum + 0.5);
if (cut <= acc) cut = acc + 1;
if (cut > 256) cut = 256;
g_mir_cut[r] = cut;
acc = cut;
}
fprintf(stderr, "[MIRROR] %d replicas, split %s:", nrep, how);
for (int r = 0; r < nrep; r++) {
int share = (r ? g_mir_cut[r] : g_mir_cut[0]) - (r ? g_mir_cut[r - 1] : 0);
fprintf(stderr, " %s %d%%", r ? "mirror" : "primary", (share * 100 + 128) / 256);
}
fprintf(stderr, "\n");
}
/* The per-drive line ENVIRONMENT.md documents ("Per-drive byte counts are
* reported in a MIRROR: stats line"), in the shape colibri.c prints. Counters
* are cumulative; passing the values from before a turn reports that turn. On
* stderr, because stdout carries the serve protocol. */
static void mirror_report(const uint64_t *bytes0, const uint64_t *reads0) {
if (!g_mirror) return;
double per[V41_MIR_REPS], total = 0;
for (int r = 0; r < g_mir_nrep; r++) {
uint64_t bytes = g_mir_bytes[r] - (bytes0 ? bytes0[r] : 0);
per[r] = (double)bytes / 1e9;
total += per[r];
}
fprintf(stderr, "MIRROR: primary %.2f GB (%llu reads)", per[0],
(unsigned long long)(g_mir_nread[0] - (reads0 ? reads0[0] : 0)));
for (int r = 1; r < g_mir_nrep; r++)
fprintf(stderr, " | mirror%d %.2f GB (%llu reads)", r, per[r],
(unsigned long long)(g_mir_nread[r] - (reads0 ? reads0[r] : 0)));
fprintf(stderr, " — %.0f%% of expert bytes from the mirrors\n",
total > 0 ? 100.0 * (total - per[0]) / total : 0.0);
}
/* One tensor still to fetch: which tensor, which window it lands in, and which
* replica answers for it. An expert is six of these -- three weight matrices and
* their three scale sidecars. */
typedef struct {
const st_tensor *t; /* resolved once, so a read is a pread and not a hash lookup */
Slot *slot; /* whose window, and which of the six, to republish after */
int part;
int rep;
} ExpertRead;
/* Bytes one part of an expert occupies, from the config rather than the file:
* this is the capacity the slot's window was sized for, and the reader refuses a
* tensor that declares more than that (see expert_fetch). */
static int64_t expert_part_bytes(const Cfg *c, int part) {
switch (part) {
case 0: case 2: return (int64_t)c->moe_inter * c->dim / 2; /* w1, w3 */
case 1: case 3: return (int64_t)c->moe_inter * (c->dim / 32); /* their scales */
case 4: return (int64_t)c->dim * c->moe_inter / 2; /* w2 */
default: return (int64_t)c->dim * (c->moe_inter / 32); /* its scale */
}
}
/* Fill in the six reads that make up one expert. Naming them into a list instead
* of reading them on the spot is the whole point: the list is what lets a layer's
* misses go to the device together rather than one behind another. Resolution
* happens here too, once per miss, so the read itself is a pread through the
* chosen replica's fd rather than a name lookup followed by a primary-drive one. */
static int expert_read_list(Model *m, const char *kind, int layer, int eid, Slot *s,
int rep, ExpertRead *out) {
static const struct { const char *suffix; int weight_part, scale_part; } part[3] = {
{ "w1", 0, 1 }, { "w3", 2, 3 }, { "w2", 4, 5 },
};
static const char *const leaf[2] = { "weight", "scale" };
char name[160];
int n = 0;
for (int i = 0; i < 3; i++) {
for (int f = 0; f < 2; f++) {
snprintf(name, sizeof(name), "%s.%d.ffn.experts.%d.%s.%s",
kind, layer, eid, part[i].suffix, leaf[f]);
const st_tensor *t = st_find(&m->S, name);
if (!t) st_die_missing(&m->S, name);
out[n].t = t;
out[n].slot = s;
out[n].part = f ? part[i].scale_part : part[i].weight_part;
out[n].rep = rep;
n++;
}
}
return n;
}
/* How many expert tensors to keep in flight. One at a time is what the engine did
* and it leaves most of the device on the floor: measured on the released
* checkpoint, 88.5 GB of expert reads landed at 2.40 GB/s, while the same blocks
* through the same page cache go at 6.2 GB/s with one reader and 12.6 GB/s with
* four (c/iobench.c, 5.6 MB blocks, caches dropped). A striped NVMe pair answers
* several requests at once; a single blocking pread never asks it to.
* V41_READ_DEPTH=1 restores the serial path for an A/B. */
static int expert_read_depth(void) {
static int depth = -1;
if (depth < 0) {
const char *env = getenv("V41_READ_DEPTH");
depth = env ? atoi(env) : 8;
if (depth < 1) depth = 1;
}
return depth;
}
/* O_DIRECT for the expert reads. On by default, as in the V4 engine, because of
* what an expert read is: each is read once and wanted no longer, the engine
* tells the kernel so afterwards (drop=1 -> POSIX_FADV_DONTNEED), and a buffered
* read of it therefore pays for the bytes twice -- once into the page cache and
* once out of it -- to leave the cache exactly as cold as it was. V41_DIRECT=0
* restores the buffered path, which is the arm of the A/B and the escape hatch
* on a device where the direct path turns out to be slower. */
/* COLI_KV_PREFIX: on this engine reuse is OPT-IN, which it is on no other.
*
* Everywhere else a reused prefix is the same computation as a cold prefill, so
* reuse changes the time and nothing else and can be on by default. Here it is
* not, and the reason is the vendor's, not ours: a query reads one set of index
* keys when its position is prefilled (its layer's own index owner, masked to
* the query's reach) and another when the position is decoded (whatever was
* published last). Both are load-bearing -- giving the prefill the decode
* schedule drops the tiny oracle to 7/8, giving the decode the prefill owner
* drops it to 1/8 -- so a position that was generated in an earlier turn does
* not attend the way the same text attends when prefilled cold.
*
* What that costs: a conversation resumed from its own state and the same
* conversation re-read from scratch can answer differently. The resumed state is
* the sequential one, so it is not the wrong answer; it is a different one, and
* on a 510 GB model it arrives in seconds instead of minutes. That is a trade
* the person running the engine should make, so it is asked for rather than
* assumed. A resumed PREFILL is exact (docs/deepseek-v41.md records the split
* measurements), so a prompt that only ever grows without generated text in
* between -- an agent resending a document, a tool loop -- reuses losslessly. */
static int kv_prefix_on(void){ const char *e = getenv("COLI_KV_PREFIX"); return e && *e != '0'; }
static int expert_direct(void) {
static int on = -1;
if (on < 0) on = getenv("V41_DIRECT") ? atoi(getenv("V41_DIRECT")) : 1;
return on != 0;
}
/* Read `n` reserved experts at once. The slot ids are stamped only after every
* read has landed, so a slot never advertises an expert it does not yet hold. */
static void expert_fetch(Model *m, const char *kind, int layer,
Slot *const *slot, const int *eid, int n) {
if (n <= 0) return;
int total = 0;
ExpertRead *list = xmalloc((size_t)n * V41_EXPERT_TENSORS * sizeof(ExpertRead),
"expert read list");
for (int i = 0; i < n; i++)
total += expert_read_list(m, kind, layer, eid[i], slot[i],
expert_route(layer, eid[i]), list + total);
/* The window is sized from the config, so a container whose tensor declares
* more than that must not be read into it -- same refusal st_read_raw_cap
* applied when the read went by name. */
for (int i = 0; i < total; i++) {
int64_t cap = expert_part_bytes(&m->c, list[i].part);
if (list[i].t->nbytes < 0 || list[i].t->nbytes > cap) {
fprintf(stderr, "%s: expert tensor declares %lld bytes, the slot holds %lld — "
"refusing (untrusted container)\n", list[i].t->name,
(long long)list[i].t->nbytes, (long long)cap);
exit(1);
}
}
double started = now_s();
int threads = expert_read_depth();
if (threads > total) threads = total;
uint64_t bytes = 0;
#if defined(_OPENMP)
#pragma omp parallel for schedule(dynamic, 1) num_threads(threads) reduction(+:bytes)
#endif
for (int i = 0; i < total; i++) {
const ExpertRead *r = &list[i];
int64_t pad = r->t->off % ST_DIRECT_ALIGN;
/* The payload pointer carries the slack st_read_range_rep needs to read
* from the enclosing block boundary, so an O_DIRECT transfer lands the
* data exactly where the slot wants it and nothing is copied afterwards. */
st_read_range_rep(&m->S, r->t->fd, r->rep, r->t->off, r->t->nbytes,
r->slot->base[r->part] + pad, r->t->nbytes, 1, expert_direct(),
"pread expert");
/* Which drive actually answered. A partial mirror leaves shards on the
* primary, so the replica the route picked is not always the one that read. */
int rep = r->rep;
if (st_fd_rep(&m->S, r->t->fd, rep) < 0) rep = 0;
__atomic_fetch_add(&g_mir_bytes[rep], (uint64_t)r->t->nbytes, __ATOMIC_RELAXED);
__atomic_fetch_add(&g_mir_nread[rep], 1, __ATOMIC_RELAXED);
bytes += (uint64_t)r->t->nbytes;
}
/* Publish the payload pointers only now. A slot that advertises an expert has
* to address that expert, and until the read landed the window base was the
* only pointer that meant anything. */
for (int i = 0; i < total; i++)
*slot_part(list[i].slot, list[i].part) =
list[i].slot->base[list[i].part] + (list[i].t->off % ST_DIRECT_ALIGN);
m->expert_bytes += bytes;
m->t_disk += now_s() - started;
for (int i = 0; i < n; i++) slot[i]->eid = eid[i];
free(list);
}
/* One byte per expert: routed in this turn or not, for the dashboard's HITS line. */
static pthread_mutex_t g_ehit_mx = PTHREAD_MUTEX_INITIALIZER;
static void ehit_mark(Model *m, int layer, int eid) {
Cfg *c = &m->c;
/* Built privately and published once under a lock, the shape #1423 gave the
* other engines: a first touch from inside a parallel region otherwise
* publishes the outer array while the rows are still being filled, and a
* sibling thread dereferences a NULL row. Nothing here routes from a
* parallel region today, but the code is the same code, and the lock costs
* one atomic load on the path that matters. */
uint8_t **ehit = __atomic_load_n(&m->ehit, __ATOMIC_ACQUIRE);
if (!ehit) {
pthread_mutex_lock(&g_ehit_mx);
ehit = m->ehit;
if (!ehit) {
ehit = xmalloc((size_t)c->n_layers * sizeof(uint8_t *), "expert hit map");
for (int i = 0; i < c->n_layers; i++) {
ehit[i] = xmalloc((size_t)c->n_routed, "expert hit row");
memset(ehit[i], 0, (size_t)c->n_routed);
}
__atomic_store_n(&m->ehit, ehit, __ATOMIC_RELEASE);
}
pthread_mutex_unlock(&g_ehit_mx);
}
if (layer >= 0 && layer < c->n_layers && eid >= 0 && eid < c->n_routed) ehit[layer][eid] = 1;
}
/* `kind` is the checkpoint namespace, "layers" for the backbone and "mtp" for a
* DSpark stage: same slot shapes, same LRU, a different set of experts. Only the
* backbone's routing reaches the dashboard's grid -- the draft head has its own,
* smaller expert set, and a row of it would not line up with anything. */
/* The Vulkan tier's index of a MoE call's layer: a backbone layer's own, a DSpark
* stage's n_layers + stage (the tier's extra layers, COLI_VK_TIER_MTP; a stage the tier
* did not take answers no to everything there). */
static int v41_tier_layer(const Model *m, const char *kind, int layer) {
return !strcmp(kind, "layers") ? layer : m->c.n_layers + layer;
}
/* The slot a full cache gives up: the least recently used, and before it the least
* recently used of those whose expert the Vulkan tier holds (vkt_ram_first). */
static int v41_victim(const Model *m, const LCache *cache, const char *kind, int layer) {
int oldest = 0, dev = -1, tl = v41_tier_layer(m, kind, layer);
for (int i = 1; i < cache->n; i++)
if (cache->slot[i].used < cache->slot[oldest].used) oldest = i;
for (int i = 0; i < cache->n; i++)
if (cache->slot[i].eid >= 0 && vkt_ram_first(tl, cache->slot[i].eid) &&
(dev < 0 || cache->slot[i].used < cache->slot[dev].used)) dev = i;
if (dev >= 0) { vkt_ram_gave(); return dev; }
return oldest;
}
static Slot *expert_slot_at(Model *m, LCache *cache, const char *kind, int layer, int eid) {
if (!strcmp(kind, "layers")) ehit_mark(m, layer, eid);
for (int i = 0; i < cache->n; i++)
if (cache->slot[i].eid == eid) {
cache->slot[i].used = ++m->clock;
m->hits++;
return &cache->slot[i];
}
m->miss++;
Slot *victim;
if (cache->n < cache->cap) {
victim = &cache->slot[cache->n++];
} else {
victim = &cache->slot[v41_victim(m, cache, kind, layer)];
}
victim->used = ++m->clock;
expert_fetch(m, kind, layer, &victim, &eid, 1);
return victim;
}
/* Resolve a whole step's routed experts to slots WITHOUT reading any of them, then
* fetch the misses in one batch. Reserving first is what makes the batch possible:
* a reserved victim is stamped with the newest clock and its id cleared, so a later
* reservation in the same step neither evicts it again nor mistakes it for a hit.
*
* The argument holds only while the cache has room for the whole step (cap >= topk):
* at reservation k at most k slots have been touched, so with cap > k there is
* always an untouched one left to evict, and the untouched ones are exactly the
* ones with an older clock. A cache smaller than topk has no such slot -- the
* caller keeps the one-at-a-time path for that case. */
static void expert_slots_at(Model *m, LCache *cache, const char *kind, int layer,
const int *chosen, int topk, Slot **out) {
int miss_eid[64];
Slot *miss_slot[64];
int misses = 0;
for (int k = 0; k < topk; k++) {
int eid = chosen[k];
if (!strcmp(kind, "layers")) ehit_mark(m, layer, eid);
Slot *found = NULL;
for (int i = 0; i < cache->n; i++)
if (cache->slot[i].eid == eid) { found = &cache->slot[i]; break; }
if (found) {
found->used = ++m->clock;
m->hits++;
out[k] = found;
continue;
}
m->miss++;
Slot *victim;
if (cache->n < cache->cap) {
victim = &cache->slot[cache->n++];
} else {
victim = &cache->slot[v41_victim(m, cache, kind, layer)];
}
victim->used = ++m->clock;
victim->eid = -1; /* not this expert yet: the read is still pending */
out[k] = victim;
miss_slot[misses] = victim;
miss_eid[misses] = eid;
misses++;
}
expert_fetch(m, kind, layer, miss_slot, miss_eid, misses);
}
/* ------------------------------------------------------------- loading ----- */
typedef struct {
WF norm1, norm2, qkv_b, o_b;
WB qkv_w, o_w, w1, w2;
} VisionBlock;
typedef struct Vision {
int active;
WB proj_w; WF proj_b;
VisionBlock *block;
WF norm;
WB align_w1, align_w2;
WF align_b1, align_b2;
WF image_start, image_end, image_newline;
} Vision;
static void vision_load(Model *m, Vision *v);
static void vision_forward(Model *m, Vision *v, const float *patches, int n_h, int n_w, float *out);
static int vision_tokens(const Cfg *c, int n_h, int n_w);
static void spec_load(Model *m, int ecap);
/* One layer's expert cache. The DSpark stages route over their own, smaller expert set
* but the experts themselves are the same shape, so the slots are too. */
static void cache_init(Model *m, LCache *cache, int ecap) {
Cfg *c = &m->c;
cache->cap = ecap; cache->n = 0;
cache->slot = xmalloc((size_t)ecap * sizeof(Slot), "expert slots");
for (int k = 0; k < ecap; k++) {
Slot *s = &cache->slot[k];
s->eid = -1; s->used = 0;
/* One window per tensor, page-aligned and with ST_DIRECT_ALIGN of slack
* past its payload. The slack is not padding for its own sake: it is what
* lets st_read_range_rep turn an unaligned safetensors range into one
* aligned O_DIRECT pread, instead of bouncing every expert through a
* scratch buffer and copying it again (see the expert reader). */
for (int part = 0; part < V41_EXPERT_TENSORS; part++) {
size_t bytes = (size_t)expert_part_bytes(c, part) + ST_DIRECT_ALIGN;
void *window = NULL;
if (posix_memalign(&window, ST_DIRECT_ALIGN, bytes) != 0) {
fprintf(stderr, "OOM allocating expert window (%zu bytes)\n", bytes);
exit(1);
}
s->base[part] = window;
*slot_part(s, part) = window; /* until the first read republishes it */
}
}
}
static void attn_project_check(const Cfg *c);
#ifdef COLI_VULKAN
static int v41c_decide(Model *m); /* deepseek_v41_chain.h: COLI_VK_CHAIN, decided once */
static int v41c_fit_now(Model *m); /* deepseek_v41_chain.h: how many layers the chain places, once */
static void v41c_fit_recount(Model *m);
static int v41c_prepare(Model *m);
static int v41c_build_layer(Model *m, int i, const char **why);
static void v41c_cut_layer(Model *m, int i, const char *why);
/* What the device would hold alone: the trunk matrices model_load reads into mappings,
* of the first `layers` layers (a partial chain's; every layer otherwise). */
static size_t v41_dho_bytes(Model *m, int layers) {
const Cfg *c = &m->c;
char name[256];
size_t total = 0;
#define B(...) do { snprintf(name, sizeof name, __VA_ARGS__); st_tensor *t = st_find(&m->S, name); \
if (t && t->nbytes > 0) total += (size_t)t->nbytes; } while (0)
for (int i = 0; i < c->n_layers && i < layers; i++) {
B("layers.%d.attn.wq_a.weight", i); B("layers.%d.attn.wq_b.weight", i); B("layers.%d.attn.wkv.weight", i);
B("layers.%d.attn.wo_a.weight", i); B("layers.%d.attn.wo_b.weight", i);
B("layers.%d.ffn.shared_experts.w1.weight", i); B("layers.%d.ffn.shared_experts.w3.weight", i);
B("layers.%d.ffn.shared_experts.w2.weight", i);
if (c->kv_source[i]) {
B("layers.%d.attn.compressor.wkv.weight", i);
if (c->compress_ratio[i] > 1) B("layers.%d.attn.compressor.wgate.weight", i);
B("layers.%d.attn.indexer.wk.weight", i);
}
if (c->index_source[i]) { B("layers.%d.attn.indexer.wq_b.weight", i); B("layers.%d.attn.indexer.weights_proj.weight", i); }
}
for (int t = 0; m->engram.active && t < m->engram.n_layers; t++)
if (m->engram.layer_of[t] < layers) B("layers.%d.engram.wkv.weight", m->engram.layer_of[t]);
#undef B
return total;
}
/* The layers whose matrices go to the device (the chain's first N with a partial chain),
* and whether the dense part is on the device at all (N = 0: the chain is off and, the fit
* being partial, the per-matrix path uploads nothing either). */
static int v41_dense_layers(Model *m) { return g_v41_fit.L > 0 ? g_v41_fit.n : m->c.n_layers; }
static int v41_dense_on(Model *m, int chain) {
if (g_v41_fit.L > 0) return g_v41_fit.n > 0 || (coli_vk_dense() && !g_v41_partial);
return chain != 0 || coli_vk_dense();
}
/* Before the layers are read: the device, the chain's decision, how many layers it
* places, and whether the trunk lives on the device only. COLI_VK_DENSE_HOST=1 (kept)
* opens the device after the weights, as before. */
static void v41_dho_open(Model *m) {
const char *on = getenv("COLI_VULKAN"), *keep = getenv("COLI_VK_DENSE_HOST");
if (!on || !atoi(on) || (keep && *keep && atoi(keep) != 0)) return;
g_vk_opened = 1;
g_vk_thread = pthread_self();
g_vk_ready = coli_vk_init_env_tier("deepseek_v41", vkt_wanted() && m->c.n_routed > 0);
if (!g_vk_ready) return;
int chain = v41c_decide(m);
g_v41_dho_try = 1;
v41c_fit_now(m); /* before any upload: the first N layers (all when everything fits) */
int layers = v41_dense_layers(m);
g_v41_dho = coli_vk_dense_host_decide("deepseek_v41", v41_dense_on(m, chain), v41_dho_bytes(m, layers));
if (g_v41_fit.L > 0) coli_vk_dense_host_layers(layers, m->c.n_layers);
if (!g_v41_dho) v41c_fit_recount(m); /* the chain's own set goes up after all */
g_v41_dho_layers = layers;
g_v41_shards = &m->S;
v41c_prepare(m); /* the chain's layers built whole as they are read */
}
/* One matrix up as a lookup will ask for it, marked as held by the device alone. */
static int v41_dho_up(const void *q, const uint8_t *tiles, int fmt, int O, int I) {
VkEntry *e = vk_entry(q, fmt, O, I);
if (!e || e->refused) return 0;
if (!e->t) {
float *scales = NULL;
if (fmt == 12) {
int groups = (I + FP8_TILE - 1) / FP8_TILE;
if (!(scales = malloc((size_t)O * groups * sizeof(float)))) return 0;
for (int o = 0; o < O; o++)
for (int g = 0; g < groups; g++) scales[(size_t)o * groups + g] = ue8m0(tiles[(size_t)(o / FP8_TILE) * groups + g]);
}
int ok = coli_vk_tensor_ensure(&e->t, q, scales, fmt, I, O, fmt == 12 ? FP8_TILE : 0);
free(scales);
if (!ok) return 0;
}
e->dho = 1;
return 1;
}
/* Every device copy of the bytes [base, base + bytes) freed (a matrix and the views
* inside it, such as wo_a's groups), their entries refused: the CPU multiplies them from
* now on. After the chain's frames that may read them. */
static void v41_vk_forget_range(const void *base, size_t bytes) {
uintptr_t b = (uintptr_t)base;
for (size_t i = 0; i < g_vk_cap; i++) {
VkEntry *e = &g_vk_map[i];
if (!e->data || (uintptr_t)e->data < b || (uintptr_t)e->data - b >= bytes) continue;
if (e->t) { if (vkc_ready()) vkc_finish(); coli_vk_tensor_free(e->t); e->t = NULL; }
e->refused = 1; e->dho = 0;
}
}
/* Layers k.. off the device again (layer k did not reach it): their device copies freed
* and what the device held alone read back from disk; from here on they load and run as
* without Vulkan. */
static void v41_dho_unplace(Model *m, int k) {
for (size_t h = 0; h < g_v41_home_n; h++) {
V41Home *x = &g_v41_home[h];
if (x->layer < k) continue;
v41_vk_forget_range(x->base, x->bytes);
if (x->gone) v41_dho_host(x->base);
x->placed = 1;
}
if (g_v41_dho_layers > k) g_v41_dho_layers = k;
coli_vk_dense_host_layers(k, m->c.n_layers);
}
/* The matrices read since the last call: up, then their pages back. wo_a goes up whole
* for the chain and per output group for the per-matrix path (attention_project's
* block views), each only where that path runs. With a partial chain (the fit) a layer
* goes up whole or not at all, its chain state with it (v41c_build_layer): its pages go
* back only once all of it is on the device, so a layer that does not get there reads
* nothing back; it ends the chain before it (vkc_fit_shrink). */
static unsigned g_v41_dho_kept;
static void v41_dho_place(Model *m, int mark) { /* mark: the layer just read (its fault count noted), -1 none */
const Cfg *c = &m->c;
int chain = v41c_decide(m), fit = g_v41_fit.L > 0, failed = -1;
const char *why = "a matrix the device refused";
size_t from = g_v41_home_n;
for (size_t k = 0; k < g_v41_home_n; k++) if (!g_v41_home[k].placed) { from = k; break; }
for (size_t k = from; k < g_v41_home_n; k++) {
V41Home *h = &g_v41_home[k];
if (h->placed) continue;
h->placed = 1;
if (failed >= 0) continue;
int ok;
size_t n = strlen(h->name), tail = strlen(".attn.wo_a.weight");
if (h->kind == 8 && n > tail && !strcmp(h->name + n - tail, ".attn.wo_a.weight")) {
ok = 1;
if (chain) ok = v41_dho_up(h->base, h->tiles, 12, h->O, h->I);
int tiles_i = (h->I + FP8_TILE - 1) / FP8_TILE;
for (int g = 0; ok && chain != COLI_VK_CHAIN_ON && g < c->o_groups; g++)
ok = v41_dho_up(h->base + (size_t)g * c->o_lora * h->I,
h->tiles + (size_t)(g * c->o_lora / FP8_TILE) * tiles_i, 12, c->o_lora, h->I);
} else ok = v41_dho_up(h->base, h->tiles, h->kind == 8 ? 12 : 11, h->O, h->I);
if (fit) { if (!ok) failed = h->layer >= 0 ? h->layer : 0; continue; }
if (ok && !v41_map_drop(h->base, h->bytes)) { h->gone = 1; coli_vk_dense_host_dropped(h->bytes); }
else g_v41_dho_kept++;
}
if (!fit) return;
if (failed < 0 && mark >= 0 && mark < g_v41_fit.n && !v41c_build_layer(m, mark, &why)) failed = mark;
if (failed >= 0) { /* everything of that layer and after off the device, the chain cut before it */
v41c_cut_layer(m, failed, why);
return;
}
for (size_t k = from; k < g_v41_home_n; k++) { /* the whole batch is there: its pages back */
V41Home *h = &g_v41_home[k];
if (h->gone || h->layer >= g_v41_dho_layers) continue;
if (!v41_map_drop(h->base, h->bytes)) { h->gone = 1; coli_vk_dense_host_dropped(h->bytes); }
else g_v41_dho_kept++;
}
if (mark >= 0 && mark < g_v41_fit.n) vkc_fit_mark(&g_v41_fit, mark);
}
#define V41_DHO(call) do { g_v41_map = g_v41_dho && g_v41_load_layer < g_v41_dho_layers; call; g_v41_map = 0; } while (0)
#else
#define V41_DHO(call) call
#endif
static void model_load(Model *m, const char *snap, int ecap, int engram_cache_rows) {
Cfg *c = &m->c;
cfg_load(c, snap);
attn_project_check(c);
/* Sized with the state it describes. CTX caps max_positions (#1526), so this
* is one int per position the engine can actually hold, not per position the
* checkpoint allows. A failure here disables reuse and nothing else. */
kv_prefix_alloc(&m->kvp, c->max_positions);
/* COLI_MODEL_DIRS: the container split across drives as DISTINCT shards, so
* the 510 GB this engine streams can live on two drives of ~256 GB instead
* of one of 512 -- which is the configuration most people can actually
* build, and it needs no second copy. The mirror below is the other half of
* the same idea: that one duplicates the bytes and splits the reads, this
* one splits the bytes and reads each shard from the one drive holding it.
* They compose: a mirror dir may copy any subset of the split's shards. */
const char *extra_dirs = getenv("COLI_MODEL_DIRS");
st_init_multi(&m->S, snap, (extra_dirs && *extra_dirs) ? extra_dirs : NULL);
/* DUAL-SSD: register COLI_MODEL_MIRROR copies and settle the read split
* before the first expert read, so nothing has been pulled from the primary
* by the time the replicas exist. */
mirror_setup(&m->S, snap, c->n_routed);
engram_load_sidecar(&m->engram, snap);
#ifdef COLI_VULKAN
v41_dho_open(m); /* COLI_VK_DENSE_HOST: the trunk to the device as it is read, its host pages back */
#endif
int dim = c->dim, hd = c->head_dim, nh = c->n_heads, hc = c->hc_mult;
m->L = xmalloc((size_t)c->n_layers * sizeof(Layer), "layers");
memset(m->L, 0, (size_t)c->n_layers * sizeof(Layer));
wb_load(&m->S, &m->embed, "embed.weight", c->vocab, dim);
wb_load(&m->S, &m->head, "head.weight", c->vocab, dim);
wf_load(&m->S, &m->norm, "norm.weight", dim);
char name[256];
#define NAME(fmt, ...) (snprintf(name, sizeof(name), fmt, __VA_ARGS__), name)
for (int i = 0; i < c->n_layers; i++) {
Layer *l = &m->L[i];
l->engram_index = -1;
#ifdef COLI_VULKAN
g_v41_load_layer = i;
#endif
V41_DHO(w8_load(&m->S, &l->wq_a, NAME("layers.%d.attn.wq_a.weight", i), c->q_lora, dim));
V41_DHO(w8_load(&m->S, &l->wq_b, NAME("layers.%d.attn.wq_b.weight", i), nh * hd, c->q_lora));
V41_DHO(w8_load(&m->S, &l->wkv, NAME("layers.%d.attn.wkv.weight", i), hd, dim));
V41_DHO(w8_load(&m->S, &l->wo_a, NAME("layers.%d.attn.wo_a.weight", i),
c->o_groups * c->o_lora, nh * hd / c->o_groups));
V41_DHO(w8_load(&m->S, &l->wo_b, NAME("layers.%d.attn.wo_b.weight", i), dim, c->o_groups * c->o_lora));
wf_load(&m->S, &l->q_norm, NAME("layers.%d.attn.q_norm.weight", i), c->q_lora);
wf_load(&m->S, &l->kv_norm, NAME("layers.%d.attn.kv_norm.weight", i), hd);
wf_load(&m->S, &l->attn_sink,NAME("layers.%d.attn.attn_sink", i), nh);
wf_load(&m->S, &l->attn_norm,NAME("layers.%d.attn_norm.weight", i), dim);
wf_load(&m->S, &l->ffn_norm, NAME("layers.%d.ffn_norm.weight", i), dim);
int mix = (2 + hc) * hc;
wf_load(&m->S, &l->hc_attn_fn, NAME("layers.%d.hc_attn_fn", i), (int64_t)mix * hc * dim);
wf_load(&m->S, &l->hc_ffn_fn, NAME("layers.%d.hc_ffn_fn", i), (int64_t)mix * hc * dim);
wf_load(&m->S, &l->hc_attn_base, NAME("layers.%d.hc_attn_base", i), mix);
wf_load(&m->S, &l->hc_ffn_base, NAME("layers.%d.hc_ffn_base", i), mix);
wf_load(&m->S, &l->hc_attn_scale,NAME("layers.%d.hc_attn_scale", i), 3);
wf_load(&m->S, &l->hc_ffn_scale, NAME("layers.%d.hc_ffn_scale", i), 3);
wb_load(&m->S, &l->gate_w, NAME("layers.%d.ffn.gate.weight", i), c->n_routed, dim);
wf_load(&m->S, &l->gate_bias, NAME("layers.%d.ffn.gate.bias", i), c->n_routed);
V41_DHO(w8_load(&m->S, &l->sh_w1, NAME("layers.%d.ffn.shared_experts.w1.weight", i), c->moe_inter, dim));
V41_DHO(w8_load(&m->S, &l->sh_w3, NAME("layers.%d.ffn.shared_experts.w3.weight", i), c->moe_inter, dim));
V41_DHO(w8_load(&m->S, &l->sh_w2, NAME("layers.%d.ffn.shared_experts.w2.weight", i), dim, c->moe_inter));
l->window = xmalloc((size_t)c->window * hd * sizeof(float), "window ring");
memset(l->window, 0, (size_t)c->window * hd * sizeof(float));
l->window_pos = xmalloc((size_t)c->window * sizeof(int), "window positions");
for (int k = 0; k < c->window; k++) l->window_pos[k] = -1;
/* Sized from spec_block, NOT from n_mtp: the released config.json
* carries the block size and omits the stage count, so n_mtp is still
* zero here and only becomes 3 inside spec_load, which runs after this
* loop. Reading it here allocated one row and the first speculative
* step then wrote six into it -- a heap overflow that the tiny fixture
* could not show, because the config this repository writes for it
* does declare n_mtp_layers. The block size is what bounds a batch. */
int rows = c->spec_block > 0 ? c->spec_block + 1 : 1;
l->ring_save = xmalloc((size_t)rows * hd * sizeof(float), "displaced ring keys");
l->ring_save_pos = xmalloc((size_t)rows * sizeof(int), "displaced ring positions");
int ratio = c->compress_ratio[i];
if (c->kv_source[i]) {
V41_DHO(wb_load(&m->S, &l->comp_wkv, NAME("layers.%d.attn.compressor.wkv.weight", i), hd, dim));
wf_load(&m->S, &l->comp_norm, NAME("layers.%d.attn.compressor.norm.weight", i), hd);
if (ratio > 1) {
V41_DHO(wb_load(&m->S, &l->comp_wgate, NAME("layers.%d.attn.compressor.wgate.weight", i), hd, dim));
l->cstate_kv = xmalloc((size_t)ratio * hd * sizeof(float), "compressor group");
l->cstate_score = xmalloc((size_t)ratio * hd * sizeof(float), "compressor scores");
l->cstate_save_kv = xmalloc((size_t)rows * hd * sizeof(float), "displaced group");
l->cstate_save_score = xmalloc((size_t)rows * hd * sizeof(float), "displaced scores");
l->cstate_save_slot = xmalloc((size_t)rows * sizeof(int), "displaced group slots");
memset(l->cstate_kv, 0, (size_t)ratio * hd * sizeof(float));
for (int k = 0; k < ratio * hd; k++) l->cstate_score[k] = -INFINITY;
}
int slots = c->max_positions / (ratio > 0 ? ratio : 1);
l->ckv = xmalloc((size_t)slots * hd * sizeof(float), "compressed kv");
memset(l->ckv, 0, (size_t)slots * hd * sizeof(float));
l->ikey = xmalloc((size_t)slots * c->index_head_dim * sizeof(float), "index keys");
memset(l->ikey, 0, (size_t)slots * c->index_head_dim * sizeof(float));
V41_DHO(wb_load(&m->S, &l->idx_wk, NAME("layers.%d.attn.indexer.wk.weight", i),
c->index_head_dim, hd));
wf_load(&m->S, &l->idx_knorm, NAME("layers.%d.attn.indexer.k_norm.weight", i),
c->index_head_dim);
}
if (c->index_source[i]) {
V41_DHO(w8_load(&m->S, &l->idx_wq_b, NAME("layers.%d.attn.indexer.wq_b.weight", i),
c->index_n_heads * c->index_head_dim, c->q_lora));
V41_DHO(wb_load(&m->S, &l->idx_wproj, NAME("layers.%d.attn.indexer.weights_proj.weight", i),
c->index_n_heads, dim));
}
/* an engram table's projection, read with its layer (the device-only placement puts
* a layer up whole); the tables themselves open below */
for (int t = 0; m->engram.active && t < m->engram.n_layers; t++)
if (m->engram.layer_of[t] == i)
V41_DHO(w8_load(&m->S, &l->eng_wkv, NAME("layers.%d.engram.wkv.weight", i),
dim * (hc + 1), m->engram.cols * m->engram.head_dim));
#ifdef COLI_VULKAN
if (g_v41_dho) v41_dho_place(m, i); /* one layer in RAM at a time */
#endif
}
if (m->engram.active) {
Engram *e = &m->engram;
for (int t = 0; t < e->n_layers; t++) {
int layer = e->layer_of[t];
if (layer < 0 || layer >= c->n_layers) {
fprintf(stderr, "[engram] table %d names layer %d\n", t, layer); exit(1); }
m->L[layer].engram_index = t;
engram_table_open(&e->table[t], &m->S, layer, e->head_dim, engram_cache_rows);
wf_load(&m->S, &m->L[layer].eng_q, NAME("layers.%d.engram.q_weight", layer),
(int64_t)hc * dim);
wf_load(&m->S, &m->L[layer].eng_k, NAME("layers.%d.engram.k_weight", layer),
(int64_t)hc * dim);
}
}
#ifdef COLI_VULKAN
g_v41_load_layer = -1;
#endif
#undef NAME
/* expert cache: `ecap` slots per layer, each holding one expert's three matrices */
m->ecap = ecap;
m->cache = xmalloc((size_t)c->n_layers * sizeof(LCache), "expert caches");
for (int i = 0; i < c->n_layers; i++) cache_init(m, &m->cache[i], ecap);
spec_load(m, ecap);
if (c->vision_layers > 0) {
m->vision = xmalloc(sizeof(Vision), "vision tower");
memset(m->vision, 0, sizeof(Vision));
vision_load(m, m->vision);
if (!m->vision->active) { free(m->vision); m->vision = NULL; }
}
m->rope_window = rope_table(c->rope_dim, c->max_positions, 0, c->rope_theta,
c->rope_factor, c->beta_fast, c->beta_slow);
m->rope_compress = rope_table(c->rope_dim, c->max_positions, c->original_seq_len,
c->compress_rope_theta, c->rope_factor,
c->beta_fast, c->beta_slow);
m->pos = 0;
}
static const float *rope_for(const Model *m, int layer) {
return m->c.compress_ratio[layer] ? m->rope_compress : m->rope_window;
}
/* ----------------------------------------------------------- attention ----- */
/* model.py get_window_topk_idxs, one batch: which ring slots query `t` may read.
* Prefill indexes the chunk itself (slot i is position i); decode indexes the ring.
*
* On decode the readable set is stated in POSITIONS, not in slot numbers: a slot is
* readable when the position it holds is at or before this query's own and still
* inside the window. For one token a step that is the same thing as the vendor's
* `i > start_pos` test, since slot i then holds position i until the ring wraps. For a
* speculative step it is not: the ring also holds the drafts that were rejected, whose
* positions are ahead of every query that follows, and the position map is what keeps
* them out. The traversal stays oldest-slot-first, per query, because the attention
* kernel sums in list order and a different order is a different rounding. */
static int window_slot_ok(const Layer *l, int slot, int pos, int win) {
int held = l->window_pos[slot];
return held >= 0 && held <= pos && held > pos - win;
}
static void window_idxs(const Cfg *c, const Layer *l, int n, int start_pos, int t,
int *out, int *count) {
int win = c->window;
if (start_pos == 0) {
int width = n < win ? n : win;
int first = t - win + 1; if (first < 0) first = 0;
for (int i = 0; i < width; i++) {
int idx = first + i;
out[i] = idx > t ? -1 : idx;
}
*count = width;
} else {
int pos = start_pos + t;
int oldest = pos % win + 1;
int at = 0;
for (int i = oldest; i < win; i++) out[at++] = window_slot_ok(l, i, pos, win) ? i : -1;
for (int i = 0; i < oldest; i++) out[at++] = window_slot_ok(l, i, pos, win) ? i : -1;
*count = win;
}
}
/* kernel.py sparse_attn for one query: `idx` lists the rows of `kv` this query reads,
* -1 meaning nothing. The sink joins the denominator only, so a query with no readable
* row comes out zero instead of NaN. */
/* The kernel itself lives in sparse_attn.h, where a test can reach it. `score` is
* the caller's scratch: this runs once per head per position per layer, and a
* malloc in that place was costing more than some of the arithmetic. */
static void sparse_attend(float *out, const float *q, const float *kv, const int *idx,
int count, int nh, int hd, float sink_h, float scale,
float *score) {
coli_sparse_attend(out, q, kv, idx, count, hd, sink_h, scale, score);
(void)nh;
}
/* model.py Compressor.forward. Writes the latents this chunk completes into `latent`
* and returns how many there are (pre-RoPE, as the indexer needs them unrotated). */
static int compressor_run(Model *m, int layer, const float *x, int n, int start_pos,
float *latent, int *rows) {
Cfg *c = &m->c;
Layer *l = &m->L[layer];
int hd = c->head_dim, ratio = c->compress_ratio[layer];
if (ratio == 1) {
float *kv = xmalloc((size_t)n * hd * sizeof(float), "compressor kv");
mvb_rows(kv, hd, &l->comp_wkv, x, c->dim, n);
for (int t = 0; t < n; t++) {
rms_into(latent + (size_t)t * hd, kv + (size_t)t * hd, l->comp_norm.w, hd, c->norm_eps);
rows[t] = start_pos + t;
}
free(kv);
return n;
}
float *kv = xmalloc((size_t)n * hd * sizeof(float), "compressor kv");
float *score = xmalloc((size_t)n * hd * sizeof(float), "compressor scores");
mvb_rows(kv, hd, &l->comp_wkv, x, c->dim, n);
mvb_rows(score, hd, &l->comp_wgate, x, c->dim, n);
int produced = 0;
if (start_pos == 0) {
int remainder = n % ratio, cutoff = n - remainder;
for (int g = 0; g * ratio < cutoff; g++) {
for (int i = 0; i < hd; i++) {
float best = -INFINITY;
for (int k = 0; k < ratio; k++) {
float value = score[(size_t)(g * ratio + k) * hd + i];
if (value > best) best = value;
}
float total = 0.0f, mixed = 0.0f;
for (int k = 0; k < ratio; k++) {
float weight = expf(score[(size_t)(g * ratio + k) * hd + i] - best);
total += weight;
mixed += weight * kv[(size_t)(g * ratio + k) * hd + i];
}
latent[(size_t)g * hd + i] = mixed / total;
}
rows[produced++] = g;
}
for (int k = 0; k < remainder; k++) {
memcpy(l->cstate_kv + (size_t)k * hd, kv + (size_t)(cutoff + k) * hd, (size_t)hd * sizeof(float));
memcpy(l->cstate_score + (size_t)k * hd, score + (size_t)(cutoff + k) * hd, (size_t)hd * sizeof(float));
}
} else {
/* One position at a time, which is what a decode step is -- and a speculative
* step is several of them in a row. The group state is indexed by position
* modulo the ratio, so a draft that is rejected leaves its slot to be rewritten
* by the token that really lands there, before the group can complete. */
for (int t = 0; t < n; t++) {
int pos = start_pos + t;
int slot = pos % ratio;
if (n > 1 && m->rollback_save) {
memcpy(l->cstate_save_kv + (size_t)t * hd, l->cstate_kv + (size_t)slot * hd,
(size_t)hd * sizeof(float));
memcpy(l->cstate_save_score + (size_t)t * hd, l->cstate_score + (size_t)slot * hd,
(size_t)hd * sizeof(float));
l->cstate_save_slot[t] = slot;
}
memcpy(l->cstate_kv + (size_t)slot * hd, kv + (size_t)t * hd, (size_t)hd * sizeof(float));
memcpy(l->cstate_score + (size_t)slot * hd, score + (size_t)t * hd, (size_t)hd * sizeof(float));
if ((pos + 1) % ratio) continue;
float *out = latent + (size_t)produced * hd;
for (int i = 0; i < hd; i++) {
float best = -INFINITY;
for (int k = 0; k < ratio; k++) {
float value = l->cstate_score[(size_t)k * hd + i];
if (value > best) best = value;
}
float total = 0.0f, mixed = 0.0f;
for (int k = 0; k < ratio; k++) {
float weight = expf(l->cstate_score[(size_t)k * hd + i] - best);
total += weight;
mixed += weight * l->cstate_kv[(size_t)k * hd + i];
}
out[i] = mixed / total;
}
rows[produced++] = pos / ratio;
}
}
if (produced) {
float *pooled = xmalloc((size_t)produced * hd * sizeof(float), "pooled latents");
memcpy(pooled, latent, (size_t)produced * hd * sizeof(float));
for (int g = 0; g < produced; g++)
rms_into(latent + (size_t)g * hd, pooled + (size_t)g * hd, l->comp_norm.w, hd, c->norm_eps);
free(pooled);
}
free(kv); free(score);
return produced;
}
/* model.py select_candidate_blocks, one query row. `keep` is `width` bytes. */
static void candidate_blocks(const Cfg *c, const float *score, int width, int lens, uint8_t *keep) {
int block = c->candidate_block_size;
int blocks = (width + block - 1) / block;
float *best = xmalloc((size_t)blocks * sizeof(float), "block scores");
for (int b = 0; b < blocks; b++) {
float top = -INFINITY;
for (int i = b * block; i < (b + 1) * block && i < width; i++)
if (score[i] > top) top = score[i];
best[b] = top;
}
int last = (lens - 1) / block;
if (last >= 0 && last < blocks) best[last] = INFINITY; /* the newest block is pinned in */
int wanted = c->candidate_topk_blocks < blocks ? c->candidate_topk_blocks : blocks;
memset(keep, 0, (size_t)width);
for (int pick = 0; pick < wanted; pick++) {
int chosen = -1;
for (int b = 0; b < blocks; b++)
if (best[b] > -INFINITY && (chosen < 0 || best[b] > best[chosen])) chosen = b;
if (chosen < 0) break;
for (int i = chosen * block; i < (chosen + 1) * block && i < width; i++) keep[i] = 1;
best[chosen] = -INFINITY;
}
free(best);
}
/* Does this layer complete a compression group at `pos`, and so republish its index
* keys? model.py: `if self.owns_k and latent is not None`. */
static int layer_publishes(const Cfg *c, int layer, int pos) {
int ratio = c->compress_ratio[layer];
if (!c->kv_source[layer] || ratio <= 0) return 0;
return ratio == 1 || (pos + 1) % ratio == 0;
}
/* Which layer's index keys row `t` reads, or -1 for "whatever the step began with".
* Sequential order is (position, layer): everything at an earlier position happened,
* whatever its layer, and at this position only the layers up to this one -- including
* this one, which publishes before its own indexer runs. */
static int published_owner(const Model *m, int layer, int start_pos, int t) {
if (m->pub_rows == 0) return -1; /* one position: the step's own slot */
for (int L = layer; L >= 0; L--)
if (layer_publishes(&m->c, L, start_pos + t)) return L;
for (int j = t - 1; j >= 0; j--)
if (m->pub_layer[j] >= 0) return m->pub_layer[j];
return -1;
}
/* model.py Indexer.forward: score every compressed position this query can reach and
* keep the best `index_topk`, in position order, shifted by `offset`. */
static void indexer_run(Model *m, int layer, const float *x, const float *qr, int n,
int start_pos, int offset, int compress_len,
int *out, int stride, int topk) {
Cfg *c = &m->c;
Layer *l = &m->L[layer];
int owner = c->index_owner[layer];
int nh = c->index_n_heads, ihd = c->index_head_dim, ratio = c->compress_ratio[layer];
int tidy = getenv("V41_INDEX_OWNER") != NULL;
const float *rope = rope_for(m, layer);
/* The query and weight projections for a block of positions at once (the
* device's GEMM instead of a GEMV per position; the CPU bits are mv8's and
* mvb's either way), then each position scores in order as before. */
int block = n < MV_ROWS_MAX ? n : MV_ROWS_MAX;
float *qblk = xmalloc((size_t)block * nh * ihd * sizeof(float), "indexer queries");
float *wblk = xmalloc((size_t)block * nh * sizeof(float), "indexer weights");
float *score = xmalloc((size_t)compress_len * sizeof(float), "indexer scores");
float scale = (1.0f / sqrtf((float)ihd)) * (1.0f / sqrtf((float)nh));
for (int t = 0; t < n; t++) {
if (t % block == 0) {
int rows = n - t < block ? n - t : block;
mv8_rows(qblk, nh * ihd, &l->idx_wq_b, qr + (size_t)t * c->q_lora, c->q_lora, rows);
mvb_rows(wblk, nh, &l->idx_wproj, x + (size_t)t * c->dim, c->dim, rows);
}
float *q = qblk + (size_t)(t % block) * nh * ihd, *weights = wblk + (size_t)(t % block) * nh;
const float *ikey = m->L[owner].ikey;
if (!tidy) {
int published = published_owner(m, layer, start_pos, t);
if (published >= 0) ikey = m->L[published].ikey;
else if (m->pub_rows > 0) { if (m->pub_before) ikey = m->pub_before; }
/* One position reads whatever was published last; a prefill reads its
* own index owner, whose keys already cover the whole chunk and are
* masked to each query's reach by `lens` above. A resumed prefill
* inherits a published pointer from the previous turn, and honouring
* it here would make the same position read different keys than it
* read when prefilled cold. */
else if (n == 1 && m->published_index_k) ikey = m->published_index_k;
}
for (int h = 0; h < nh; h++)
rope_apply(q + (size_t)h * ihd + ihd - c->rope_dim, rope, start_pos + t, c->rope_dim, 0);
for (int h = 0; h < nh; h++) weights[h] *= scale;
/* how many compressed positions this query can reach: the groups closed at or
* before its own position, which is one expression for prefill and decode
* alike -- and, for a speculative step, per row rather than per step */
int lens = (start_pos + t + 1) / ratio;
for (int j = 0; j < compress_len; j++) {
if (j >= lens) { score[j] = -INFINITY; continue; }
float total = 0.0f;
for (int h = 0; h < nh; h++) {
const float *qh = q + (size_t)h * ihd;
const float *k = ikey + (size_t)j * ihd;
float dot = 0.0f;
for (int i = 0; i < ihd; i++) dot += qh[i] * k[i];
if (dot > 0.0f) total += dot * weights[h]; /* relu, then the head weight */
}
score[j] = total;
}
if (layer == c->candidate_source) {
if (m->candidate_width != compress_len || m->candidate_rows != n) {
free(m->candidates);
m->candidates = xmalloc((size_t)n * compress_len, "candidate mask");
m->candidate_width = compress_len;
m->candidate_rows = n;
}
candidate_blocks(c, score, compress_len, lens, m->candidates + (size_t)t * compress_len);
} else if (c->candidate_source >= 0 && c->candidate_source < layer && m->candidates) {
const uint8_t *keep = m->candidates + (size_t)t * m->candidate_width;
for (int j = 0; j < compress_len && j < m->candidate_width; j++)
if (!keep[j]) score[j] = -INFINITY;
}
/* top-k, then back into position order: the attention kernel treats every slot
* independently, but keeping them sorted makes the concatenated index list
* readable and matches the reference's `.sort()`. */
int *row = out + (size_t)t * stride; /* the compressed half of this query's list */
for (int k = 0; k < topk; k++) row[k] = -1;
int taken = 0;
for (int k = 0; k < topk; k++) {
int best = -1;
for (int j = 0; j < compress_len; j++) {
if (score[j] == -INFINITY) continue;
int already = 0;
for (int u = 0; u < taken; u++) if (row[u] == j) { already = 1; break; }
if (already) continue;
if (best < 0 || score[j] > score[best]) best = j;
}
if (best < 0) break;
row[taken++] = best;
}
for (int a = 0; a < taken; a++) /* insertion sort: taken <= index_topk */
for (int b = a + 1; b < taken; b++)
if (row[b] < row[a]) { int swap = row[a]; row[a] = row[b]; row[b] = swap; }
for (int k = 0; k < topk; k++)
row[k] = (k < taken && row[k] < lens) ? row[k] + offset : -1;
}
free(qblk); free(wblk); free(score);
}
/* wo_a is block diagonal over o_groups: each group projects only its own heads, and
* only then does wo_b mix the groups. Shared with the DSpark stages, whose attention
* differs in what it reads but not in how it comes out. */
static void attn_project_out_rows(Model *m, Layer *l, const float *heads, int rows, float *out) {
Cfg *c = &m->c;
int per_group = c->n_heads * c->head_dim / c->o_groups;
int width = c->o_groups * c->o_lora;
int head_stride = c->n_heads * c->head_dim;
float *grouped = xmalloc((size_t)rows * width * sizeof(float), "grouped");
for (int g = 0; g < c->o_groups; g++) {
/* A W8 view of group g's block. The scale array is indexed by the GLOBAL
* output row, so the view is only the same matrix when a group's rows start
* on a tile boundary -- which o_lora being a multiple of 32 is what makes
* true, and what the fallback below covers when it is not. */
W8 block;
block.q = l->wo_a.q + (size_t)g * c->o_lora * per_group;
block.s = l->wo_a.s + (size_t)(g * c->o_lora / FP8_TILE)
* ((per_group + FP8_TILE - 1) / FP8_TILE);
block.O = c->o_lora;
block.I = per_group;
mv8_rows(grouped + (size_t)g * c->o_lora, width, &block,
heads + (size_t)g * per_group, head_stride, rows);
}
mv8_rows(out, c->dim, &l->wo_b, grouped, width, rows);
free(grouped);
}
/* wo_a's groups have to start on a scale tile for the block view above to name the
* same weights. Checked once at load rather than asserted in the hot path. */
static void attn_project_check(const Cfg *c) {
if (c->o_lora % FP8_TILE != 0) {
fprintf(stderr, "[v41] o_lora_rank %d is not a multiple of the %d-wide scale tile: "
"the grouped output projection cannot be read as blocks\n",
c->o_lora, FP8_TILE);
exit(1);
}
}
/* model.py Attention.forward. `x` is the normalized block input [n, dim]. */
static void attention_run(Model *m, int layer, const float *x, int n, int start_pos, float *out) {
Cfg *c = &m->c;
Layer *l = &m->L[layer];
int dim = c->dim, hd = c->head_dim, nh = c->n_heads, rd = c->rope_dim;
int ratio = c->compress_ratio[layer], owner = c->index_owner[layer];
const float *rope = rope_for(m, layer);
double started = now_s();
float *qr = xmalloc((size_t)n * c->q_lora * sizeof(float), "q latent");
float *q = xmalloc((size_t)n * nh * hd * sizeof(float), "queries");
float *kv = xmalloc((size_t)n * hd * sizeof(float), "kv");
float *scratch = xmalloc((size_t)n * c->q_lora * sizeof(float), "q projection");
float *kv_raw = xmalloc((size_t)n * hd * sizeof(float), "kv projection");
/* One pass of each matrix for the whole block of positions, rather than one
* pass per position: wq_b alone is 42 MB, and during prefill this is the
* difference between reading it once and reading it once per prompt token. */
mv8_rows(scratch, c->q_lora, &l->wq_a, x, dim, n);
for (int t = 0; t < n; t++)
rms_into(qr + (size_t)t * c->q_lora, scratch + (size_t)t * c->q_lora,
l->q_norm.w, c->q_lora, c->norm_eps);
mv8_rows(q, nh * hd, &l->wq_b, qr, c->q_lora, n);
mv8_rows(kv_raw, hd, &l->wkv, x, dim, n);
for (int t = 0; t < n; t++) {
for (int h = 0; h < nh; h++)
rope_apply(q + (size_t)t * nh * hd + (size_t)h * hd + hd - rd, rope, start_pos + t, rd, 0);
rms_into(kv + (size_t)t * hd, kv_raw + (size_t)t * hd, l->kv_norm.w, hd, c->norm_eps);
rope_apply(kv + (size_t)t * hd + hd - rd, rope, start_pos + t, rd, 0);
}
/* the window half: prefill reads the chunk, decode reads the ring */
const float *window_kv;
int window_rows;
if (start_pos == 0) {
int win = c->window;
if (n <= win) {
memcpy(l->window, kv, (size_t)n * hd * sizeof(float));
} else {
int cut = n % win;
const float *tail = kv + (size_t)(n - win) * hd;
memcpy(l->window + (size_t)cut * hd, tail, (size_t)(win - cut) * hd * sizeof(float));
memcpy(l->window, tail + (size_t)(win - cut) * hd, (size_t)cut * hd * sizeof(float));
}
/* both branches above leave position p in slot p % win */
for (int p = n > win ? n - win : 0; p < n; p++) l->window_pos[p % win] = p;
window_kv = kv; window_rows = n;
} else {
/* The ring is written one position at a time, down in the attention loop, and
* not here: a step that carries several positions would otherwise overwrite the
* oldest slot before the first row has read it. The window a query sees has to
* be the window it would have seen alone, which means writing its key, reading,
* and only then writing the next one. */
window_kv = l->window; window_rows = c->window;
}
int window_width = start_pos == 0 ? (n < c->window ? n : c->window) : c->window;
int compress_len = ratio ? (start_pos + n) / ratio : 0;
int index_topk = 0;
if (ratio) {
index_topk = c->index_topk < compress_len ? c->index_topk : compress_len;
if (index_topk < 0) index_topk = 0;
}
int total_idx = window_width + index_topk;
int *idx = xmalloc((size_t)n * total_idx * sizeof(int), "attention indices");
for (int t = 0; t < n; t++) {
int count = 0;
window_idxs(c, l, n, start_pos, t, idx + (size_t)t * total_idx, &count);
for (int k = count; k < window_width; k++) idx[(size_t)t * total_idx + k] = -1;
}
float *kv_all = NULL;
if (ratio) {
/* the compressor runs before the indexer: the indexer scores the latent before
* RoPE, and the cache write below rotates it in place */
float *latent = NULL;
int *latent_row = NULL;
int produced = 0;
if (c->kv_source[layer]) {
int room = n / (ratio ? ratio : 1) + 1;
latent = xmalloc((size_t)room * hd * sizeof(float), "latents");
latent_row = xmalloc((size_t)room * sizeof(int), "latent rows");
produced = compressor_run(m, layer, x, n, start_pos, latent, latent_row);
if (produced) {
/* index keys first, from the un-rotated latent. A latent rotates at the
* FIRST position of the group it pools, which is its row times the
* ratio -- the same value the vendor spells two different ways on the
* prefill and decode paths. */
float *keys = xmalloc((size_t)produced * c->index_head_dim * sizeof(float), "index keys");
mvb_rows(keys, c->index_head_dim, &l->idx_wk, latent, hd, produced);
for (int g = 0; g < produced; g++) {
float *dest = l->ikey + (size_t)latent_row[g] * c->index_head_dim;
rms_into(dest, keys + (size_t)g * c->index_head_dim, l->idx_knorm.w,
c->index_head_dim, c->norm_eps);
rope_apply(dest + c->index_head_dim - rd, rope, latent_row[g] * ratio, rd, 0);
}
free(keys);
/* publish, as model.py does, only when a group completed */
m->published_index_k = l->ikey;
m->published_index_layer = layer;
}
}
if (c->index_source[layer]) {
if (compress_len > 0 && index_topk > 0)
/* The offset is how many ROWS the window half occupies in kv_all, not
* how many index columns it takes: during prefill the window half is
* the whole chunk (n rows) while each query lists at most `window`
* columns of it. model.py passes kv.size(1) here for the same reason. */
indexer_run(m, layer, x, qr, n, start_pos, window_rows, compress_len,
idx + window_width, total_idx, index_topk);
else
for (int t = 0; t < n; t++)
for (int k = window_width; k < total_idx; k++) idx[(size_t)t * total_idx + k] = -1;
/* publish for the layers between here and the next index source */
if (m->shared_topk_rows != n || m->shared_topk_width != index_topk) {
free(m->shared_topk);
m->shared_topk = xmalloc((size_t)n * (index_topk > 0 ? index_topk : 1) * sizeof(int),
"published index list");
m->shared_topk_rows = n; m->shared_topk_width = index_topk;
}
for (int t = 0; t < n; t++)
for (int k = 0; k < index_topk; k++)
m->shared_topk[(size_t)t * index_topk + k] = idx[(size_t)t * total_idx + window_width + k];
} else {
/* Not an index source: reuse what the last one published, exactly as
* model.py does through shared_attn.topk_idxs. Recomputing here would be
* wrong as well as wasteful -- these layers have no indexer weights. */
for (int t = 0; t < n; t++)
for (int k = 0; k < index_topk; k++) {
int value = -1;
if (m->shared_topk && t < m->shared_topk_rows && k < m->shared_topk_width)
value = m->shared_topk[(size_t)t * m->shared_topk_width + k];
idx[(size_t)t * total_idx + window_width + k] = value;
}
}
if (produced) {
for (int g = 0; g < produced; g++) {
float *dest = l->ckv + (size_t)latent_row[g] * hd;
memcpy(dest, latent + (size_t)g * hd, (size_t)hd * sizeof(float));
rope_apply(dest + hd - rd, rope, latent_row[g] * ratio, rd, 0);
}
}
free(latent); free(latent_row);
kv_all = xmalloc((size_t)(window_rows + compress_len) * hd * sizeof(float), "kv all");
memcpy(kv_all, window_kv, (size_t)window_rows * hd * sizeof(float));
memcpy(kv_all + (size_t)window_rows * hd, m->L[owner].ckv,
(size_t)compress_len * hd * sizeof(float));
}
const float *kv_read = ratio ? kv_all : window_kv;
/* A block of positions' heads, so the output projection can run as one block
* too. Bounded rather than the whole chunk: at 64 heads of 512 this is 128 KB a
* position, and a long prompt would otherwise ask for a quarter of a gigabyte
* of it. */
int hblock = n < MV_ROWS_MAX ? n : MV_ROWS_MAX;
float *heads = xmalloc((size_t)hblock * nh * hd * sizeof(float), "attention output");
float *head_score = xmalloc((size_t)nh * total_idx * sizeof(float), "attention scores");
float attn_scale = 1.0f / sqrtf((float)hd);
for (int t = 0; t < n; t++) {
if (start_pos > 0) {
int pos = start_pos + t, slot = pos % c->window;
if (n > 1 && m->rollback_save) {
memcpy(l->ring_save + (size_t)t * hd, l->window + (size_t)slot * hd,
(size_t)hd * sizeof(float));
l->ring_save_pos[t] = l->window_pos[slot];
}
memcpy(l->window + (size_t)slot * hd, kv + (size_t)t * hd, (size_t)hd * sizeof(float));
l->window_pos[slot] = pos;
/* kv_all holds a copy of the ring, so it takes the same write */
if (kv_all) memcpy(kv_all + (size_t)slot * hd, kv + (size_t)t * hd,
(size_t)hd * sizeof(float));
int count = 0;
window_idxs(c, l, n, start_pos, t, idx + (size_t)t * total_idx, &count);
for (int k = count; k < window_width; k++) idx[(size_t)t * total_idx + k] = -1;
}
const int *row = idx + (size_t)t * total_idx;
float *row_heads = heads + (size_t)(t % hblock) * nh * hd;
#pragma omp parallel for schedule(static)
for (int h = 0; h < nh; h++)
sparse_attend(row_heads + (size_t)h * hd, q + (size_t)t * nh * hd + (size_t)h * hd,
kv_read, row, total_idx, nh, hd, l->attn_sink.w[h], attn_scale,
head_score + (size_t)h * total_idx);
for (int h = 0; h < nh; h++)
rope_apply(row_heads + (size_t)h * hd + hd - rd, rope, start_pos + t, rd, 1);
/* The projection reads the heads and writes `out`; it touches no cached
* state, so it waits until the block is full and then runs once for it. */
if (t % hblock == hblock - 1 || t == n - 1) {
int first = t - (t % hblock);
attn_project_out_rows(m, l, heads, t - first + 1, out + (size_t)first * dim);
}
}
m->t_attn += now_s() - started;
free(head_score); free(kv_raw);
free(heads); free(kv_all); free(idx); free(scratch); free(kv); free(q); free(qr);
}
/* ------------------------------------------------------------------ MoE ---- */
/* One expert's SwiGLU. The clamps are the training kernel's, and they matter: they
* are what keeps fp4 activations in range. */
/* silu(gate) * clamp(up), in place over `gate`, for however many rows are in it */
static void swiglu_into(const Cfg *c, float *gate, const float *up, int64_t count) {
for (int64_t i = 0; i < count; i++) {
float g = gate[i], u = up[i];
if (c->swiglu_limit > 0.0f) {
if (u > c->swiglu_limit) u = c->swiglu_limit;
if (u < -c->swiglu_limit) u = -c->swiglu_limit;
if (g > c->swiglu_limit) g = c->swiglu_limit;
}
gate[i] = (g / (1.0f + expf(-g))) * u;
}
}
/* One routed expert applied to every position that chose it. The expert is 18.8 MB
* on the released checkpoint and matmul_mxfp4 already walks its rows once for a
* block of inputs, so a position that shares an expert with another costs the
* arithmetic and not a second pass over the weights. Each output row is what the
* one-at-a-time call produced, bit for bit: the S loop sits inside the row loop,
* so a row's groups fold in the same order whatever else is in the block. */
static void expert_ffn_rows(Model *m, const uint8_t *w1, const uint8_t *s1,
const uint8_t *w3, const uint8_t *s3,
const uint8_t *w2, const uint8_t *s2,
const float *x, int rows, float *down) {
Cfg *c = &m->c;
int inter = c->moe_inter, dim = c->dim;
float *gate = xmalloc((size_t)rows * inter * sizeof(float), "expert gate");
float *up = xmalloc((size_t)rows * inter * sizeof(float), "expert up");
double started = now_s();
matmul_mxfp4(gate, x, w1, s1, rows, dim, inter);
matmul_mxfp4(up, x, w3, s3, rows, dim, inter);
swiglu_into(c, gate, up, (int64_t)rows * inter);
matmul_mxfp4(down, gate, w2, s2, rows, inter, dim);
m->t_expert += now_s() - started;
free(up); free(gate);
}
/* The shared expert every token pays for; fp8, so it stays resident and never
* streams -- but it is still 35 MB of matrices, and reading them once for a block
* of positions rather than once per position is the same saving mv8_rows makes in
* attention. */
/* The shared expert's output rows into `down` ... */
static void shared_ffn_down(Model *m, Layer *l, const float *x, int rows, float *down) {
Cfg *c = &m->c;
int inter = c->moe_inter, dim = c->dim;
float *gate = xmalloc((size_t)rows * inter * sizeof(float), "shared gate");
float *up = xmalloc((size_t)rows * inter * sizeof(float), "shared up");
mv8_rows(gate, inter, &l->sh_w1, x, dim, rows);
mv8_rows(up, inter, &l->sh_w3, x, dim, rows);
swiglu_into(c, gate, up, (int64_t)rows * inter);
mv8_rows(down, dim, &l->sh_w2, gate, inter, rows);
free(up); free(gate);
}
/* ... and added to `out` */
static void shared_ffn_rows(Model *m, Layer *l, const float *x, int rows, float *out) {
int dim = m->c.dim;
float *down = xmalloc((size_t)rows * dim * sizeof(float), "shared down");
shared_ffn_down(m, l, x, rows, down);
for (int64_t i = 0; i < (int64_t)rows * dim; i++) out[i] += down[i];
free(down);
}
/* model.py Gate + MoE. The bias steers the choice of experts and nothing else: the
* weights come from the unbiased scores, which is the whole point of noaux_tc.
* Split out of the MoE proper so a whole block of positions can be routed before
* any expert is read -- which is what lets the reads and the matmuls be shared.
* `scores` holds the position's gate logits (the caller projects the block at once)
* and is overwritten. */
static void moe_gate(Model *m, Layer *l, int E, int topk, float *scores,
int *chosen, float *weights) {
Cfg *c = &m->c;
for (int e = 0; e < E; e++) {
float value = scores[e] / c->gate_temp;
/* sqrtsoftplus: softplus then square root, in fp32 as the vendor does */
float softplus = value > 20.0f ? value : logf(1.0f + expf(value));
scores[e] = sqrtf(softplus);
}
for (int k = 0; k < topk; k++) {
int best = -1;
for (int e = 0; e < E; e++) {
int taken = 0;
for (int u = 0; u < k; u++) if (chosen[u] == e) { taken = 1; break; }
if (taken) continue;
if (best < 0 || scores[e] + l->gate_bias.w[e] > scores[best] + l->gate_bias.w[best]) best = e;
}
chosen[k] = best;
weights[k] = scores[best];
}
if (c->norm_topk_prob && topk > 1) {
float total = 0.0f;
for (int k = 0; k < topk; k++) total += weights[k];
total += 1e-20f; /* the vendor's constant, not norm_eps */
for (int k = 0; k < topk; k++) weights[k] /= total;
}
for (int k = 0; k < topk; k++) weights[k] *= c->route_scale;
}
#define MOE_TOPK_MAX 64
/* Positions routed together. Bounded so the per-chunk scratch stays a few
* megabytes however long the prompt is, and so the experts one chunk asks for
* stay a plausible working set for the cache. */
#define MOE_ROW_CHUNK 32
#ifdef COLI_VULKAN
/* ---- the Vulkan routed-expert tier (vk_tier.c) ---------------------------------
* One block of positions of a backbone layer with the tier on. The block's resident
* experts go to the device as one batch (vkt_issue) while the CPU computes the other
* draws expert-major, as moe_run_at does, and the shared expert into a buffer of its
* own; then every rank of every position joins the output in rank order, the
* device's rows and the CPU's alike (w x row, then the add), and the shared expert
* after them, as moe_run_at adds them. So the order of the sum never depends on
* which experts were resident. Every expert the CPU computed passes its slot to the
* tier (vkt_note), which may promote it. The DSpark stages ("mtp"), with caches of
* their own, are the tier's extra layers n_layers + stage when it took them
* (COLI_VK_TIER_MTP). */
static char g_vk_usage[2100]; /* the tier's history (vk_tier_start); empty = none */
static void vk_tier_save(void) { if (g_vk_usage[0]) rt_save(g_vk_usage, 1); }
static VktExpertSrc vk_slot_src(const Slot *s) {
return (VktExpertSrc){s->w1, s->w3, s->w2, s->s1, s->s3, s->s2};
}
/* The draws want[d] of a block on the CPU into contrib[d] = w x expert(x): moe_run_at's
* expert-major union over those draws only. */
static void moe_vk_cpu(Model *m, LCache *cache, const char *kind, int layer, int topk, const float *xc, int rows,
const int *chosen, const float *weights, const uint8_t *want, float *contrib) {
Cfg *c = &m->c;
int dim = c->dim, draws = rows * topk;
/* a block of up to MOE_ROW_CHUNK rows, or a streamed prompt chunk (vkt_step_rows) */
int *uniq = xmalloc((size_t)draws * 5 * sizeof(int), "expert union");
int *head = uniq + draws, *tail = head + draws, *count = tail + draws, *next = count + draws;
int n_uniq = 0;
for (int d = 0; d < draws; d++) {
if (!want[d]) continue;
int eid = chosen[d], at = -1;
for (int u = 0; u < n_uniq; u++) if (uniq[u] == eid) { at = u; break; }
next[d] = -1;
if (at < 0) { at = n_uniq++; uniq[at] = eid; head[at] = d; count[at] = 0; }
else next[tail[at]] = d;
tail[at] = d;
count[at]++;
}
if (!n_uniq) { free(uniq); return; }
float *gathered = xmalloc((size_t)rows * dim * sizeof(float), "expert inputs");
float *down = xmalloc((size_t)rows * dim * sizeof(float), "expert outputs");
int step = cache->cap < MOE_ROW_CHUNK ? cache->cap : MOE_ROW_CHUNK;
for (int u0 = 0; u0 < n_uniq; u0 += step) {
int ne = n_uniq - u0 < step ? n_uniq - u0 : step;
Slot *slot[MOE_ROW_CHUNK];
expert_slots_at(m, cache, kind, layer, uniq + u0, ne, slot);
for (int u = 0; u < ne; u++) {
int cnt = count[u0 + u], at = 0;
for (int d = head[u0 + u]; d >= 0; d = next[d], at++)
memcpy(gathered + (size_t)at * dim, xc + (size_t)(d / topk) * dim, (size_t)dim * sizeof(float));
expert_ffn_rows(m, slot[u]->w1, slot[u]->s1, slot[u]->w3, slot[u]->s3,
slot[u]->w2, slot[u]->s2, gathered, cnt, down);
at = 0;
for (int d = head[u0 + u]; d >= 0; d = next[d], at++) {
float w = weights[d];
float *dst = contrib + (size_t)d * dim;
const float *src = down + (size_t)at * dim;
for (int i = 0; i < dim; i++) dst[i] = w * src[i];
}
m->hits += (uint64_t)(cnt - 1);
VktExpertSrc src = vk_slot_src(slot[u]);
vkt_note(v41_tier_layer(m, kind, layer), uniq[u0 + u], &src);
}
}
free(down); free(gathered); free(uniq);
}
/* kind "layers" (the backbone) or "mtp" (a DSpark stage, the tier's extra layer: no
* history row and no EMAP mark, as on the CPU path) */
static void moe_vk_block(Model *m, Layer *l, LCache *cache, const char *kind, int layer, int topk, const float *xc,
int rows, float *outc, const int *chosen, const float *weights, int with_shared) {
int dim = m->c.dim, draws = rows * topk, backbone = !strcmp(kind, "layers");
uint8_t *taken = xmalloc((size_t)draws * 2, "device draws"), *want = taken + draws;
const float **dev = xmalloc((size_t)draws * sizeof(*dev), "device rows");
for (int r = 0; backbone && r < rows; r++) rt_count(layer, chosen + r * topk, topk);
int ndev = vkt_issue(v41_tier_layer(m, kind, layer), xc, rows, topk, chosen, taken);
for (int d = 0; d < draws; d++) { want[d] = !taken[d]; if (taken[d] && backbone) ehit_mark(m, layer, chosen[d]); }
float *contrib = xmalloc((size_t)draws * dim * sizeof(float), "expert contributions");
float *shared = xmalloc((size_t)rows * dim * sizeof(float), "shared expert");
moe_vk_cpu(m, cache, kind, layer, topk, xc, rows, chosen, weights, want, contrib);
if (with_shared) shared_ffn_down(m, l, xc, rows, shared);
if (ndev && !vkt_join(dev)) { /* the batch failed (the tier stops): those draws here */
moe_vk_cpu(m, cache, kind, layer, topk, xc, rows, chosen, weights, taken, contrib);
memset(taken, 0, (size_t)draws);
}
for (int r = 0; r < rows; r++) {
float *o = outc + (size_t)r * dim;
memset(o, 0, (size_t)dim * sizeof(float));
for (int k = 0; k < topk; k++) {
int d = r * topk + k;
if (taken[d]) {
float w = weights[d];
const float *src = dev[d];
for (int i = 0; i < dim; i++) o[i] += w * src[i];
} else {
const float *cvec = contrib + (size_t)d * dim;
for (int i = 0; i < dim; i++) o[i] += cvec[i];
}
}
const float *sh = shared + (size_t)r * dim;
if (with_shared) for (int i = 0; i < dim; i++) o[i] += sh[i];
}
free(shared); free(contrib); free(taken); free(dev);
}
/* The tier's streaming (a big prompt chunk's cold experts on the device): a group of a
* backbone layer's experts through its cache as moe_vk_cpu reads them (expert_slots_at:
* the misses fetched together), up to the cache's capacity, valid until the next read. */
static int vk_load_batch(void *ctx, int layer, const int *e, int n, VktExpertSrc *srcs, void **h) {
Model *m = ctx;
LCache *cache = &m->cache[layer];
if (n > cache->cap) n = cache->cap;
if (n > 64) n = 64;
if (n < 1) return 0;
Slot *slot[64];
expert_slots_at(m, cache, "layers", layer, e, n, slot);
for (int u = 0; u < n; u++) { srcs[u] = vk_slot_src(slot[u]); h[u] = slot[u]; }
return n;
}
static int vk_load(void *ctx, int layer, int e, VktExpertSrc *src, void **h) {
return vk_load_batch(ctx, layer, &e, 1, src, h) == 1;
}
static void vk_unhold(void *ctx, void *h) { (void)ctx; (void)h; }
#endif
/* The MoE for a block of positions, expert-major.
*
* Position-major is the obvious reading and it is what this did: route a
* position, read its six experts, multiply, move on. During prefill that reads
* the same expert again for the next position that wants it -- 18.8 MB off the
* device or out of RAM, for arithmetic that could have ridden along with the
* first. Routing the whole block first turns the six-per-position draws into a
* list of DISTINCT experts, each read once and applied to every position that
* asked for it, with matmul_mxfp4 walking its rows a single time for all of them.
*
* The arithmetic is unchanged and so are the values. Each expert's contribution
* is kept apart and the contributions are summed into the output in rank order
* afterwards, exactly as the position-major loop accumulated them; a row's
* matmul folds its groups in the same order whatever else shares the block.
*
* A cache too small to hold one position's experts keeps the old path: there the
* slots genuinely cannot all be live at once. */
/* with_shared 0: the routed experts only (the dense chain runs the shared expert on the
* device and adds it there, in the same place of the sum). */
static void moe_run_at(Model *m, Layer *l, LCache *cache, const char *kind, int layer,
int E, int topk, const float *x, int n, float *out, int with_shared) {
Cfg *c = &m->c;
int dim = c->dim;
if (topk > MOE_TOPK_MAX) {
fprintf(stderr, "[moe] n_activated %d exceeds the scratch\n", topk); exit(1); }
#ifdef COLI_VULKAN
/* the tier streams a prompt chunk's cold experts: the whole step in one block, so
* each expert's rows meet in one GEMM (moe_vk_block's arrays follow its size) */
if (vkt_ready() && !strcmp(kind, "layers") && vkt_step_rows(n, MOE_ROW_CHUNK) > MOE_ROW_CHUNK) {
int *chosen = xmalloc((size_t)n * topk * sizeof(int), "routing");
float *weights = xmalloc((size_t)n * topk * sizeof(float), "routing weights");
float *scores = xmalloc((size_t)n * E * sizeof(float), "gate scores");
mvb_rows(scores, E, &l->gate_w, x, dim, n);
for (int r = 0; r < n; r++)
moe_gate(m, l, E, topk, scores + (size_t)r * E, chosen + r * topk, weights + r * topk);
free(scores);
moe_vk_block(m, l, cache, kind, layer, topk, x, n, out, chosen, weights, with_shared);
free(chosen); free(weights);
return;
}
#endif
for (int r0 = 0; r0 < n; r0 += MOE_ROW_CHUNK) {
int rows = n - r0 < MOE_ROW_CHUNK ? n - r0 : MOE_ROW_CHUNK;
const float *xc = x + (size_t)r0 * dim;
float *outc = out + (size_t)r0 * dim;
int chosen[MOE_ROW_CHUNK * MOE_TOPK_MAX];
float weights[MOE_ROW_CHUNK * MOE_TOPK_MAX];
float *scores = xmalloc((size_t)rows * E * sizeof(float), "gate scores");
mvb_rows(scores, E, &l->gate_w, xc, dim, rows); /* the chunk's gate logits at once */
for (int r = 0; r < rows; r++)
moe_gate(m, l, E, topk, scores + (size_t)r * E,
chosen + r * topk, weights + r * topk);
free(scores);
#ifdef COLI_VULKAN
if (vkt_ready() && v41_tier_layer(m, kind, layer) < vkt_layers()) { /* the backbone, a stage the tier took */
moe_vk_block(m, l, cache, kind, layer, topk, xc, rows, outc, chosen, weights, with_shared);
continue;
}
#endif
int draws = rows * topk;
if (cache->cap < topk) {
/* one position at a time, and one expert at a time inside it */
for (int r = 0; r < rows; r++) {
float *o = outc + (size_t)r * dim;
memset(o, 0, (size_t)dim * sizeof(float));
float *down = xmalloc((size_t)dim * sizeof(float), "expert down");
for (int k = 0; k < topk; k++) {
Slot *s = expert_slot_at(m, cache, kind, layer, chosen[r * topk + k]);
expert_ffn_rows(m, s->w1, s->s1, s->w3, s->s3, s->w2, s->s2,
xc + (size_t)r * dim, 1, down);
float w = weights[r * topk + k];
for (int i = 0; i < dim; i++) o[i] += w * down[i];
}
free(down);
}
if (with_shared) shared_ffn_rows(m, l, xc, rows, outc);
continue;
}
/* the distinct experts this block asks for, each with the draws that asked */
int *uniq = xmalloc((size_t)draws * sizeof(int), "routed expert set");
int *head = xmalloc((size_t)draws * sizeof(int), "expert draw list");
int *tail = xmalloc((size_t)draws * sizeof(int), "expert draw tail");
int *count = xmalloc((size_t)draws * sizeof(int), "expert draw count");
int *next = xmalloc((size_t)draws * sizeof(int), "expert draw chain");
int n_uniq = 0;
for (int d = 0; d < draws; d++) {
int eid = chosen[d], at = -1;
for (int u = 0; u < n_uniq; u++) if (uniq[u] == eid) { at = u; break; }
next[d] = -1;
if (at < 0) {
at = n_uniq++;
uniq[at] = eid; head[at] = d; count[at] = 0;
} else {
next[tail[at]] = d;
}
tail[at] = d;
count[at]++;
}
float *contrib = xmalloc((size_t)draws * dim * sizeof(float), "expert contributions");
float *gathered = xmalloc((size_t)rows * dim * sizeof(float), "expert inputs");
float *down = xmalloc((size_t)rows * dim * sizeof(float), "expert outputs");
int step = cache->cap < MOE_ROW_CHUNK ? cache->cap : MOE_ROW_CHUNK;
/* Two ways of overlapping these reads with the matmuls were built and
* measured here, and both were worse. A posix_fadvise(WILLNEED) hint for the
* next chunk blocks against a device already saturated by the demand reads:
* four seconds of wall to save one of disk, the same result the V4 engine
* got. A pool of reader threads, so each expert is multiplied as soon as its
* own six tensors land, does overlap -- the main thread's wait on disk falls
* from 10.4 s to 1.0 s -- but the expert matmul rises from 7.6 s to 18.1 s
* and the turn is 7% slower, 25% slower at four readers and 68% slower at two. A 5.9 MB pread is
* not free CPU: it is a kernel-side copy competing for the same memory
* bandwidth the matmuls are already bound by, and the idle disk wait it
* removes costs more than it was worth. Both are measurements, not opinions;
* they are written down so the third attempt starts from them. */
for (int u0 = 0; u0 < n_uniq; u0 += step) {
int ne = n_uniq - u0 < step ? n_uniq - u0 : step;
Slot *slot[MOE_ROW_CHUNK];
expert_slots_at(m, cache, kind, layer, uniq + u0, ne, slot);
for (int u = 0; u < ne; u++) {
int cnt = count[u0 + u], at = 0;
for (int d = head[u0 + u]; d >= 0; d = next[d], at++)
memcpy(gathered + (size_t)at * dim, xc + (size_t)(d / topk) * dim,
(size_t)dim * sizeof(float));
expert_ffn_rows(m, slot[u]->w1, slot[u]->s1, slot[u]->w3, slot[u]->s3,
slot[u]->w2, slot[u]->s2, gathered, cnt, down);
at = 0;
for (int d = head[u0 + u]; d >= 0; d = next[d], at++) {
float w = weights[d];
float *dst = contrib + (size_t)d * dim;
const float *src = down + (size_t)at * dim;
for (int i = 0; i < dim; i++) dst[i] = w * src[i];
}
/* the draws after the first found the expert resident, which is what
* the position-major loop would have recorded for them too */
m->hits += (uint64_t)(cnt - 1);
}
}
for (int r = 0; r < rows; r++) {
float *o = outc + (size_t)r * dim;
memset(o, 0, (size_t)dim * sizeof(float));
for (int k = 0; k < topk; k++) {
const float *cvec = contrib + (size_t)(r * topk + k) * dim;
for (int i = 0; i < dim; i++) o[i] += cvec[i];
}
}
if (with_shared) shared_ffn_rows(m, l, xc, rows, outc);
free(down); free(gathered); free(contrib);
free(next); free(count); free(tail); free(head); free(uniq);
}
}
static void moe_run(Model *m, int layer, const float *x, int n, float *out) {
moe_run_at(m, &m->L[layer], &m->cache[layer], "layers", layer,
m->c.n_routed, m->c.n_activated, x, n, out, 1);
}
/* --------------------------------------------------------------- engram ---- */
/* model.py Engram.forward: look the n-grams up, project them into one key per residual
* copy plus a shared value, and gate the value in by how well key and stream agree. */
static void engram_run(Model *m, int layer, float *h, int n, int start_pos) {
Cfg *c = &m->c;
Engram *e = &m->engram;
Layer *l = &m->L[layer];
int table = l->engram_index, hc = c->hc_mult, dim = c->dim;
double started = now_s();
/* Blocked like the rest: the n-gram rows are looked up for a block of positions,
* then eng_wkv projects the whole block in one pass. The matrix is 157 MB on the
* released head, which is a lot to walk once per position. */
int width = e->cols * e->head_dim, kv_width = dim * (hc + 1);
int chunk = n < MOE_ROW_CHUNK ? n : MOE_ROW_CHUNK;
float *rows = xmalloc((size_t)chunk * width * sizeof(float), "engram rows");
float *kv_all = xmalloc((size_t)chunk * kv_width * sizeof(float), "engram kv");
int64_t ids[V41_MAX_NGRAM * V41_MAX_EHEADS];
for (int t0 = 0; t0 < n; t0 += chunk) {
int rows_here = n - t0 < chunk ? n - t0 : chunk;
for (int r = 0; r < rows_here; r++) {
engram_hash(e, table, start_pos + t0 + r, ids);
for (int col = 0; col < e->cols; col++) {
const float *row = engram_row(&e->table[table], ids[col], e->head_dim);
memcpy(rows + (size_t)r * width + (size_t)col * e->head_dim, row,
(size_t)e->head_dim * sizeof(float));
}
}
mv8_rows(kv_all, kv_width, &l->eng_wkv, rows, width, rows_here);
for (int r = 0; r < rows_here; r++) {
int t = t0 + r;
const float *kv = kv_all + (size_t)r * kv_width;
const float *value = kv + (size_t)hc * dim;
float *stream = h + (size_t)t * hc * dim;
for (int copy = 0; copy < hc; copy++) {
const float *key = kv + (size_t)copy * dim;
const float *qw = l->eng_q.w + (size_t)copy * dim;
const float *kw = l->eng_k.w + (size_t)copy * dim;
float *stream_copy = stream + (size_t)copy * dim;
double stream_square = 0.0, key_square = 0.0;
double dot = 0.0;
for (int i = 0; i < dim; i++) {
stream_square += (double)stream_copy[i] * stream_copy[i];
key_square += (double)key[i] * key[i];
dot += (double)stream_copy[i] * qw[i] * kw[i] * key[i];
}
float rstd = (1.0f / sqrtf((float)(stream_square / dim) + c->norm_eps)) *
(1.0f / sqrtf((float)(key_square / dim) + c->norm_eps));
float scaled = (float)dot * rstd / sqrtf((float)dim);
/* signed square root before the sigmoid, matching the training kernel */
float magnitude = fabsf(scaled);
if (magnitude < 1e-6f) magnitude = 1e-6f;
float signed_root = sqrtf(magnitude);
if (scaled < 0.0f) signed_root = -signed_root;
float gate = coli_hc_sigmoid(signed_root);
for (int i = 0; i < dim; i++) stream_copy[i] += gate * value[i];
}
}
}
trace("engram", layer, h + (size_t)(n - 1) * hc * dim, hc * dim);
free(kv_all); free(rows);
m->t_engram += now_s() - started;
}
/* --------------------------------------------------------------- vision ---- */
/* The ViT is small next to the language model (about 600 MB in the released
* checkpoint, bf16) and every image needs all of it, so it stays resident: there is
* nothing to stream here. What matters for correctness is that its two conventions
* differ from the language model's and are easy to get silently wrong:
*
* - RoPE is SPLIT-HALF (x[:d/2] against x[d/2:]), not the interleaved pairs the
* text side uses, and it is 2D: each patch carries its row and column angle.
* - the aligner's 3x3 block is ordered channel-major, because torch's F.unfold
* lays a block out as (channel, kh, kw) -- read it row-major and the projection
* sees a permuted image.
*/
static void vision_load(Model *m, Vision *v) {
Cfg *c = &m->c;
if (c->vision_layers < 1) { v->active = 0; return; }
if (!st_has(&m->S, "vision.patch_embed.proj.weight")) { v->active = 0; return; }
char name[256];
#define VNAME(fmt, ...) (snprintf(name, sizeof(name), fmt, __VA_ARGS__), name)
int vd = c->vision_dim, patch_in = 3 * c->vision_patch * c->vision_patch;
wb_load(&m->S, &v->proj_w, "vision.patch_embed.proj.weight", vd, patch_in);
wf_load(&m->S, &v->proj_b, "vision.patch_embed.proj.bias", vd);
v->block = xmalloc((size_t)c->vision_layers * sizeof(VisionBlock), "vision blocks");
for (int i = 0; i < c->vision_layers; i++) {
VisionBlock *b = &v->block[i];
wf_load(&m->S, &b->norm1, VNAME("vision.blocks.%d.norm1.weight", i), vd);
wf_load(&m->S, &b->norm2, VNAME("vision.blocks.%d.norm2.weight", i), vd);
wb_load(&m->S, &b->qkv_w, VNAME("vision.blocks.%d.attn.wqkv.weight", i), 3 * vd, vd);
wf_load(&m->S, &b->qkv_b, VNAME("vision.blocks.%d.attn.wqkv.bias", i), 3 * vd);
wb_load(&m->S, &b->o_w, VNAME("vision.blocks.%d.attn.wo.weight", i), vd, vd);
wf_load(&m->S, &b->o_b, VNAME("vision.blocks.%d.attn.wo.bias", i), vd);
wb_load(&m->S, &b->w1, VNAME("vision.blocks.%d.mlp.w1.weight", i), 2 * c->vision_inter, vd);
wb_load(&m->S, &b->w2, VNAME("vision.blocks.%d.mlp.w2.weight", i), vd, c->vision_inter);
}
wf_load(&m->S, &v->norm, "vision.norm.weight", vd);
int ratio = c->vision_ratio;
wb_load(&m->S, &v->align_w1, "aligner.w1.weight", c->dim, vd * ratio * ratio);
wf_load(&m->S, &v->align_b1, "aligner.w1.bias", c->dim);
wb_load(&m->S, &v->align_w2, "aligner.w2.weight", c->dim, c->dim);
wf_load(&m->S, &v->align_b2, "aligner.w2.bias", c->dim);
wf_load(&m->S, &v->image_start, "image_start", c->dim);
wf_load(&m->S, &v->image_end, "image_end", c->dim);
wf_load(&m->S, &v->image_newline, "image_newline", c->dim);
#undef VNAME
v->active = 1;
}
/* How many rows the aligner returns for a grid: the grid is padded up to the ratio. */
static int vision_tokens(const Cfg *c, int n_h, int n_w) {
int ratio = c->vision_ratio;
return ((n_h + ratio - 1) / ratio) * ((n_w + ratio - 1) / ratio);
}
/* patches: [n_h * n_w, 3 * patch * patch] row-major. out: [vision_tokens, dim]. */
static void vision_forward(Model *m, Vision *v, const float *patches, int n_h, int n_w, float *out) {
Cfg *c = &m->c;
int vd = c->vision_dim, heads = c->vision_heads, hd = vd / heads, half = hd / 2;
int n = n_h * n_w, patch_in = 3 * c->vision_patch * c->vision_patch;
float *x = xmalloc((size_t)n * vd * sizeof(float), "vision stream");
/* every projection below runs over all n patches at once (mvb_rows: one device
* GEMM, mvb's own bits on the CPU) */
mvb_rows(x, vd, &v->proj_w, patches, patch_in, n);
for (int i = 0; i < n; i++)
for (int j = 0; j < vd; j++) x[(size_t)i * vd + j] += v->proj_b.w[j];
/* 2D RoPE tables: one angle per (patch, frequency), row and column interleaved
* exactly as vision.py stacks them */
float *cosine = xmalloc((size_t)n * half * sizeof(float), "vision cos");
float *sine = xmalloc((size_t)n * half * sizeof(float), "vision sin");
for (int p = 0; p < n; p++) {
int row = p / n_w, col = p % n_w;
for (int f = 0; f < half; f++) {
int axis = f < half / 2 ? 0 : 1; /* first half rows, second columns */
int index = axis ? f - half / 2 : f;
float inv = 1.0f / powf(c->vision_rope_theta, (float)(2 * index) / (float)half);
float angle = (axis ? col : row) * inv;
cosine[(size_t)p * half + f] = cosf(angle);
sine[(size_t)p * half + f] = sinf(angle);
}
}
float *qkv_all = xmalloc((size_t)n * 3 * vd * sizeof(float), "vision qkv");
float *q = xmalloc((size_t)n * vd * sizeof(float), "vision q");
float *k = xmalloc((size_t)n * vd * sizeof(float), "vision k");
float *val = xmalloc((size_t)n * vd * sizeof(float), "vision v");
float *normed = xmalloc((size_t)n * vd * sizeof(float), "vision normed");
float *attended = xmalloc((size_t)n * vd * sizeof(float), "vision attended");
float *hidden_all = xmalloc((size_t)n * 2 * c->vision_inter * sizeof(float), "vision mlp");
float *projected_all = xmalloc((size_t)n * vd * sizeof(float), "vision projected");
float *scores = xmalloc((size_t)n * sizeof(float), "vision scores");
for (int layer = 0; layer < c->vision_layers; layer++) {
VisionBlock *b = &v->block[layer];
for (int i = 0; i < n; i++)
rms_into(normed + (size_t)i * vd, x + (size_t)i * vd, b->norm1.w, vd, 1e-6f);
mvb_rows(qkv_all, 3 * vd, &b->qkv_w, normed, vd, n);
for (int i = 0; i < n; i++) {
float *qkv = qkv_all + (size_t)i * 3 * vd;
for (int j = 0; j < 3 * vd; j++) qkv[j] += b->qkv_b.w[j];
for (int h = 0; h < heads; h++) {
const float *co = cosine + (size_t)i * half, *si = sine + (size_t)i * half;
for (int part = 0; part < 2; part++) {
const float *src = qkv + part * vd + (size_t)h * hd;
float *dst = (part ? k : q) + (size_t)i * vd + (size_t)h * hd;
for (int f = 0; f < half; f++) {
float a = src[f], bb = src[f + half];
dst[f] = a * co[f] - bb * si[f];
dst[f + half] = bb * co[f] + a * si[f];
}
}
memcpy(val + (size_t)i * vd + (size_t)h * hd, qkv + 2 * vd + (size_t)h * hd,
(size_t)hd * sizeof(float));
}
}
float scale = 1.0f / sqrtf((float)hd);
for (int i = 0; i < n; i++)
for (int h = 0; h < heads; h++) {
const float *qi = q + (size_t)i * vd + (size_t)h * hd;
float best = -INFINITY;
for (int j = 0; j < n; j++) {
const float *kj = k + (size_t)j * vd + (size_t)h * hd;
float dot = 0.0f;
for (int d = 0; d < hd; d++) dot += qi[d] * kj[d];
scores[j] = dot * scale;
if (scores[j] > best) best = scores[j];
}
float total = 0.0f;
for (int j = 0; j < n; j++) { scores[j] = expf(scores[j] - best); total += scores[j]; }
float *dst = attended + (size_t)i * vd + (size_t)h * hd;
for (int d = 0; d < hd; d++) dst[d] = 0.0f;
for (int j = 0; j < n; j++) {
const float *vj = val + (size_t)j * vd + (size_t)h * hd;
float weight = scores[j] / total;
for (int d = 0; d < hd; d++) dst[d] += weight * vj[d];
}
}
mvb_rows(projected_all, vd, &b->o_w, attended, vd, n);
for (int i = 0; i < n; i++) {
const float *projected = projected_all + (size_t)i * vd;
for (int j = 0; j < vd; j++) x[(size_t)i * vd + j] += projected[j] + b->o_b.w[j];
}
for (int i = 0; i < n; i++)
rms_into(normed + (size_t)i * vd, x + (size_t)i * vd, b->norm2.w, vd, 1e-6f);
mvb_rows(hidden_all, 2 * c->vision_inter, &b->w1, normed, vd, n);
for (int i = 0; i < n; i++) {
float *hidden = hidden_all + (size_t)i * 2 * c->vision_inter;
for (int j = 0; j < c->vision_inter; j++) {
float gate = hidden[j], up = hidden[c->vision_inter + j];
hidden[j] = (gate / (1.0f + expf(-gate))) * up;
}
}
mvb_rows(projected_all, vd, &b->w2, hidden_all, 2 * c->vision_inter, n);
for (int i = 0; i < n; i++)
for (int j = 0; j < vd; j++) x[(size_t)i * vd + j] += projected_all[(size_t)i * vd + j];
}
for (int i = 0; i < n; i++)
rms_into(normed + (size_t)i * vd, x + (size_t)i * vd, v->norm.w, vd, 1e-6f);
/* aligner: pad the grid, take ratio x ratio blocks channel-major, project */
int ratio = c->vision_ratio;
int blocks_h = (n_h + ratio - 1) / ratio, blocks_w = (n_w + ratio - 1) / ratio;
int block_dim = vd * ratio * ratio;
int nblocks = blocks_h * blocks_w;
float *blocks = xmalloc((size_t)nblocks * block_dim * sizeof(float), "aligner blocks");
float *projected = xmalloc((size_t)nblocks * c->dim * sizeof(float), "aligner hidden");
for (int bh = 0; bh < blocks_h; bh++)
for (int bw = 0; bw < blocks_w; bw++) {
float *block = blocks + (size_t)(bh * blocks_w + bw) * block_dim;
for (int ch = 0; ch < vd; ch++)
for (int kh = 0; kh < ratio; kh++)
for (int kw = 0; kw < ratio; kw++) {
int row = bh * ratio + kh, col = bw * ratio + kw;
float value = 0.0f; /* F.pad fills the ragged edge with zeros */
if (row < n_h && col < n_w)
value = normed[(size_t)(row * n_w + col) * vd + ch];
block[((size_t)ch * ratio + kh) * ratio + kw] = value;
}
}
mvb_rows(projected, c->dim, &v->align_w1, blocks, block_dim, nblocks);
for (size_t j = 0; j < (size_t)nblocks * c->dim; j++) {
float value = projected[j] + v->align_b1.w[j % c->dim];
/* exact GELU, matching torch's default (erf, not the tanh approximation) */
projected[j] = 0.5f * value * (1.0f + erff(value / sqrtf(2.0f)));
}
mvb_rows(out, c->dim, &v->align_w2, projected, c->dim, nblocks);
for (size_t j = 0; j < (size_t)nblocks * c->dim; j++) out[j] += v->align_b2.w[j % c->dim];
free(projected); free(blocks); free(scores); free(hidden_all); free(projected_all); free(attended);
free(normed); free(val); free(k); free(q); free(qkv_all); free(sine); free(cosine); free(x);
}
/* -------------------------------------------------------------- forward ---- */
/* The mix coefficients a sublayer computes are used by the NEXT one (model.py
* Block.forward): attention collapses with what the previous block's FFN produced,
* the FFN with what this attention produced. Keeping `pre` out of coli_hc_pre is
* exactly why this engine calls coli_hc_split_sinkhorn directly. */
static void hc_mixes(Model *m, const float *hc_fn, const float *scale, const float *base,
const float *stream, float *pre, float *post, float *comb) {
Cfg *c = &m->c;
int hc = c->hc_mult, dim = c->dim, flat = hc * dim, mix = (2 + hc) * hc;
double square = 0.0;
for (int i = 0; i < flat; i++) square += (double)stream[i] * stream[i];
float inverse = 1.0f / sqrtf((float)(square / flat) + c->norm_eps);
float *mixes = xmalloc((size_t)mix * sizeof(float), "hc mixes");
#pragma omp parallel for schedule(static)
for (int row = 0; row < mix; row++) {
const float *w = hc_fn + (size_t)row * flat;
float sum = 0.0f;
for (int i = 0; i < flat; i++) sum += w[i] * stream[i];
mixes[row] = sum * inverse;
}
if (coli_hc_split_sinkhorn(pre, post, comb, mixes, scale, base, hc, c->hc_iters, c->hc_eps) != 0) {
fprintf(stderr, "[hc] sinkhorn refused its arguments\n"); exit(1); }
free(mixes);
}
/* Run one chunk: the whole prompt on the first call, one token per call after.
* Returns the logits of the last position (the caller owns the buffer). */
/* `image_rows` (or NULL) is the aligner's output for a pending image and `image_at`
* the position where its span starts; the span's embeddings are replaced rather than
* looked up. `image_mask` marks those positions for the engram. */
/* ---------------------------------------------------------------- DSpark ---- */
static int argmax(const float *values, int n);
static void spec_load(Model *m, int ecap) {
Cfg *c = &m->c;
Spec *sp = &m->spec;
if (g_v41_mux_slots > 1) { /* speculation follows one conversation */
fprintf(stderr, "[v41] KV_SLOTS=%d: DSpark stays unloaded, nothing drafts\n", g_v41_mux_slots);
return;
}
/* 60%, and what it buys is not what I expected. The three cold runs:
*
* drafts off 24 forwards 116.6 s
* drafts on, guard at 60 16 forwards 99.3 s
* drafts on, guard disabled 9 forwards 111.5 s
*
* Speculating pays, and so does stopping. Fewer forwards is not less time:
* the early rounds batch six positions through a cold cache and win, the
* later ones pay six times the per-row attention for a cache that is now
* warm and would have served single tokens cheaply. The guard fires at 53%
* acceptance and turns 111 s into 99. It is not really measuring
* acceptance, it is measuring the moment batching stops being free -- but
* acceptance is the signal that moves with it, and it is the one we have.
*/
sp->min_accept = getenv("V41_DSPARK_MINACC") ? atoi(getenv("V41_DSPARK_MINACC")) : 60;
/* how many of the drafted tokens to put in front of the main model. The head always
* writes its whole block; verifying fewer of them trades acceptance for a smaller
* bill when a round is rejected early. */
sp->max_verify = getenv("V41_DSPARK_MAX") ? atoi(getenv("V41_DSPARK_MAX")) : 0;
if (sp->min_accept < 0) sp->min_accept = 0;
if (sp->min_accept > 100) sp->min_accept = 100;
/* The released config.json carries every dspark_* key and NOT n_mtp_layers:
* that one lives only in the vendor's inference/config.json, which a user
* who downloads the checkpoint never sees. Requiring it meant the draft
* head was silently skipped on the real weights while the tiny fixture,
* whose config we write ourselves, loaded it fine. Count the stages in the
* checkpoint instead, which is the one source that cannot disagree with
* itself: they are contiguous from zero. */
if (c->n_mtp <= 0 && c->spec_block > 0 && c->n_spec_targets > 0) {
char probe[128];
int stages = 0;
while (stages < 16) {
snprintf(probe, sizeof(probe), "mtp.%d.attn.wq_a.weight", stages);
if (!st_find(&m->S, probe)) break;
stages++;
}
if (stages > 0) {
c->n_mtp = stages;
fprintf(stderr, "[v41] n_mtp_layers is absent from config.json; the "
"checkpoint carries %d DSpark stages\n", stages);
}
}
if (c->n_mtp <= 0) return;
/* On by default, and the default is a measurement. On a 16-thread CPU
* server holding 68% of the experts, from a dropped page cache, the same
* 24-token turn takes 116.6 s with no drafting and 99.3 s with it. The
* first comparison I ran said the opposite, and it was wrong: the two runs
* had not started from the same cache state, which on an engine that reads
* its weights from disk decides everything. Cold or it is not a number.
*
* V41_DSPARK=0 turns the head off and does not load it. */
const char *flag = getenv("V41_DSPARK");
if (flag && !atoi(flag)) {
fprintf(stderr, "[v41] DSpark drafts off (V41_DSPARK=0)\n");
return;
}
if (!st_find(&m->S, "mtp.0.attn.wq_a.weight")) {
fprintf(stderr, "[v41] no DSpark head in this checkpoint: drafts off\n");
return;
}
int dim = c->dim, hd = c->head_dim, nh = c->n_heads, hc = c->hc_mult;
sp->stage = xmalloc((size_t)c->n_mtp * sizeof(Layer), "DSpark stages");
memset(sp->stage, 0, (size_t)c->n_mtp * sizeof(Layer));
sp->cache = xmalloc((size_t)c->n_mtp * sizeof(LCache), "DSpark expert caches");
char name[256];
#define MNAME(fmt, ...) (snprintf(name, sizeof(name), fmt, __VA_ARGS__), name)
for (int i = 0; i < c->n_mtp; i++) {
Layer *l = &sp->stage[i];
l->engram_index = -1;
w8_load(&m->S, &l->wq_a, MNAME("mtp.%d.attn.wq_a.weight", i), c->q_lora, dim);
w8_load(&m->S, &l->wq_b, MNAME("mtp.%d.attn.wq_b.weight", i), nh * hd, c->q_lora);
w8_load(&m->S, &l->wkv, MNAME("mtp.%d.attn.wkv.weight", i), hd, dim);
w8_load(&m->S, &l->wo_a, MNAME("mtp.%d.attn.wo_a.weight", i),
c->o_groups * c->o_lora, nh * hd / c->o_groups);
w8_load(&m->S, &l->wo_b, MNAME("mtp.%d.attn.wo_b.weight", i), dim, c->o_groups * c->o_lora);
wf_load(&m->S, &l->q_norm, MNAME("mtp.%d.attn.q_norm.weight", i), c->q_lora);
wf_load(&m->S, &l->kv_norm, MNAME("mtp.%d.attn.kv_norm.weight", i), hd);
wf_load(&m->S, &l->attn_sink, MNAME("mtp.%d.attn.attn_sink", i), nh);
wf_load(&m->S, &l->attn_norm, MNAME("mtp.%d.attn_norm.weight", i), dim);
wf_load(&m->S, &l->ffn_norm, MNAME("mtp.%d.ffn_norm.weight", i), dim);
int mix = (2 + hc) * hc;
wf_load(&m->S, &l->hc_attn_fn, MNAME("mtp.%d.hc_attn_fn", i), (int64_t)mix * hc * dim);
wf_load(&m->S, &l->hc_ffn_fn, MNAME("mtp.%d.hc_ffn_fn", i), (int64_t)mix * hc * dim);
wf_load(&m->S, &l->hc_attn_base, MNAME("mtp.%d.hc_attn_base", i), mix);
wf_load(&m->S, &l->hc_ffn_base, MNAME("mtp.%d.hc_ffn_base", i), mix);
wf_load(&m->S, &l->hc_attn_scale, MNAME("mtp.%d.hc_attn_scale", i), 3);
wf_load(&m->S, &l->hc_ffn_scale, MNAME("mtp.%d.hc_ffn_scale", i), 3);
wb_load(&m->S, &l->gate_w, MNAME("mtp.%d.ffn.gate.weight", i), c->spec_routed, dim);
wf_load(&m->S, &l->gate_bias, MNAME("mtp.%d.ffn.gate.bias", i), c->spec_routed);
w8_load(&m->S, &l->sh_w1, MNAME("mtp.%d.ffn.shared_experts.w1.weight", i), c->moe_inter, dim);
w8_load(&m->S, &l->sh_w3, MNAME("mtp.%d.ffn.shared_experts.w3.weight", i), c->moe_inter, dim);
w8_load(&m->S, &l->sh_w2, MNAME("mtp.%d.ffn.shared_experts.w2.weight", i), dim, c->moe_inter);
l->window = xmalloc((size_t)c->window * hd * sizeof(float), "DSpark window ring");
memset(l->window, 0, (size_t)c->window * hd * sizeof(float));
l->window_pos = xmalloc((size_t)c->window * sizeof(int), "DSpark window positions");
for (int k = 0; k < c->window; k++) l->window_pos[k] = -1;
cache_init(m, &sp->cache[i], ecap);
}
w8_load(&m->S, &sp->main_proj, "mtp.0.main_proj.weight", dim, dim * c->n_spec_targets);
wf_load(&m->S, &sp->main_norm, "mtp.0.main_norm.weight", dim);
int last = c->n_mtp - 1;
wf_load(&m->S, &sp->norm, MNAME("mtp.%d.norm.weight", last), dim);
wb_load(&m->S, &sp->markov_embed, MNAME("mtp.%d.markov_head.embed.weight", last),
c->vocab, c->markov_rank);
wb_load(&m->S, &sp->markov_out, MNAME("mtp.%d.markov_head.head.weight", last),
c->vocab, c->markov_rank);
wb_load(&m->S, &sp->conf_proj, MNAME("mtp.%d.confidence_head.proj.weight", last),
1, dim + c->markov_rank);
#undef MNAME
sp->active = 1;
fprintf(stderr, "[v41] DSpark on: %d stages, %d experts (%d routed), %d tokens per draft\n",
c->n_mtp, c->spec_routed, c->spec_activated, c->spec_block);
}
/* model.py DSparkAttention.forward.
*
* `main_x` carries the positions the main model has just committed: their keys go into
* this stage's ring, which is what makes the draft head see the real sequence rather
* than its own guesses. With `out` NULL that is all this does, which is the prefill
* call and the catch-up for positions that were accepted from a previous draft.
*
* The drafts then query that ring plus each other. Every draft row reads every other
* one, future included: the vendor hands the same index list to all of them, and it is
* right that it does -- the block is one guess, not a sequence anything committed to. */
static void spec_attention(Model *m, int stage, const float *x, int block, int start_pos,
const float *main_x, int main_rows, float *out) {
Cfg *c = &m->c;
Layer *l = &m->spec.stage[stage];
int dim = c->dim, hd = c->head_dim, nh = c->n_heads, rd = c->rope_dim, win = c->window;
const float *rope = m->rope_window; /* a stage never compresses */
double started = now_s();
int scratch_rows = main_rows > block ? main_rows : block;
float *scratch = xmalloc((size_t)scratch_rows * (c->q_lora > hd ? c->q_lora : hd)
* sizeof(float), "spec scratch");
mv8_rows(scratch, hd, &l->wkv, main_x, dim, main_rows);
for (int t = 0; t < main_rows; t++) {
int pos = start_pos + t;
float *slot = l->window + (size_t)(pos % win) * hd;
rms_into(slot, scratch + (size_t)t * hd, l->kv_norm.w, hd, c->norm_eps);
rope_apply(slot + hd - rd, rope, pos, rd, 0);
l->window_pos[pos % win] = pos;
}
if (!out) { free(scratch); m->t_attn += now_s() - started; return; }
float *qr = xmalloc((size_t)block * c->q_lora * sizeof(float), "spec q latent");
float *q = xmalloc((size_t)block * nh * hd * sizeof(float), "spec queries");
float *kv = xmalloc((size_t)block * hd * sizeof(float), "spec kv");
mv8_rows(scratch, c->q_lora, &l->wq_a, x, dim, block);
for (int i = 0; i < block; i++)
rms_into(qr + (size_t)i * c->q_lora, scratch + (size_t)i * c->q_lora,
l->q_norm.w, c->q_lora, c->norm_eps);
mv8_rows(q, nh * hd, &l->wq_b, qr, c->q_lora, block);
mv8_rows(scratch, hd, &l->wkv, x, dim, block);
for (int i = 0; i < block; i++) {
int pos = start_pos + main_rows + i;
for (int h = 0; h < nh; h++)
rope_apply(q + (size_t)i * nh * hd + (size_t)h * hd + hd - rd, rope, pos, rd, 0);
rms_into(kv + (size_t)i * hd, scratch + (size_t)i * hd, l->kv_norm.w, hd, c->norm_eps);
rope_apply(kv + (size_t)i * hd + hd - rd, rope, pos, rd, 0);
}
int filled = start_pos + main_rows;
int reach = win < filled ? win : filled;
int total = reach + block;
int *idx = xmalloc((size_t)total * sizeof(int), "spec indices");
for (int j = 0; j < reach; j++) idx[j] = j;
for (int j = 0; j < block; j++) idx[reach + j] = win + j;
float *kv_all = xmalloc((size_t)(win + block) * hd * sizeof(float), "spec kv all");
memcpy(kv_all, l->window, (size_t)win * hd * sizeof(float));
memcpy(kv_all + (size_t)win * hd, kv, (size_t)block * hd * sizeof(float));
float *heads = xmalloc((size_t)block * nh * hd * sizeof(float), "spec heads");
float *head_score = xmalloc((size_t)nh * total * sizeof(float), "spec scores");
float scale = 1.0f / sqrtf((float)hd);
for (int i = 0; i < block; i++) {
int pos = start_pos + main_rows + i;
float *row_heads = heads + (size_t)i * nh * hd;
#pragma omp parallel for schedule(static)
for (int h = 0; h < nh; h++)
sparse_attend(row_heads + (size_t)h * hd, q + (size_t)i * nh * hd + (size_t)h * hd,
kv_all, idx, total, nh, hd, l->attn_sink.w[h], scale,
head_score + (size_t)h * total);
for (int h = 0; h < nh; h++)
rope_apply(row_heads + (size_t)h * hd + hd - rd, rope, pos, rd, 1);
}
attn_project_out_rows(m, l, heads, block, out);
m->t_attn += now_s() - started;
free(head_score);
free(heads); free(kv_all); free(idx); free(kv); free(q); free(qr); free(scratch);
}
/* One DSpark round. `main_rows` positions have just been committed by the main model,
* starting at `start_pos`; `token` is the one it produced from the last of them.
*
* Returns how many drafts were written, 0 when there is nothing to draft from -- the
* prefill call, which exists only to fill the stages' rings from the prompt. */
static int spec_step(Model *m, int token, int start_pos, int main_rows,
int *draft, float *confidence) {
Cfg *c = &m->c;
Spec *sp = &m->spec;
if (!sp->active || !m->main_hidden || m->main_hidden_rows < main_rows) return 0;
int dim = c->dim, hc = c->hc_mult, block = c->spec_block;
int targets = c->n_spec_targets, rank = c->markov_rank, vocab = c->vocab;
double started = now_s();
float *scratch = xmalloc((size_t)dim * sizeof(float), "spec projection");
float *main_x = xmalloc((size_t)main_rows * dim * sizeof(float), "spec main input");
for (int t = 0; t < main_rows; t++) {
mv8(scratch, &sp->main_proj, m->main_hidden + (size_t)t * targets * dim);
rms_into(main_x + (size_t)t * dim, scratch, sp->main_norm.w, dim, c->norm_eps);
}
/* Seed only, no drafting: that is what a NULL draft buffer asks for. Until
* prefix reuse existed this was spelled `start_pos == 0`, because the only
* forward that ever started at position 0 was the prompt's. A prefill that
* resumes mid-sequence is the same call at a different position, and it has
* to seed the stages at THAT position -- writing the tail's keys into slots
* 0..rows-1 would leave the stages describing a sequence nobody fed. */
if (!draft) {
for (int stage = 0; stage < c->n_mtp; stage++)
spec_attention(m, stage, NULL, 0, start_pos, main_x, main_rows, NULL);
free(main_x); free(scratch);
return 0;
}
/* A cold run still declines to draft on the first decode step, where
* start_pos is still the prefill's 0 and the stages were seeded a moment
* ago. Kept separate from the guard above so this change does not move when
* a cold conversation starts drafting; a resumed one starts immediately,
* which costs a draft round's reads and can only change the acceptance
* rate, never a token -- every draft is verified before it is emitted. */
if (start_pos == 0) { free(main_x); free(scratch); return 0; }
float *h = xmalloc((size_t)block * hc * dim * sizeof(float), "spec streams");
for (int i = 0; i < block; i++) {
int id = i == 0 ? token : c->spec_noise;
const uint16_t *row = m->embed.w + (size_t)id * dim;
for (int copy = 0; copy < hc; copy++)
for (int j = 0; j < dim; j++)
h[((size_t)i * hc + copy) * dim + j] = bf16_to_f32(row[j]);
}
float *pre_mix = xmalloc((size_t)block * hc * sizeof(float), "spec pre mix");
for (int i = 0; i < block; i++)
for (int copy = 0; copy < hc; copy++) pre_mix[(size_t)i * hc + copy] = copy == 0 ? 1.0f : 0.0f;
float *branch_in = xmalloc((size_t)block * dim * sizeof(float), "spec sublayer in");
float *branch_out = xmalloc((size_t)block * dim * sizeof(float), "spec sublayer out");
float *residual = xmalloc((size_t)block * hc * dim * sizeof(float), "spec residual");
float *pre = xmalloc((size_t)block * hc * sizeof(float), "spec hc pre");
float *post = xmalloc((size_t)block * hc * sizeof(float), "spec hc post");
float *comb = xmalloc((size_t)block * hc * hc * sizeof(float), "spec hc comb");
float *collapsed = xmalloc((size_t)dim * sizeof(float), "spec collapsed");
for (int stage = 0; stage < c->n_mtp; stage++) {
Layer *l = &sp->stage[stage];
memcpy(residual, h, (size_t)block * hc * dim * sizeof(float));
for (int i = 0; i < block; i++) {
hc_mixes(m, l->hc_attn_fn.w, l->hc_attn_scale.w, l->hc_attn_base.w,
h + (size_t)i * hc * dim, pre + (size_t)i * hc,
post + (size_t)i * hc, comb + (size_t)i * hc * hc);
const float *mix = pre_mix + (size_t)i * hc;
for (int j = 0; j < dim; j++) {
float sum = 0.0f;
for (int copy = 0; copy < hc; copy++)
sum += mix[copy] * h[((size_t)i * hc + copy) * dim + j];
collapsed[j] = sum;
}
rms_into(branch_in + (size_t)i * dim, collapsed, l->attn_norm.w, dim, c->norm_eps);
}
spec_attention(m, stage, branch_in, block, start_pos, main_x, main_rows, branch_out);
for (int i = 0; i < block; i++)
coli_hc_post(h + (size_t)i * hc * dim, branch_out + (size_t)i * dim,
residual + (size_t)i * hc * dim, post + (size_t)i * hc,
comb + (size_t)i * hc * hc, hc, dim);
memcpy(pre_mix, pre, (size_t)block * hc * sizeof(float));
memcpy(residual, h, (size_t)block * hc * dim * sizeof(float));
for (int i = 0; i < block; i++) {
hc_mixes(m, l->hc_ffn_fn.w, l->hc_ffn_scale.w, l->hc_ffn_base.w,
h + (size_t)i * hc * dim, pre + (size_t)i * hc,
post + (size_t)i * hc, comb + (size_t)i * hc * hc);
const float *mix = pre_mix + (size_t)i * hc;
for (int j = 0; j < dim; j++) {
float sum = 0.0f;
for (int copy = 0; copy < hc; copy++)
sum += mix[copy] * h[((size_t)i * hc + copy) * dim + j];
collapsed[j] = sum;
}
rms_into(branch_in + (size_t)i * dim, collapsed, l->ffn_norm.w, dim, c->norm_eps);
}
moe_run_at(m, l, &sp->cache[stage], "mtp", stage, c->spec_routed, c->spec_activated,
branch_in, block, branch_out, 1);
for (int i = 0; i < block; i++)
coli_hc_post(h + (size_t)i * hc * dim, branch_out + (size_t)i * dim,
residual + (size_t)i * hc * dim, post + (size_t)i * hc,
comb + (size_t)i * hc * hc, hc, dim);
memcpy(pre_mix, pre, (size_t)block * hc * sizeof(float));
}
/* the head, then the Markov chain over the block: each position is biased by the
* token drafted for the one before it, which is the only thing tying the block
* together -- the stages saw the noise id at every position but the first */
float *final_x = xmalloc((size_t)block * dim * sizeof(float), "spec final");
float *logits = xmalloc((size_t)block * vocab * sizeof(float), "spec logits");
float *bias = xmalloc((size_t)vocab * sizeof(float), "spec markov bias");
float *embed = xmalloc((size_t)rank * sizeof(float), "spec markov embed");
float *joined = xmalloc((size_t)(dim + rank) * sizeof(float), "spec confidence input");
for (int i = 0; i < block; i++) {
const float *mix = pre_mix + (size_t)i * hc;
float *x = final_x + (size_t)i * dim;
for (int j = 0; j < dim; j++) {
float sum = 0.0f;
for (int copy = 0; copy < hc; copy++)
sum += mix[copy] * h[((size_t)i * hc + copy) * dim + j];
x[j] = sum;
}
rms_into(collapsed, x, sp->norm.w, dim, c->norm_eps);
mvb(logits + (size_t)i * vocab, &m->head, collapsed);
}
draft[0] = token;
for (int i = 0; i < block; i++) {
const uint16_t *row = sp->markov_embed.w + (size_t)draft[i] * rank;
for (int j = 0; j < rank; j++) embed[j] = bf16_to_f32(row[j]);
mvb(bias, &sp->markov_out, embed);
float *row_logits = logits + (size_t)i * vocab;
for (int v = 0; v < vocab; v++) row_logits[v] += bias[v];
draft[i + 1] = argmax(row_logits, vocab); /* greedy: the drafts are verified */
memcpy(joined, final_x + (size_t)i * dim, (size_t)dim * sizeof(float));
memcpy(joined + dim, embed, (size_t)rank * sizeof(float));
float score = 0.0f;
mvb(&score, &sp->conf_proj, joined);
confidence[i] = score;
}
sp->rounds++;
m->t_spec += now_s() - started;
free(joined); free(embed); free(bias); free(logits); free(final_x);
free(collapsed); free(comb); free(post); free(pre); free(residual);
free(branch_out); free(branch_in); free(pre_mix); free(h);
free(main_x); free(scratch);
return block;
}
/* `spec_batch` dice che queste righe sono un lotto di verifica speculativa, e
* cambia il calcolo: schedule di pubblicazione per riga e buffer di rollback.
* `keep_rows` dice soltanto di TENERE i logit di ogni posizione invece della
* sola ultima, e non cambia niente altro. Erano lo stesso parametro, e usarlo
* per leggere il prefill faceva prendere a un prefill lo schedule del decode:
* righe diverse dell'indice, top-k diverso, numeri plausibili e sbagliati.
* Sul fixture minuscolo uscivano NaN, che e come si e visto. */
static void forward_full(Model *m, const int *ids, int n, float *logits, int spec_batch,
const float *image_rows, int image_at, int image_h, int image_w,
const uint8_t *image_mask, int keep_rows);
/* Fotografia dello stato e lettura del prefill, per il canale logprobs.
*
* Qui e piu semplice che su qwen36: questo motore e ad attenzione pura, quindi
* non c'e stato ricorrente da salvare. Riavvolgere vuol dire solo dichiarare
* che il prefisso tenuto e quello fotografato: le righe KV di quelle posizioni
* non sono state toccate da nessuno. Si salvano gli id e il vettore di logit
* finale, che e il predittore del primo token fresco. */
static ColiPinPool g_pins; /* piu scatti annidati, vedi pin_pool.h */
static const float *g_pin_logit = NULL;
static int g_pin_use_logit = 0;
static void v41_echo(const char *id, int pos, int token, const float *lo, int V, int k,
Tok *tokenizer) {
char tail[1024]; coli_logprob_tail(tail, sizeof tail, lo, V, token, k);
char piece[512]; int n = tok_decode(tokenizer, &token, 1, piece, (int)sizeof(piece));
if (n < 0) n = 0;
printf("ECHO %s %d %d%s\n", id, n, pos, tail);
if (n > 0) fwrite(piece, 1, (size_t)n, stdout);
fputc('\n', stdout); fflush(stdout);
}
static void forward(Model *m, const int *ids, int n, float *logits) {
forward_full(m, ids, n, logits, 0, NULL, -1, 0, 0, NULL, 0);
}
static void forward_with_image(Model *m, const int *ids, int n, float *logits,
const float *image_rows, int image_at, int image_h, int image_w,
const uint8_t *image_mask) {
forward_full(m, ids, n, logits, 0, image_rows, image_at, image_h, image_w, image_mask, 0);
}
/* Several positions past the prompt, with one row of logits each: what a speculative
* step verifies its drafts with. Every piece of per-position state below -- the window
* ring's position map, the compressor's group slots, the indexer's reach, the index
* keys each row reads -- is written so that this is the same computation as the same
* tokens decoded one at a time, because a draft is only worth anything if accepting it
* is indistinguishable from having generated it. */
static void forward_batch(Model *m, const int *ids, int n, float *logits) {
forward_full(m, ids, n, logits, 1, NULL, -1, 0, 0, NULL, 0);
}
#ifdef COLI_VULKAN
#include "deepseek_v41_chain.h" /* the layers as a dense chain on the device (COLI_VK_CHAIN) */
#endif
static void forward_full(Model *m, const int *ids, int n, float *logits, int spec_batch,
const float *image_rows, int image_at, int image_h, int image_w,
const uint8_t *image_mask, int keep_rows) {
Cfg *c = &m->c;
int dim = c->dim, hc = c->hc_mult;
int start_pos = m->pos;
/* The index-key schedule for a batched step: which layer publishes at each of its
* sub-steps. It has to be known before the layers run, because a layer that comes
* later in the stack still publishes before this layer's later rows read. */
m->pub_before = m->published_index_k;
m->pub_before_layer = m->published_index_layer;
m->pub_rows = 0;
/* Only the speculative verify batch is ever rolled back (spec_verify is its
* only caller), and only it may write the per-layer undo buffers. */
m->rollback_save = spec_batch;
/* Several positions at once is either the speculative verify batch, which
* needs this per-row schedule because its rows are consecutive decode steps,
* or a prefill, which reads its index owner's keys for the whole chunk. The
* condition used to be `start_pos > 0`, which picked out the speculative
* case only because a prefill always started at position 0. A prefill that
* resumes mid-sequence is many rows past position 0 and is still a prefill:
* given the decode schedule it reads a different layer's keys than the same
* positions read when they were prefilled cold, and picks a different
* index top-k. `spec_batch` is what actually distinguishes the two --
* spec_verify is its only caller. */
if (n > 1 && spec_batch) {
m->pub_layer = realloc(m->pub_layer, (size_t)n * sizeof(int));
if (!m->pub_layer) { fprintf(stderr, "OOM sizing the publish schedule\n"); exit(1); }
m->pub_rows = n;
for (int t = 0; t < n; t++) {
m->pub_layer[t] = -1;
for (int layer = 0; layer < c->n_layers; layer++)
if (layer_publishes(c, layer, start_pos + t)) m->pub_layer[t] = layer;
}
}
if (start_pos + n > c->max_positions) {
fprintf(stderr, "CONTEXT_EXCEEDED %d %d\n", start_pos + n, c->max_positions);
exit(1);
}
if (m->engram.active) engram_push(&m->engram, ids, n, image_mask);
float *h = xmalloc((size_t)n * hc * dim * sizeof(float), "residual streams");
float *embedded = xmalloc((size_t)dim * sizeof(float), "embedding row");
for (int t = 0; t < n; t++) {
const uint16_t *row = m->embed.w + (size_t)ids[t] * dim;
for (int i = 0; i < dim; i++) embedded[i] = bf16_to_f32(row[i]);
/* An image span carries the placeholder id at every position; what each one
* MEANS follows from the grid, exactly as image_processor.py lays it out:
* [start] then h rows of (w image tokens + newline) then [end].
* The delimiters take learned embeddings, the image slots take the aligner's
* rows in order. Reconstructing it from the grid keeps the wire protocol at
* one IMAGE frame plus placeholders, with no parallel token-type channel to
* fall out of step with the ids. */
if (image_rows && m->vision && t >= image_at) {
int offset = t - image_at;
int span = 1 + (image_w + 1) * image_h + 1;
if (offset < span) {
const float *source = NULL;
if (offset == 0) source = m->vision->image_start.w;
else if (offset == span - 1) source = m->vision->image_end.w;
else {
int inner = offset - 1;
int row_index = inner / (image_w + 1), column = inner % (image_w + 1);
source = column == image_w ? m->vision->image_newline.w
: image_rows + (size_t)(row_index * image_w + column) * dim;
}
memcpy(embedded, source, (size_t)dim * sizeof(float));
}
}
for (int copy = 0; copy < hc; copy++)
memcpy(h + ((size_t)t * hc + copy) * dim, embedded, (size_t)dim * sizeof(float));
}
free(embedded);
float *pre_mix = xmalloc((size_t)n * hc * sizeof(float), "pre mix");
for (int t = 0; t < n; t++)
for (int copy = 0; copy < hc; copy++) pre_mix[(size_t)t * hc + copy] = copy == 0 ? 1.0f : 0.0f;
/* V41_TRACE=2 follows the FIRST row instead of the last: on a speculative step
* that is the position the sequential decode would have run on its own, which is
* what a divergence has to be compared against. */
int trace_row = (g_trace >= 2 && n > 1) ? 0 : n - 1;
float *branch_in = xmalloc((size_t)n * dim * sizeof(float), "sublayer input");
float *branch_out = xmalloc((size_t)n * dim * sizeof(float), "sublayer output");
float *residual = xmalloc((size_t)n * hc * dim * sizeof(float), "residual copy");
float *pre = xmalloc((size_t)n * hc * sizeof(float), "hc pre");
float *post = xmalloc((size_t)n * hc * sizeof(float), "hc post");
float *comb = xmalloc((size_t)n * hc * hc * sizeof(float), "hc comb");
float *collapsed = xmalloc((size_t)dim * sizeof(float), "collapsed stream");
/* What the DSpark head reads: the attention input of its target layers, the hc
* copies averaged. Kept even when no draft head is loaded is pointless, so it is
* gated on the model actually having one. */
int targets = m->spec.active ? c->n_spec_targets : 0;
if (targets) {
m->main_hidden = realloc(m->main_hidden,
(size_t)n * targets * dim * sizeof(float));
if (!m->main_hidden) { fprintf(stderr, "OOM sizing the DSpark input\n"); exit(1); }
m->main_hidden_rows = n;
}
int rows_out = spec_batch || keep_rows;
#ifdef COLI_VULKAN
/* the dense chain runs its layers on the device (every layer, or a partial chain's
* first N, the streams of every row then back here); 0: the CPU runs them all */
int chained = v41c_forward(m, h, pre_mix, n, start_pos, spec_batch, rows_out);
if (!chained) v41c_cpu_step(m, start_pos);
#else
int chained = 0;
#endif
for (int layer = chained; layer < c->n_layers; layer++) {
Layer *l = &m->L[layer];
if (l->engram_index >= 0) engram_run(m, layer, h, n, start_pos);
for (int k = 0; k < targets; k++) {
if (c->spec_targets[k] != layer) continue;
for (int t = 0; t < n; t++) {
float *dest = m->main_hidden + ((size_t)t * targets + k) * dim;
for (int i = 0; i < dim; i++) {
float sum = 0.0f;
for (int copy = 0; copy < hc; copy++)
sum += h[((size_t)t * hc + copy) * dim + i];
dest[i] = sum / (float)hc;
}
}
}
trace("stream", layer, h + (size_t)trace_row * hc * dim, hc * dim);
memcpy(residual, h, (size_t)n * hc * dim * sizeof(float));
for (int t = 0; t < n; t++) {
hc_mixes(m, l->hc_attn_fn.w, l->hc_attn_scale.w, l->hc_attn_base.w,
h + (size_t)t * hc * dim, pre + (size_t)t * hc,
post + (size_t)t * hc, comb + (size_t)t * hc * hc);
const float *mix = pre_mix + (size_t)t * hc;
for (int i = 0; i < dim; i++) {
float sum = 0.0f;
for (int copy = 0; copy < hc; copy++)
sum += mix[copy] * h[((size_t)t * hc + copy) * dim + i];
collapsed[i] = sum;
}
rms_into(branch_in + (size_t)t * dim, collapsed, l->attn_norm.w, dim, c->norm_eps);
}
trace("attn_in", layer, branch_in + (size_t)trace_row * dim, dim);
trace("attn_pre", layer, pre + (size_t)trace_row * hc, hc);
trace("attn_post", layer, post + (size_t)trace_row * hc, hc);
trace("attn_comb", layer, comb + (size_t)trace_row * hc * hc, hc * hc);
attention_run(m, layer, branch_in, n, start_pos, branch_out);
trace("attn_out", layer, branch_out + (size_t)trace_row * dim, dim);
for (int t = 0; t < n; t++)
coli_hc_post(h + (size_t)t * hc * dim, branch_out + (size_t)t * dim,
residual + (size_t)t * hc * dim, post + (size_t)t * hc,
comb + (size_t)t * hc * hc, hc, dim);
/* the attention's own `pre` collapses the FFN input */
memcpy(pre_mix, pre, (size_t)n * hc * sizeof(float));
memcpy(residual, h, (size_t)n * hc * dim * sizeof(float));
for (int t = 0; t < n; t++) {
hc_mixes(m, l->hc_ffn_fn.w, l->hc_ffn_scale.w, l->hc_ffn_base.w,
h + (size_t)t * hc * dim, pre + (size_t)t * hc,
post + (size_t)t * hc, comb + (size_t)t * hc * hc);
const float *mix = pre_mix + (size_t)t * hc;
for (int i = 0; i < dim; i++) {
float sum = 0.0f;
for (int copy = 0; copy < hc; copy++)
sum += mix[copy] * h[((size_t)t * hc + copy) * dim + i];
collapsed[i] = sum;
}
rms_into(branch_in + (size_t)t * dim, collapsed, l->ffn_norm.w, dim, c->norm_eps);
}
/* Every position's FFN input is ready before any expert is read, which is
* what lets the block share both the reads and the matmuls. */
moe_run(m, layer, branch_in, n, branch_out);
if (trace_row >= 0 && trace_row < n) {
trace("ffn_in", layer, branch_in + (size_t)trace_row * dim, dim);
trace("ffn_out", layer, branch_out + (size_t)trace_row * dim, dim);
}
for (int t = 0; t < n; t++)
coli_hc_post(h + (size_t)t * hc * dim, branch_out + (size_t)t * dim,
residual + (size_t)t * hc * dim, post + (size_t)t * hc,
comb + (size_t)t * hc * hc, hc, dim);
memcpy(pre_mix, pre, (size_t)n * hc * sizeof(float)); /* the FFN's, for the next block */
}
/* the last block's FFN mix collapses the stream one final time */
for (int t = rows_out ? 0 : n - 1; t < n; t++) {
const float *mix = pre_mix + (size_t)t * hc;
for (int i = 0; i < dim; i++) {
float sum = 0.0f;
for (int copy = 0; copy < hc; copy++)
sum += mix[copy] * h[((size_t)t * hc + copy) * dim + i];
collapsed[i] = sum;
}
if (t == n - 1) trace("final", -1, collapsed, dim);
rms_into(branch_in, collapsed, m->norm.w, dim, c->norm_eps);
mvb(logits + (size_t)(rows_out ? t : 0) * c->vocab, &m->head, branch_in);
}
trace("logits", -1, logits + (size_t)(rows_out ? n - 1 : 0) * c->vocab, c->vocab);
#ifdef COLI_VULKAN
{ /* DUMP=<path>: every logits row this forward computed, for the Vulkan gates */
static FILE *dump; static int dump_init;
if (!dump_init) { const char *d = getenv("DUMP"); dump_init = 1; if (d && *d) dump = fopen(d, "wb"); }
if (dump) { fwrite(logits, sizeof(float), (size_t)(rows_out ? n : 1) * c->vocab, dump); fflush(dump); }
}
#endif
m->pos += n;
/* Recorded where the tokens entered the state, and only for the MAIN stream:
* the draft stages read this window and never write it, so a speculative
* round adds nothing here until spec_verify commits through forward_batch. */
if (image_rows || image_mask) kv_prefix_taint(&m->kvp);
kv_prefix_record(&m->kvp, ids, start_pos, n);
m->last_start = start_pos;
m->last_rows = n;
m->forwards++;
free(collapsed); free(comb); free(post); free(pre);
free(residual); free(branch_out); free(branch_in); free(pre_mix); free(h);
}
static int argmax(const float *values, int n) {
int best = 0;
for (int i = 1; i < n; i++) if (values[i] > values[best]) best = i;
return best;
}
/* ---------------------------------------------------------------- serve ---- */
/* ---- several conversations at once (KV_SLOTS>1, serve_mux) -------------------
* Each conversation owns a V41Seq: every layer's window ring and its positions, its
* compressed KV, index keys and partial compressor group, the engram history, what
* an index source last published (its keys and the shared top-k), the candidate
* mask, where it stands and the record of the tokens it holds. v41_mux_bind trades
* the Model's set for a parked one. A decode step runs one forward over a row of each
* conversation (forward_rows): the hyper-connection mixes, the routed experts and the
* head once over all rows, the attention and the engram row by row, each with its
* own conversation bound (attention_run reads the cross-layer state from the Model,
* so it is bound whole: pointers, not copies). Nothing drafts. */
static V41Seq *g_v41_mux_seq; /* [slots]: the conversations the Model does not hold */
static int g_v41_mux_cur; /* the slot the Model holds, -1 when every one is parked */
static void v41_seq_swap(Model *m, V41Seq *q) {
#define V41_SWAP(T,a,b) do { T t_ = (a); (a) = (b); (b) = t_; } while (0)
for (int i = 0; i < m->c.n_layers; i++) {
Layer *l = &m->L[i];
V41_SWAP(float *, l->window, q->window[i]); V41_SWAP(int *, l->window_pos, q->window_pos[i]);
V41_SWAP(float *, l->ckv, q->ckv[i]); V41_SWAP(float *, l->ikey, q->ikey[i]);
V41_SWAP(float *, l->cstate_kv, q->cstate_kv[i]); V41_SWAP(float *, l->cstate_score, q->cstate_score[i]);
}
V41_SWAP(int, m->pos, q->pos); V41_SWAP(kv_prefix, m->kvp, q->kvp);
V41_SWAP(int32_t *, m->engram.history, q->history);
V41_SWAP(int, m->engram.history_len, q->history_len); V41_SWAP(int, m->engram.history_cap, q->history_cap);
V41_SWAP(const float *, m->published_index_k, q->published_index_k);
V41_SWAP(int, m->published_index_layer, q->published_index_layer);
V41_SWAP(int *, m->shared_topk, q->shared_topk);
V41_SWAP(int, m->shared_topk_rows, q->shared_topk_rows); V41_SWAP(int, m->shared_topk_width, q->shared_topk_width);
V41_SWAP(uint8_t *, m->candidates, q->candidates);
V41_SWAP(int, m->candidate_width, q->candidate_width); V41_SWAP(int, m->candidate_rows, q->candidate_rows);
#undef V41_SWAP
}
static void v41_mux_bind(Model *m, int slot) {
if (g_v41_mux_cur == slot) return;
if (g_v41_mux_cur >= 0) v41_seq_swap(m, &g_v41_mux_seq[g_v41_mux_cur]);
if (slot >= 0) v41_seq_swap(m, &g_v41_mux_seq[slot]);
g_v41_mux_cur = slot;
}
/* A conversation's state of its own, as model_load gives the Model's and model_reset
* clears it; slot 0's place holds only the pointer arrays. */
static void v41_seq_alloc(Model *m, V41Seq *q, int with_state) {
Cfg *c = &m->c; int hd = c->head_dim;
memset(q, 0, sizeof *q);
q->window = xcalloc((size_t)c->n_layers, sizeof(float *), "seq windows");
q->window_pos = xcalloc((size_t)c->n_layers, sizeof(int *), "seq window positions");
q->ckv = xcalloc((size_t)c->n_layers, sizeof(float *), "seq compressed kv");
q->ikey = xcalloc((size_t)c->n_layers, sizeof(float *), "seq index keys");
q->cstate_kv = xcalloc((size_t)c->n_layers, sizeof(float *), "seq compressor groups");
q->cstate_score = xcalloc((size_t)c->n_layers, sizeof(float *), "seq compressor scores");
q->published_index_layer = m->published_index_layer;
if (!with_state) return;
kv_prefix_alloc(&q->kvp, c->max_positions);
for (int i = 0; i < c->n_layers; i++) {
q->window[i] = xcalloc((size_t)c->window * hd, sizeof(float), "window ring");
q->window_pos[i] = xmalloc((size_t)c->window * sizeof(int), "window positions");
for (int k = 0; k < c->window; k++) q->window_pos[i][k] = -1;
int ratio = c->compress_ratio[i];
if (c->kv_source[i]) {
if (ratio > 1) {
q->cstate_kv[i] = xcalloc((size_t)ratio * hd, sizeof(float), "compressor group");
q->cstate_score[i] = xmalloc((size_t)ratio * hd * sizeof(float), "compressor scores");
for (int k = 0; k < ratio * hd; k++) q->cstate_score[i][k] = -INFINITY;
}
int slots = c->max_positions / (ratio > 0 ? ratio : 1);
q->ckv[i] = xcalloc((size_t)slots * hd, sizeof(float), "compressed kv");
q->ikey[i] = xcalloc((size_t)slots * c->index_head_dim, sizeof(float), "index keys");
}
}
}
/* One decode step of several conversations: row t is the token ids[t] of the
* conversation in slot slots[t], at its next position. The streams, their mixes,
* the routed experts and the head run once over the n rows; the engram and the
* attention run row by row with that row's conversation bound. The CPU kernels give
* a row the same bits whatever n is, so each conversation gets the logits it would
* alone: [n, V] into logits. */
static void forward_rows(Model *m, const int *slots, const int *ids, int n, float *logits) {
Cfg *c = &m->c;
int dim = c->dim, hc = c->hc_mult;
int *pos = xmalloc((size_t)n * sizeof(int), "row positions");
for (int t = 0; t < n; t++) {
v41_mux_bind(m, slots[t]);
pos[t] = m->pos;
if (pos[t] + 1 > c->max_positions) { fprintf(stderr, "CONTEXT_EXCEEDED %d %d\n", pos[t] + 1, c->max_positions); exit(1); }
if (m->engram.active) engram_push(&m->engram, ids + t, 1, NULL);
}
m->rollback_save = 0; m->pub_rows = 0;
float *h = xmalloc((size_t)n * hc * dim * sizeof(float), "residual streams");
for (int t = 0; t < n; t++) {
const uint16_t *row = m->embed.w + (size_t)ids[t] * dim;
for (int i = 0; i < dim; i++) h[(size_t)t * hc * dim + i] = bf16_to_f32(row[i]);
for (int copy = 1; copy < hc; copy++)
memcpy(h + ((size_t)t * hc + copy) * dim, h + (size_t)t * hc * dim, (size_t)dim * sizeof(float));
}
float *pre_mix = xmalloc((size_t)n * hc * sizeof(float), "pre mix");
for (int t = 0; t < n; t++)
for (int copy = 0; copy < hc; copy++) pre_mix[(size_t)t * hc + copy] = copy == 0 ? 1.0f : 0.0f;
float *branch_in = xmalloc((size_t)n * dim * sizeof(float), "sublayer input");
float *branch_out = xmalloc((size_t)n * dim * sizeof(float), "sublayer output");
float *residual = xmalloc((size_t)n * hc * dim * sizeof(float), "residual copy");
float *pre = xmalloc((size_t)n * hc * sizeof(float), "hc pre");
float *post = xmalloc((size_t)n * hc * sizeof(float), "hc post");
float *comb = xmalloc((size_t)n * hc * hc * sizeof(float), "hc comb");
float *collapsed = xmalloc((size_t)dim * sizeof(float), "collapsed stream");
for (int layer = 0; layer < c->n_layers; layer++) {
Layer *l = &m->L[layer];
if (l->engram_index >= 0)
for (int t = 0; t < n; t++) { v41_mux_bind(m, slots[t]); engram_run(m, layer, h + (size_t)t * hc * dim, 1, pos[t]); }
memcpy(residual, h, (size_t)n * hc * dim * sizeof(float));
for (int t = 0; t < n; t++) {
hc_mixes(m, l->hc_attn_fn.w, l->hc_attn_scale.w, l->hc_attn_base.w,
h + (size_t)t * hc * dim, pre + (size_t)t * hc,
post + (size_t)t * hc, comb + (size_t)t * hc * hc);
const float *mix = pre_mix + (size_t)t * hc;
for (int i = 0; i < dim; i++) {
float sum = 0.0f;
for (int copy = 0; copy < hc; copy++) sum += mix[copy] * h[((size_t)t * hc + copy) * dim + i];
collapsed[i] = sum;
}
rms_into(branch_in + (size_t)t * dim, collapsed, l->attn_norm.w, dim, c->norm_eps);
}
for (int t = 0; t < n; t++) {
v41_mux_bind(m, slots[t]);
attention_run(m, layer, branch_in + (size_t)t * dim, 1, pos[t], branch_out + (size_t)t * dim);
}
for (int t = 0; t < n; t++)
coli_hc_post(h + (size_t)t * hc * dim, branch_out + (size_t)t * dim,
residual + (size_t)t * hc * dim, post + (size_t)t * hc,
comb + (size_t)t * hc * hc, hc, dim);
memcpy(pre_mix, pre, (size_t)n * hc * sizeof(float));
memcpy(residual, h, (size_t)n * hc * dim * sizeof(float));
for (int t = 0; t < n; t++) {
hc_mixes(m, l->hc_ffn_fn.w, l->hc_ffn_scale.w, l->hc_ffn_base.w,
h + (size_t)t * hc * dim, pre + (size_t)t * hc,
post + (size_t)t * hc, comb + (size_t)t * hc * hc);
const float *mix = pre_mix + (size_t)t * hc;
for (int i = 0; i < dim; i++) {
float sum = 0.0f;
for (int copy = 0; copy < hc; copy++) sum += mix[copy] * h[((size_t)t * hc + copy) * dim + i];
collapsed[i] = sum;
}
rms_into(branch_in + (size_t)t * dim, collapsed, l->ffn_norm.w, dim, c->norm_eps);
}
moe_run(m, layer, branch_in, n, branch_out);
for (int t = 0; t < n; t++)
coli_hc_post(h + (size_t)t * hc * dim, branch_out + (size_t)t * dim,
residual + (size_t)t * hc * dim, post + (size_t)t * hc,
comb + (size_t)t * hc * hc, hc, dim);
memcpy(pre_mix, pre, (size_t)n * hc * sizeof(float));
}
for (int t = 0; t < n; t++) {
const float *mix = pre_mix + (size_t)t * hc;
for (int i = 0; i < dim; i++) {
float sum = 0.0f;
for (int copy = 0; copy < hc; copy++) sum += mix[copy] * h[((size_t)t * hc + copy) * dim + i];
collapsed[i] = sum;
}
rms_into(branch_in, collapsed, m->norm.w, dim, c->norm_eps);
mvb(logits + (size_t)t * c->vocab, &m->head, branch_in);
}
for (int t = 0; t < n; t++) {
v41_mux_bind(m, slots[t]);
kv_prefix_record(&m->kvp, ids + t, pos[t], 1);
m->pos = pos[t] + 1;
m->last_start = pos[t]; m->last_rows = 1;
}
m->forwards++;
free(pos); free(collapsed); free(comb); free(post); free(pre);
free(residual); free(branch_out); free(branch_in); free(pre_mix); free(h);
}
/* Every turn starts from a clean state: the window rings, the compressed caches, the
* compressor's partial group and the engram history all describe one sequence. Prefix
* reuse across turns is a later step (the KV here is four caches and a ring, not one
* tensor), and getting it wrong would be silent, so it is refused rather than faked. */
static void model_reset(Model *m) {
Cfg *c = &m->c;
for (int i = 0; i < c->n_layers; i++) {
Layer *l = &m->L[i];
memset(l->window, 0, (size_t)c->window * c->head_dim * sizeof(float));
for (int k = 0; k < c->window; k++) l->window_pos[k] = -1;
int ratio = c->compress_ratio[i];
if (c->kv_source[i]) {
int slots = c->max_positions / (ratio > 0 ? ratio : 1);
memset(l->ckv, 0, (size_t)slots * c->head_dim * sizeof(float));
memset(l->ikey, 0, (size_t)slots * c->index_head_dim * sizeof(float));
if (ratio > 1) {
memset(l->cstate_kv, 0, (size_t)ratio * c->head_dim * sizeof(float));
for (int k = 0; k < ratio * c->head_dim; k++) l->cstate_score[k] = -INFINITY;
}
}
}
m->engram.history_len = 0;
m->published_index_k = NULL;
m->shared_topk_rows = m->shared_topk_width = 0;
m->pos = 0;
#ifdef COLI_VULKAN
v41c_reset(m); /* nothing on the device describes the new sequence */
#endif
/* Paired with the reset on purpose: whoever drops the state must also forget
* what it was built from, or the two disagree in favour of the one nobody
* can check. */
kv_prefix_clear(&m->kvp);
}
static const ColiServeWireProfile v41_wire = {
.max_header_bytes = 511,
.max_payload_bytes = 1u << 24,
.max_tokens = 1 << 20,
.require_exact_lf = 1,
.require_finite_sampling = 1,
};
typedef struct { float p; int id; } SampleProb;
static int sample_desc(const void *a, const void *b) {
float pa = ((const SampleProb *)a)->p, pb = ((const SampleProb *)b)->p;
return (pb > pa) - (pa > pb);
}
static int serve_sample(const float *logits, int vocab, float temperature, float top_p) {
if (temperature <= 0.0f) return argmax(logits, vocab);
SampleProb *rank = xmalloc((size_t)vocab * sizeof(SampleProb), "sampling");
float top = logits[0];
for (int i = 1; i < vocab; i++) if (logits[i] > top) top = logits[i];
double total = 0.0;
for (int i = 0; i < vocab; i++) {
float p = expf((logits[i] - top) / temperature);
total += p;
rank[i].p = p; rank[i].id = i;
}
qsort(rank, (size_t)vocab, sizeof(SampleProb), sample_desc);
double cut = (top_p > 0.0f && top_p < 1.0f) ? top_p * total : total, kept = 0.0;
int n = 0;
while (n < vocab && kept < cut) kept += rank[n++].p;
double draw = ((double)rand() / RAND_MAX) * kept, acc = 0.0;
int pick = rank[0].id;
for (int i = 0; i < n; i++) { acc += rank[i].p; if (acc >= draw) { pick = rank[i].id; break; } }
free(rank);
return pick;
}
/* The probability `serve_sample` would give this token: the same temperature, the same
* top-p truncation, zero if the token falls outside the kept set. A greedy drafter
* proposes with probability 1, so accepting with probability p(draft) is exactly the
* speculative-sampling rule, and on a rejection the residual distribution is this one
* with the rejected token removed -- which is what banning it in the row does. */
static double spec_accept_prob(const float *logits, int vocab, float temperature,
float top_p, int token) {
SampleProb *rank = xmalloc((size_t)vocab * sizeof(SampleProb), "spec sampling");
float top = logits[0];
for (int i = 1; i < vocab; i++) if (logits[i] > top) top = logits[i];
double total = 0.0;
for (int i = 0; i < vocab; i++) {
float p = expf((logits[i] - top) / temperature);
total += p;
rank[i].p = p; rank[i].id = i;
}
qsort(rank, (size_t)vocab, sizeof(SampleProb), sample_desc);
double cut = (top_p > 0.0f && top_p < 1.0f) ? top_p * total : total, kept = 0.0;
int n = 0;
while (n < vocab && kept < cut) kept += rank[n++].p;
double probability = 0.0;
for (int i = 0; i < n; i++) if (rank[i].id == token) { probability = rank[i].p / kept; break; }
free(rank);
return probability;
}
/* Undo the part of a speculative step that was not committed.
*
* Most of the state needs nothing: the window ring keeps the rejected keys but they
* carry positions ahead of every query that follows, the compressed rows sit past what
* any query can reach, and the compressor's group slots are addressed by position, so
* the token that really lands there overwrites the draft before the group closes.
* Three things are not self-healing: how far the sequence has got, the engram history,
* and the two slots a layer publishes to -- all three are set here to what they would
* be after decoding exactly the committed prefix. */
static void spec_rollback(Model *m, int start_pos, int committed, int rows) {
Cfg *c = &m->c;
int hd = c->head_dim;
/* Newest first, so a slot two rows touched comes back to what it held before the
* first of them -- which is the value the committed prefix left there. */
for (int layer = 0; layer < c->n_layers; layer++) {
Layer *l = &m->L[layer];
for (int t = rows - 1; t >= committed; t--) {
int slot = (start_pos + t) % c->window;
memcpy(l->window + (size_t)slot * hd, l->ring_save + (size_t)t * hd,
(size_t)hd * sizeof(float));
l->window_pos[slot] = l->ring_save_pos[t];
if (!l->cstate_save_kv) continue;
int group = l->cstate_save_slot[t];
memcpy(l->cstate_kv + (size_t)group * hd, l->cstate_save_kv + (size_t)t * hd,
(size_t)hd * sizeof(float));
memcpy(l->cstate_score + (size_t)group * hd, l->cstate_save_score + (size_t)t * hd,
(size_t)hd * sizeof(float));
}
}
m->pos = start_pos + committed;
m->last_rows = committed;
#ifdef COLI_VULKAN
v41c_rollback(m, m->pos); /* the dense chain's copies of the rejected rows go stale */
#endif
if (m->engram.active && m->engram.history_len > m->pos) m->engram.history_len = m->pos;
/* The record follows the rollback for the same reason the engram history
* does: the rejected rows are no longer in the state, and a record that
* still claimed them would hand the next turn positions nothing holds. The
* ids below m->pos are the committed ones, so truncating is enough. */
if (m->kvp.len > m->pos) m->kvp.len = m->pos;
if (m->pub_rows > 0) {
const float *published = m->pub_before;
int layer = m->pub_before_layer;
for (int t = committed - 1; t >= 0; t--)
if (m->pub_layer[t] >= 0) {
layer = m->pub_layer[t];
published = m->L[layer].ikey;
break;
}
m->published_index_k = published;
m->published_index_layer = layer;
}
/* the index list the layers before the next index source read is the one for the
* last committed position, not for the first row of the batch */
if (m->shared_topk && m->shared_topk_width > 0 && m->shared_topk_rows > committed - 1) {
int keep = committed - 1;
if (keep > 0)
memmove(m->shared_topk, m->shared_topk + (size_t)keep * m->shared_topk_width,
(size_t)m->shared_topk_width * sizeof(int));
m->shared_topk_rows = 1;
}
}
/* Verify a block of drafts in one forward. `ids[0]` is the token the main model
* produced and is always committed; `ids[1..n-1]` are the drafts, accepted while they
* agree with what this forward says. `logits` comes back holding the continuation of
* the last committed position, and the caches are rolled back to it, so the caller can
* carry on as if the accepted tokens had been decoded one at a time -- which, position
* by position, is what they were. */
static int spec_verify(Model *m, const int *ids, int n, float *logits,
float temperature, float top_p) {
int vocab = m->c.vocab, start_pos = m->pos;
float *rows = xmalloc((size_t)n * vocab * sizeof(float), "verification logits");
forward_batch(m, ids, n, rows);
int accepted = 0, rejected = -1;
while (accepted < n - 1) {
const float *row = rows + (size_t)accepted * vocab;
int proposed = ids[accepted + 1];
int ok;
if (temperature <= 0.0f) ok = argmax(row, vocab) == proposed;
else ok = (double)rand() / RAND_MAX
< spec_accept_prob(row, vocab, temperature, top_p, proposed);
if (!ok) { rejected = proposed; break; }
accepted++;
}
memcpy(logits, rows + (size_t)accepted * vocab, (size_t)vocab * sizeof(float));
if (rejected >= 0 && temperature > 0.0f) logits[rejected] = -INFINITY;
spec_rollback(m, start_pos, accepted + 1, n);
free(rows);
return accepted;
}
/* Is it worth drafting right now? Drafting reads the stages' experts whether or not
* anything is accepted, so a run of refusals is a disk bill with no token to show for
* it. The window is the vendor-independent part of every speculative decoder: measure,
* pause, measure again. */
static int spec_ready(Spec *sp) {
if (!sp->active) return 0;
if (sp->pause > 0) {
if (--sp->pause == 0) sp->window_prop = sp->window_acc = 0;
return 0;
}
/* Ten proposals, not twenty-four: a round on a streamed model costs
* seconds, so a window that needs five rounds to decide spends the whole
* turn deciding. Two rounds is enough to see an acceptance rate that is
* half of what it needs to be. */
if (sp->window_prop >= 10 &&
sp->window_acc * 100 < sp->window_prop * (uint64_t)sp->min_accept) {
if (v41_stats())
fprintf(stderr, "[v41] DSpark: %.0f%% of the last %llu drafts accepted "
"(under %d%%), pausing for 64 tokens\n",
100.0 * (double)sp->window_acc / (double)sp->window_prop,
(unsigned long long)sp->window_prop, sp->min_accept);
sp->pause = 64;
sp->window_prop = sp->window_acc = 0;
return 0;
}
return 1;
}
static void serve_line(const char *format, ...) {
va_list args;
va_start(args, format);
vfprintf(stdout, format, args);
va_end(args);
fflush(stdout);
}
/* --- the dashboard's four lines -------------------------------------------
* Same bytes colibri.c emits, so the gateway parses this engine like any other:
* EMAP rows cols hex the grid, after READY and STAT and after every turn
* HITS rows cols hex the experts a turn routed, after DONE next to PROF
* PROF ... wall, tokens, disk, wait, matmul, attention, head, forwards
* Rows are the MoE layers (every layer here), columns the routed experts. */
static void serve_emap(Model *m) {
Cfg *c = &m->c;
int rows = c->n_layers, cols = c->n_routed;
char *hex = xmalloc((size_t)rows * cols * 2 + 1, "emap");
int w = 0;
for (int i = 0; i < rows; i++) {
LCache *cache = &m->cache[i];
for (int e = 0; e < cols; e++) {
int resident = 0;
for (int k = 0; k < cache->n; k++) if (cache->slot[k].eid == e) { resident = 1; break; }
int byte = resident << 6; /* tier in bits 6-7; no usage counter here, heat 0 */
#ifdef COLI_VULKAN
if (vkt_resident(i, e)) byte = 2 << 6; /* on the Vulkan device */
#endif
hex[w++] = "0123456789abcdef"[byte >> 4];
hex[w++] = "0123456789abcdef"[byte & 15];
}
}
hex[w] = 0;
serve_line("EMAP %d %d %s\n", rows, cols, hex);
free(hex);
}
static void serve_hits(Model *m) {
Cfg *c = &m->c;
int rows = c->n_layers, cols = c->n_routed, nb = (rows * cols + 7) / 8;
uint8_t *bitmap = xmalloc((size_t)nb, "hits bitmap");
memset(bitmap, 0, (size_t)nb);
int bit = 0;
for (int i = 0; i < rows; i++)
for (int e = 0; e < cols; e++, bit++)
if (m->ehit && m->ehit[i][e]) { bitmap[bit >> 3] |= (uint8_t)(1 << (bit & 7)); m->ehit[i][e] = 0; }
char *hex = xmalloc((size_t)nb * 2 + 1, "hits hex");
int w = 0;
for (int b = 0; b < nb; b++) {
hex[w++] = "0123456789abcdef"[bitmap[b] >> 4];
hex[w++] = "0123456789abcdef"[bitmap[b] & 15];
}
hex[w] = 0;
serve_line("HITS %d %d %s\n", rows, cols, hex);
free(hex); free(bitmap);
}
static int serve_eos(Model *m, const char *snap, int *ids, int cap) {
(void)m;
char path[1024];
snprintf(path, sizeof(path), "%s/config.json", snap);
FILE *f = fopen(path, "rb");
int n = 0;
if (!f) return 0;
fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET);
char *text = xmalloc((size_t)size + 1, "config");
if (fread(text, 1, (size_t)size, f) == (size_t)size) {
text[size] = 0;
char *arena = NULL;
jval *root = json_parse(text, &arena);
if (root) {
jval *eos = json_get(root, "eos_token_id");
if (eos && eos->t == J_NUM && n < cap) ids[n++] = (int)eos->num;
else if (eos && eos->t == J_ARR)
for (int i = 0; i < eos->len && n < cap; i++) ids[n++] = (int)eos->kids[i]->num;
json_free(root); free(arena);
}
}
fclose(f); free(text);
return n;
}
/* max_tokens is a ceiling, as on V4 and Qwen (#1641). A score-only
* request may fill the context; generation needs at least one free position. */
static int serve_budget(int prompt, int requested, int context, int logprobs) {
if (prompt < 1 || prompt > context) return -1;
int budget = requested > 0 ? requested : (logprobs > 0 ? 0 : 256);
int room = context - prompt;
if (budget > 0 && room == 0) return -1;
return budget < room ? budget : room;
}
/* When a request was accepted, and the counters then: DONE, PROF and the turn's
* stderr report its share. */
typedef struct {
double turn_started, disk0, expert0, attn0, engram0;
uint64_t forwards0, hits0, miss0, ebytes0;
uint64_t mir_bytes0[V41_MIR_REPS], mir_reads0[V41_MIR_REPS];
int n_prompt, budget;
} V41Req;
/* A request's prompt into the state the Model holds: its tokens and image, the budget,
* the prefix reuse and the photos, ACCEPT, the prefill and its read-out. 1 with the
* logits after the prompt; 0 when the request ended here, its ERROR written and its
* command disposed. serve_loop and serve_mux start every request here. */
static int v41_serve_start(Model *m, Tok *tokenizer, ColiServeCommand *cmd, int *ids, float *logits,
float **pending_image, int *pending_h, int *pending_w, V41Req *rq) {
Cfg *c = &m->c;
double turn_started = now_s();
double disk0 = m->t_disk, expert0 = m->t_expert, attn0 = m->t_attn, engram0 = m->t_engram;
uint64_t forwards0 = m->forwards, hits0 = m->hits, miss0 = m->miss;
uint64_t ebytes0 = m->expert_bytes;
uint64_t mir_bytes0[V41_MIR_REPS], mir_reads0[V41_MIR_REPS];
for (int r = 0; r < V41_MIR_REPS; r++) {
mir_bytes0[r] = g_mir_bytes[r];
mir_reads0[r] = g_mir_nread[r];
}
int n_prompt = tok_encode(tokenizer, (const char *)cmd->payload,
(int)cmd->payload_bytes, ids, c->max_positions + 1);
int budget = serve_budget(n_prompt, cmd->max_tokens, c->max_positions,
cmd->logprobs);
if (budget < 0) {
char message[128];
snprintf(message, sizeof(message),
"CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
n_prompt, cmd->max_tokens, c->max_positions);
coli_serve_write_error(stdout, cmd->id,
n_prompt < 1 ? "EMPTY_PROMPT" : message);
coli_serve_command_dispose(cmd); return 0;
}
if (cmd->max_tokens > budget)
fprintf(stderr, "[serve] max_tokens %d clamped to %d (context %d - prompt %d); "
"raise CTX for longer answers\n",
cmd->max_tokens, budget, c->max_positions, n_prompt);
/* Decided BEFORE the reset, because the reset is what it decides about.
* A chat client resends the whole transcript every turn; if this prompt
* begins with the ids the state was built from, that state already IS
* the state at those positions, so only the tail is fed. Reuse is all or
* nothing -- nothing here can rewind four caches and a ring. An image
* refuses it outright: the placeholder ids describe the span but not the
* picture, and the span's offsets are computed against the whole prompt.
* COLI_KV_PREFIX=0 disables it, COLI_PREFIX_LOG=1 reports the decision. */
int reuse = 0;
if (n_prompt >= 1 && kv_prefix_on() && !(*pending_image))
reuse = kv_prefix_reuse(&m->kvp, ids, n_prompt);
/* La fotografia si prova sempre: copre anche il caso in cui lo stato
* vivo e gia il prompt, dove senza di essa il primo token fresco
* resterebbe senza predittore e quindi senza logprob. */
g_pin_use_logit = 0; g_pin_logit = NULL;
if (n_prompt >= 1 && !(*pending_image)) {
/* Il piu profondo degli scatti valido: con due livelli annidati
* (istruzioni, istruzioni+domanda) vince il secondo, e se le sue
* righe non ci sono piu si ripiega sul primo. */
int ps = coli_pin_best(&g_pins, ids, n_prompt);
while (ps >= 0) {
ColiPin *k = &g_pins.slot[ps];
if (kv_prefix_holds(&m->kvp, k->ids, k->len)) {
kv_prefix_clear(&m->kvp);
kv_prefix_record(&m->kvp, k->ids, 0, k->len);
reuse = k->len;
g_pin_logit = k->logit; g_pin_use_logit = k->logit != NULL;
coli_pin_touch(&g_pins, ps);
break;
}
k->len = 0;
ps = coli_pin_best(&g_pins, ids, n_prompt);
}
}
if (getenv("COLI_PREFIX_LOG")) {
if (reuse)
fprintf(stderr, "[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",
reuse, n_prompt, 100.0 * reuse / n_prompt);
else
fprintf(stderr, "[PREFIX] no reuse: held=%d cap=%d prompt=%d%s%s%s\n",
m->kvp.len, m->kvp.cap, n_prompt,
m->kvp.tainted ? " tainted" : "",
(*pending_image) ? " (image)" : "",
kv_prefix_on() ? "" : " (off: set COLI_KV_PREFIX=1)");
fflush(stderr);
}
if (!reuse) model_reset(m);
coli_serve_write_accept(stdout, cmd->id, n_prompt);
float *aligned = NULL;
uint8_t *image_mask = NULL;
int image_at = -1, image_h = 0, image_w = 0;
if ((*pending_image) && m->vision && c->image_token_id >= 0) {
/* the span is the run of placeholder ids the gateway inserted */
for (int t = 0; t < n_prompt; t++)
if (ids[t] == c->image_token_id) { image_at = t; break; }
int ratio = c->vision_ratio;
image_h = ((*pending_h) + ratio - 1) / ratio;
image_w = ((*pending_w) + ratio - 1) / ratio;
int span = 1 + (image_w + 1) * image_h + 1;
int run = 0;
for (int t = image_at; t >= 0 && t < n_prompt && ids[t] == c->image_token_id; t++) run++;
if (image_at < 0 || run != span) {
fprintf(stderr, "[v41] image span is %d tokens, the %dx%d grid needs %d: "
"answering without the image\n", run, image_h, image_w, span);
image_at = -1;
} else {
int rows = vision_tokens(c, (*pending_h), (*pending_w));
aligned = xmalloc((size_t)rows * c->dim * sizeof(float), "image rows");
double vision_started = now_s();
vision_forward(m, m->vision, (*pending_image), (*pending_h), (*pending_w), aligned);
fprintf(stderr, "[v41] image %dx%d patches -> %d rows in %.2fs\n",
(*pending_h), (*pending_w), rows, now_s() - vision_started);
image_mask = xmalloc((size_t)n_prompt, "image mask");
memset(image_mask, 0, (size_t)n_prompt);
for (int t = image_at; t < image_at + span; t++) image_mask[t] = 1;
}
}
free((*pending_image)); (*pending_image) = NULL;
/* `reuse` is the ABSOLUTE position of the first fresh token: every cache
* here is position-indexed, so this has to be the real offset. */
int nfresh = n_prompt - reuse;
float *all = NULL;
int echoed = 0; /* il prefill l'ha gia fatto il ramo della lettura */
if (cmd->logprobs > 0 && nfresh > 0 && !(*pending_image)) {
all = (float *)malloc((size_t)nfresh * (size_t)c->vocab * sizeof(float));
if (all) {
/* un prefill normale (spec_batch=0) che tiene tutte le righe */
forward_full(m, ids + reuse, nfresh, all, 0, NULL, -1, 0, 0, NULL, 1);
/* La posizione p predice il token p+1; il primo token fresco e
* predetto dalla fotografia. Cosi ogni token dell'opzione ha il
* suo logprob, anche se non e fra i primi k di nessuna classifica. */
if (g_pin_use_logit && g_pin_logit)
v41_echo(cmd->id, reuse, ids[reuse], g_pin_logit, c->vocab,
cmd->logprobs, tokenizer);
for (int p = 0; p + 1 < nfresh; p++)
v41_echo(cmd->id, reuse + p + 1, ids[reuse + p + 1],
all + (size_t)p * c->vocab, c->vocab, cmd->logprobs, tokenizer);
memcpy(logits, all + (size_t)(nfresh - 1) * c->vocab,
(size_t)c->vocab * sizeof(float));
free(all);
echoed = 1;
}
}
if (!echoed)
forward_with_image(m, ids + reuse, n_prompt - reuse, logits, aligned,
image_at, image_h, image_w, image_mask);
free(aligned); free(image_mask);
if (cmd->pin) {
coli_pin_pool_init(&g_pins, c->vocab);
if (coli_pin_store(&g_pins, ids, n_prompt, logits)) {
fprintf(stderr, "[PIN] scatto a %d token\n", n_prompt);
fflush(stderr);
}
}
rq->turn_started = turn_started; rq->disk0 = disk0; rq->expert0 = expert0; rq->attn0 = attn0;
rq->engram0 = engram0; rq->forwards0 = forwards0; rq->hits0 = hits0; rq->miss0 = miss0;
rq->ebytes0 = ebytes0; rq->n_prompt = n_prompt; rq->budget = budget;
memcpy(rq->mir_bytes0, mir_bytes0, sizeof mir_bytes0); memcpy(rq->mir_reads0, mir_reads0, sizeof mir_reads0);
return 1;
}
/* ---- several conversations at once (KV_SLOTS>1) ---------------------------------
* The gateway's cache slots, each a conversation with a state of its own (V41Seq). A
* SUBMIT on a free slot starts its request at once through v41_serve_start, on that
* slot's state: its prefix reuse and photos work as a lone serve's. Then every step
* picks the next token of each active request and runs one forward over a row of each
* (forward_rows). A request's frames are a lone request's; they interleave by id. As
* alone, STOP ends a request with DONE and CANCEL with ERROR CANCELLED. */
typedef struct {
char id[COLI_SERVE_ID_CAP];
float temperature, top_p;
int logprobs, active, stop, cancel, limited, emitted;
float *logits; /* the logits the next pick reads */
V41Req rq;
} V41MuxReq;
static void v41_mux_finish(Model *m, V41MuxReq *r) {
r->active = 0;
if (r->cancel) { coli_serve_write_error(stdout, r->id, "CANCELLED"); return; }
if (r->stop) r->limited = 0;
double wall = now_s() - r->rq.turn_started;
uint64_t turn_hits = m->hits - r->rq.hits0, turn_miss = m->miss - r->rq.miss0;
ColiServeDone done = {
r->emitted, wall > 0 ? r->emitted / wall : 0.0,
(turn_hits + turn_miss) ? 100.0 * turn_hits / (double)(turn_hits + turn_miss) : 0.0,
rss_gb(), r->rq.n_prompt, r->limited,
};
coli_serve_write_done(stdout, r->id, &done);
serve_line("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n", wall, r->rq.n_prompt, r->emitted,
m->t_disk - r->rq.disk0, 0.0, m->t_expert - r->rq.expert0, m->t_attn - r->rq.attn0,
m->t_engram - r->rq.engram0, (unsigned long long)(m->forwards - r->rq.forwards0));
serve_hits(m);
}
/* The next token of an active request, as serve_loop picks and sends it: 1 with the
* token when the request goes on, 0 when it ended. */
static int v41_mux_pick(Model *m, Tok *tokenizer, V41MuxReq *r, const int *eos_ids, int n_eos, int *tk_out) {
Cfg *c = &m->c;
if (r->cancel || r->stop || r->emitted >= r->rq.budget) { v41_mux_finish(m, r); return 0; }
int token = serve_sample(r->logits, c->vocab, r->temperature, r->top_p);
for (int i = 0; i < n_eos; i++) if (token == eos_ids[i]) { r->limited = 0; v41_mux_finish(m, r); return 0; }
char piece[512];
int written = tok_decode(tokenizer, &token, 1, piece, (int)sizeof(piece));
if (written > 0) {
if (r->logprobs > 0) {
char lp[1024];
coli_logprob_tail(lp, sizeof lp, r->logits, c->vocab, token, r->logprobs);
coli_serve_write_data_lp(stdout, r->id, piece, (size_t)written, lp);
} else coli_serve_write_data(stdout, r->id, piece, (size_t)written);
}
r->emitted++;
if (r->emitted >= r->rq.budget) { v41_mux_finish(m, r); return 0; }
*tk_out = token; return 1;
}
static void serve_mux(Model *m, Tok *tokenizer, const char *snap) {
Cfg *c = &m->c;
int n = g_v41_mux_slots, V = c->vocab, input_eof = 0;
int eos_ids[8];
int n_eos = serve_eos(m, snap, eos_ids, 8);
coli_serve_write_ready_caps(stdout, rss_gb(), m->vision ? "vision=1" : "vision=0");
serve_emap(m);
V41MuxReq *rq = xcalloc((size_t)n, sizeof *rq, "requests");
int *slots = xmalloc((size_t)n * sizeof(int), "rows"), *tok = xmalloc((size_t)n * sizeof(int), "tokens");
float *lo = xmalloc((size_t)n * V * sizeof(float), "step logits");
int *ids = xmalloc(((size_t)c->max_positions + 1) * sizeof(int), "prompt ids");
for (int i = 0; i < n; i++) rq[i].logits = xmalloc((size_t)V * sizeof(float), "logits");
float *pending_image = NULL;
int pending_h = 0, pending_w = 0;
unsigned long long steps = 0, nrows = 0;
fprintf(stderr, "[v41] serving %d conversations at once (KV_SLOTS)\n", n);
for (;;) {
int active = 0; for (int i = 0; i < n; i++) active += rq[i].active;
/* idle: wait for a command; decoding: take one only when one is there */
if (!input_eof && (!active || coli_serve_stdin_ready())) {
ColiServeCommand command;
ColiServeReadResult result = coli_serve_read_command(stdin, &v41_wire, &command);
if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) input_eof = 1;
else if (result == COLI_SERVE_READ_NOMEM) { coli_serve_write_error(stdout, command.id, "out of memory"); input_eof = 1; }
else if (result == COLI_SERVE_READ_BAD_REQUEST) {
if (command.kind == COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout, command.id, "bad submit header");
coli_serve_command_dispose(&command);
} else if (command.kind == COLI_SERVE_COMMAND_IMAGE) {
int patch_in = 3 * c->vision_patch * c->vision_patch;
uint64_t expected = (uint64_t)command.grid_h * command.grid_w * patch_in * sizeof(float);
if (!m->vision) coli_serve_write_error(stdout, command.id, "this container has no vision tower");
else if (command.grid_h < 1 || command.grid_w < 1 || command.payload_bytes != expected)
coli_serve_write_error(stdout, command.id, "IMAGE payload does not match its grid");
else {
if (pending_image) fprintf(stderr, "[v41] a second IMAGE replaced the first\n");
free(pending_image);
pending_image = (float *)coli_serve_command_take_payload(&command);
pending_h = command.grid_h; pending_w = command.grid_w;
}
coli_serve_command_dispose(&command);
} else if (command.kind == COLI_SERVE_COMMAND_CANCEL || command.kind == COLI_SERVE_COMMAND_STOP) {
int found = 0;
for (int i = 0; i < n; i++) if (rq[i].active && !strcmp(rq[i].id, command.id)) {
found = 1;
if (command.kind == COLI_SERVE_COMMAND_STOP) rq[i].stop = 1; else rq[i].cancel = 1;
}
if (!found && command.kind == COLI_SERVE_COMMAND_CANCEL) coli_serve_write_error(stdout, command.id, "NOT_FOUND");
coli_serve_command_dispose(&command);
} else if (command.kind != COLI_SERVE_COMMAND_SUBMIT) {
coli_serve_command_dispose(&command);
} else if (command.slot < 0 || command.slot >= n || rq[command.slot].active) {
coli_serve_write_error(stdout, command.id, command.slot < 0 || command.slot >= n ? "invalid cache slot" : "SLOT_BUSY");
coli_serve_command_dispose(&command);
} else {
V41MuxReq *t = &rq[command.slot];
float *keep = t->logits;
memset(t, 0, sizeof *t); t->logits = keep;
snprintf(t->id, sizeof t->id, "%s", command.id);
t->temperature = command.temperature; t->top_p = command.top_p; t->logprobs = command.logprobs;
v41_mux_bind(m, command.slot);
if (v41_serve_start(m, tokenizer, &command, ids, t->logits, &pending_image, &pending_h, &pending_w, &t->rq)) {
t->active = 1; t->limited = 1;
coli_serve_command_dispose(&command);
}
}
}
active = 0; for (int i = 0; i < n; i++) active += rq[i].active;
if (!active) { if (input_eof) break; continue; }
int S = 0, ended = 0;
for (int i = 0; i < n; i++) if (rq[i].active) {
int tk;
if (!v41_mux_pick(m, tokenizer, &rq[i], eos_ids, n_eos, &tk)) { ended = 1; continue; }
slots[S] = i; tok[S] = tk; S++;
}
if (S) {
forward_rows(m, slots, tok, S, lo);
steps++; nrows += (unsigned long long)S;
for (int s = 0; s < S; s++) memcpy(rq[slots[s]].logits, lo + (size_t)s * V, (size_t)V * sizeof(float));
}
if (ended) serve_emap(m);
}
fprintf(stderr, "[v41] KV_SLOTS=%d: %llu decode steps, %llu rows (%.2f a step)\n", n, steps, nrows,
steps ? (double)nrows / (double)steps : 0.0);
v41_mux_bind(m, 0);
for (int i = 0; i < n; i++) free(rq[i].logits);
free(rq); free(slots); free(tok); free(lo); free(ids); free(pending_image);
}
static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {
if (g_v41_mux_slots > 1) { serve_mux(m, tokenizer, snap); return; }
Cfg *c = &m->c;
coli_serve_stdio_init();
int eos_ids[8];
int n_eos = serve_eos(m, snap, eos_ids, 8);
/* CAPS between READY and STAT: the gateway reads it in the handshake, so it
* knows the served modalities before the first request. m->vision is NULL
* for a text-only container and for a VL config whose tower is missing. */
coli_serve_write_ready_caps(stdout, rss_gb(), m->vision ? "vision=1" : "vision=0");
serve_emap(m);
float *logits = xmalloc((size_t)c->vocab * sizeof(float), "logits");
/* tok_encode stops at its output capacity: one extra id distinguishes
* a full, valid read-only prompt from a silently truncated one. */
int *ids = xmalloc(((size_t)c->max_positions + 1) * sizeof(int), "prompt ids");
float *pending_image = NULL;
int pending_h = 0, pending_w = 0;
for (;;) {
ColiServeCommand command;
ColiServeReadResult result = coli_serve_read_command(stdin, &v41_wire, &command);
if (result == COLI_SERVE_READ_EOF || result == COLI_SERVE_READ_BAD_FRAME) break;
if (result == COLI_SERVE_READ_NOMEM) {
coli_serve_write_error(stdout, command.id, "out of memory"); break; }
if (result == COLI_SERVE_READ_BAD_REQUEST) {
if (command.kind == COLI_SERVE_COMMAND_SUBMIT)
coli_serve_write_error(stdout, command.id, "bad submit header");
coli_serve_command_dispose(&command);
continue;
}
if (command.kind == COLI_SERVE_COMMAND_IMAGE) {
/* One image waits for the SUBMIT that names it. A second one before that
* SUBMIT replaces the first and says so: answering about the wrong photo
* is worse than refusing. */
if (!m->vision) {
coli_serve_write_error(stdout, command.id, "this container has no vision tower");
coli_serve_command_dispose(&command);
continue;
}
int patch_in = 3 * c->vision_patch * c->vision_patch;
uint64_t expected = (uint64_t)command.grid_h * command.grid_w * patch_in * sizeof(float);
if (command.grid_h < 1 || command.grid_w < 1 || command.payload_bytes != expected) {
coli_serve_write_error(stdout, command.id, "IMAGE payload does not match its grid");
coli_serve_command_dispose(&command);
continue;
}
if (pending_image) fprintf(stderr, "[v41] a second IMAGE replaced the first\n");
free(pending_image);
pending_image = (float *)coli_serve_command_take_payload(&command);
pending_h = command.grid_h; pending_w = command.grid_w;
coli_serve_command_dispose(&command);
continue;
}
if (command.kind != COLI_SERVE_COMMAND_SUBMIT) {
/* CANCEL and STOP for a turn that is not running: the protocol's NOT_FOUND */
if (command.kind == COLI_SERVE_COMMAND_CANCEL)
coli_serve_write_error(stdout, command.id, "NOT_FOUND");
coli_serve_command_dispose(&command);
continue;
}
V41Req rq;
if (!v41_serve_start(m, tokenizer, &command, ids, logits, &pending_image, &pending_h, &pending_w, &rq)) continue;
double turn_started = rq.turn_started;
double disk0 = rq.disk0, expert0 = rq.expert0, attn0 = rq.attn0, engram0 = rq.engram0;
uint64_t forwards0 = rq.forwards0, hits0 = rq.hits0, miss0 = rq.miss0;
uint64_t ebytes0 = rq.ebytes0;
uint64_t mir_bytes0[V41_MIR_REPS], mir_reads0[V41_MIR_REPS];
memcpy(mir_bytes0, rq.mir_bytes0, sizeof mir_bytes0); memcpy(mir_reads0, rq.mir_reads0, sizeof mir_reads0);
int n_prompt = rq.n_prompt, budget = rq.budget;
uint64_t prefill_bytes = m->expert_bytes - ebytes0;
double prefill_disk = m->t_disk - disk0;
double prefill_expert = m->t_expert - expert0, prefill_wall = now_s() - turn_started;
int emitted = 0, limited = 1, cancelled = 0, done_early = 0;
char piece[512];
int block = m->spec.active ? c->spec_block : 0;
int *batch = block ? xmalloc((size_t)(block + 1) * sizeof(int), "draft batch") : NULL;
float *confidence = block ? xmalloc((size_t)block * sizeof(float), "draft confidence") : NULL;
uint64_t proposed0 = m->spec.proposed, accepted0 = m->spec.accepted;
/* the prompt seeds the draft head's windows; nothing is drafted from it */
if (m->spec.active) spec_step(m, 0, m->last_start, m->last_rows, NULL, NULL);
while (emitted < budget && !cancelled && !done_early) {
int token = serve_sample(logits, c->vocab, command.temperature, command.top_p);
int stop = 0;
for (int i = 0; i < n_eos; i++) if (token == eos_ids[i]) stop = 1;
if (stop) { limited = 0; break; }
int written = tok_decode(tokenizer, &token, 1, piece, (int)sizeof(piece));
if (written > 0) {
if (command.logprobs > 0) {
/* La distribuzione da cui il token e stato estratto e ancora
* quella in `logits`: serve_sample non la modifica. */
char lp[1024];
coli_logprob_tail(lp, sizeof lp, logits, c->vocab, token, command.logprobs);
coli_serve_write_data_lp(stdout, command.id, piece, (size_t)written, lp);
} else coli_serve_write_data(stdout, command.id, piece, (size_t)written);
}
emitted++;
/* #1332: look at stdin once per token so a client that walked away stops
* paying for a turn nobody wants. A CANCEL aborts, a STOP ends it through
* the normal DONE path -- the two are not the same frame and must not
* share an outcome. */
while (coli_serve_stdin_ready()) {
ColiServeCommand control;
ColiServeReadResult inner = coli_serve_read_command(stdin, &v41_wire, &control);
if (inner == COLI_SERVE_READ_EOF) { done_early = 1; break; }
if (control.kind == COLI_SERVE_COMMAND_CANCEL &&
!strcmp(control.id, command.id)) cancelled = 1;
else if (control.kind == COLI_SERVE_COMMAND_STOP &&
!strcmp(control.id, command.id)) { limited = 0; done_early = 1; }
else if (control.kind == COLI_SERVE_COMMAND_SUBMIT)
coli_serve_write_error(stdout, control.id, "SLOT_BUSY");
coli_serve_command_dispose(&control);
}
if (cancelled || done_early || emitted >= budget) break;
/* DSpark: draft the next few tokens, then verify them in ONE forward. What
* survives verification is emitted here; what does not costs the round its
* remaining drafts and nothing else, because the caches are rolled back to
* the last token that agreed. */
int drafted = spec_ready(&m->spec)
? spec_step(m, token, m->last_start, m->last_rows, batch, confidence) : 0;
if (m->spec.max_verify > 0 && drafted > m->spec.max_verify) drafted = m->spec.max_verify;
if (drafted > budget - emitted) drafted = budget - emitted;
if (drafted <= 0) { forward(m, &token, 1, logits); continue; }
batch[0] = token;
int accepted = spec_verify(m, batch, drafted + 1, logits,
command.temperature, command.top_p);
m->spec.proposed += (uint64_t)drafted;
m->spec.accepted += (uint64_t)accepted;
m->spec.window_prop += (uint64_t)drafted;
m->spec.window_acc += (uint64_t)accepted;
for (int i = 0; i < accepted; i++) {
int drafted_token = batch[1 + i];
int drafted_stop = 0;
for (int k = 0; k < n_eos; k++) if (drafted_token == eos_ids[k]) drafted_stop = 1;
if (drafted_stop) { limited = 0; done_early = 1; break; }
written = tok_decode(tokenizer, &drafted_token, 1, piece, (int)sizeof(piece));
if (written > 0) {
if (command.logprobs > 0) {
char lp[1024];
coli_logprob_tail(lp, sizeof lp, logits, c->vocab, token, command.logprobs);
coli_serve_write_data_lp(stdout, command.id, piece, (size_t)written, lp);
} else coli_serve_write_data(stdout, command.id, piece, (size_t)written);
}
emitted++;
}
}
free(batch); free(confidence);
/* Per-turn accounting goes to stderr, and stderr is what the user reads
* their answer next to in `coli chat`: four lines a turn between the
* question and the reply is noise, not instrumentation. Off unless asked
* for, and the benches ask for it. */
int stats = v41_stats();
if (stats && m->engram.active) {
/* What the n-gram memory actually cost this turn. Rows are 264
* bytes and the traffic is Zipfian, so the useful question is not
* how many rows were read but how many of them the cache already
* had -- and, on a turn short enough that capacity cannot bind,
* what is left is pure repetition within the text. */
for (int t = 0; t < m->engram.n_layers; t++) {
EngramTable *tab = &m->engram.table[t];
uint64_t total = tab->hits + tab->misses;
fprintf(stderr, "[v41] engram table %d (layer %d): %llu lookups, "
"%.0f%% cache hits, %llu rows read (%.1f MB)\n",
t, m->engram.layer_of[t], (unsigned long long)total,
total ? 100.0 * (double)tab->hits / (double)total : 0.0,
(unsigned long long)tab->misses,
tab->misses * (double)(m->engram.head_dim + m->engram.head_dim / 32) / 1e6);
}
}
if (stats) {
/* Expert streaming, per turn: how many bytes the LRU had to fetch and
* what rate the reads actually achieved. The rate is the number that
* matters -- the device is worth several GB/s, so a figure far under
* that says the loads are serialised, not that the disk is slow. */
uint64_t turn_bytes = m->expert_bytes - ebytes0;
double seconds = m->t_disk - disk0;
if (turn_bytes)
fprintf(stderr, "[v41] expert I/O: %llu misses, %.1f MB in %.2fs (%.2f GB/s); "
"prefill %.1f MB, disk %.2fs, matmul %.2fs, wall %.2fs\n",
(unsigned long long)(m->miss - miss0), turn_bytes / 1e6, seconds,
seconds > 0 ? turn_bytes / 1e9 / seconds : 0.0,
prefill_bytes / 1e6, prefill_disk, prefill_expert, prefill_wall);
/* Which drive answered, per turn. The split is the whole point of a
* mirror and the only honest way to check it is to count: a replica
* that was registered but never read from would look like a win in
* every number except the one that matters. */
if (turn_bytes) mirror_report(mir_bytes0, mir_reads0);
}
if (stats && m->spec.active && m->spec.proposed > proposed0)
fprintf(stderr, "[v41] DSpark: %llu of %llu drafts accepted this turn\n",
(unsigned long long)(m->spec.accepted - accepted0),
(unsigned long long)(m->spec.proposed - proposed0));
if (cancelled) {
coli_serve_write_error(stdout, command.id, "CANCELLED");
coli_serve_command_dispose(&command);
continue;
}
double wall = now_s() - turn_started;
uint64_t turn_hits = m->hits - hits0, turn_miss = m->miss - miss0;
ColiServeDone done = {
emitted, wall > 0 ? emitted / wall : 0.0,
(turn_hits + turn_miss) ? 100.0 * turn_hits / (double)(turn_hits + turn_miss) : 0.0,
rss_gb(), n_prompt, limited,
};
coli_serve_write_done(stdout, command.id, &done);
serve_line("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n", wall, n_prompt, emitted,
m->t_disk - disk0, 0.0, m->t_expert - expert0, m->t_attn - attn0,
m->t_engram - engram0, (unsigned long long)(m->forwards - forwards0));
#ifdef COLI_VULKAN
if (vkt_ready()) { vkt_report("turn", turn_hits, turn_miss); vk_tier_save(); }
v41c_report(); /* the dense chain's line, when it ran */
#endif
serve_hits(m);
serve_emap(m);
coli_serve_command_dispose(&command);
}
free(ids); free(logits);
}
/* ----------------------------------------------------------------- main ---- */
static int *load_ids(jval *root, const char *key, int *count) {
jval *array = json_get(root, key);
if (!array || array->t != J_ARR) { *count = 0; return NULL; }
int *out = xmalloc((size_t)array->len * sizeof(int), "reference ids");
for (int i = 0; i < array->len; i++) out[i] = (int)array->kids[i]->num;
*count = array->len;
return out;
}
/* One line at the end of a run with COLI_VULKAN=1: how many matmuls the device ran.
* coli_vk_matmul_calls() counts only calls that completed there, so a path that
* initialised and never ran cannot pass for one that did. */
static void vk_report(void) {
#ifdef COLI_VULKAN
if (!g_vk_ready) return;
size_t bytes = 0, tensors = 0;
coli_vk_mem_info(&bytes, &tensors);
fprintf(stderr, "[VK] deepseek_v41: %llu matmuls on the GPU\n", coli_vk_matmul_calls());
fprintf(stderr, "[VK] deepseek_v41: %zu resident matrices on the device, %.1f MiB "
"(fp8 as fmt 12, bf16 as fmt 11)\n", tensors, bytes / 1048576.0);
#endif
}
#ifdef COLI_VULKAN
/* ---- the Vulkan routed-expert tier: start ----------------------------------------
* After the weights and the device. The experts sit in the slots as the checkpoint
* stores them, fp4 with a ue8m0 scale per 32 inputs (MXFP4_E8M0, fmt 7 on the
* device); the activation is the clamped SwiGLU. The history (.coli_usage, beside
* the container or COLI_USAGE) is kept only while the tier is on; it gives the warm
* start, read straight from the container in parallel. */
static int vk_placed(const void *data) { /* on the device already (the dense weights on the device only) */
if (!g_v41_dho) return 0;
V41Home *h = v41_home_of(data);
return h && h->gone;
}
static size_t vk_w8_bytes(const W8 *w) { /* as the device holds it: fmt 12, an f32 per 32 inputs */
if (vk_placed(w->q)) return 0;
return w->q ? (size_t)w->O * w->I + (size_t)w->O * ((w->I + FP8_TILE - 1) / FP8_TILE) * 4 : 0;
}
static size_t vk_wb_bytes(const WB *w) { return w->w && !vk_placed(w->w) ? (size_t)w->O * w->I * 2 : 0; }
/* The trunk the dense hook would put on the device, for the tier's budget. */
static size_t vk_dense_bytes(Model *m) {
if (!coli_vk_dense() || g_v41_partial) return 0; /* a partial chain: the per-matrix path uploads nothing new */
size_t b = vk_wb_bytes(&m->head);
for (int i = 0; i < m->c.n_layers; i++) {
Layer *L = &m->L[i];
const W8 *w8[] = {&L->wq_a, &L->wq_b, &L->wkv, &L->wo_a, &L->wo_b, &L->idx_wq_b,
&L->sh_w1, &L->sh_w3, &L->sh_w2, &L->eng_wkv};
const WB *wb[] = {&L->comp_wkv, &L->comp_wgate, &L->idx_wk, &L->idx_wproj, &L->gate_w};
for (size_t k = 0; k < sizeof w8 / sizeof *w8; k++) b += vk_w8_bytes(w8[k]);
for (size_t k = 0; k < sizeof wb / sizeof *wb; k++) b += vk_wb_bytes(wb[k]);
}
return b;
}
/* Is the expert in this layer's slots now (the tier's balance asks)? */
static int vk_in_ram(void *ctx, int layer, int e) {
Model *m = ctx;
if (layer < 0 || layer >= m->c.n_layers + (m->spec.active ? m->c.n_mtp : 0)) return 0;
LCache *cache = layer < m->c.n_layers ? &m->cache[layer] : &m->spec.cache[layer - m->c.n_layers]; /* a DSpark stage's */
for (int i = 0; i < cache->n; i++) if (cache->slot[i].eid == e) return 1;
return 0;
}
static void vk_tier_start(Model *m, const char *snap, int cap) {
Cfg *c = &m->c;
if (!g_vk_ready || c->n_routed < 1 || !vkt_wanted()) return;
VktFmt f = {VKT_SRC_MXFP4_E8M0, 32};
size_t expert = 0;
for (int part = 0; part < V41_EXPERT_TENSORS; part++) expert += (size_t)expert_part_bytes(c, part);
/* The DSpark stages (index n_layers + stage): their experts are the backbone's form,
* the tier's extra layers. By default on a discrete GPU only: on a GPU that shares
* the RAM, qwen38's head measured no faster there (docs/vulkan.md, "The MTP head's
* layer on the tier"); COLI_VK_TIER_MTP=1 or 0 decides. */
const char *tm = getenv("COLI_VK_TIER_MTP");
int want_mtp = tm && *tm ? *tm != '0' : !coli_vk_device_shares_ram();
int stages = want_mtp && m->spec.active && c->spec_routed <= c->n_routed && c->spec_activated <= MOE_TOPK_MAX ? c->n_mtp : 0;
VktConfig vc = {.engine = "deepseek_v41", .layers = c->n_layers, .experts = c->n_routed,
.hidden = c->dim, .inter = c->moe_inter, .topk = c->n_activated,
.gate_up = f, .down = f, .act = VKT_ACT_SWIGLU, .act_limit = c->swiglu_limit,
.extra_layers = stages, .extra_gate_up = f, .extra_down = f,
.max_rows = MOE_ROW_CHUNK * c->n_activated,
.ram_reserve = expert * (size_t)cap * (size_t)c->n_layers,
.dense_bytes = vk_dense_bytes(m), .in_ram = vk_in_ram, .ram_ctx = m,
.load = vk_load, .release = vk_unhold, .load_ctx = m, .load_batch = vk_load_batch};
rt_init("deepseek_v41", c->n_layers, c->n_routed);
rt_drop_row(c->n_layers); /* no MTP row: the history is the backbone's */
const char *up = getenv("COLI_USAGE");
if (up && *up) snprintf(g_vk_usage, sizeof g_vk_usage, "%s", up);
else snprintf(g_vk_usage, sizeof g_vk_usage, "%s/.coli_usage", snap);
int64_t h = rt_load(g_vk_usage);
if (h > 0) fprintf(stderr, "[USAGE] expert history: %lld selections (%s)\n", (long long)h, g_vk_usage);
/* The device goes before the drivers unload, after the tier's teardown (atexit runs
* last-registered first), and whether or not the tier starts: vkt_init makes the
* expert batch's pipelines before it can refuse (no room), and a driver still
* compiling them at exit crashed one run in sixteen on Mesa's Dozen. */
atexit(coli_vk_shutdown);
if (!vkt_init(&vc, rt_counts_all())) { rt_destroy(); g_vk_usage[0] = 0; return; }
atexit(vkt_shutdown);
int all = c->n_layers * c->n_routed;
int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));
const char *warm = getenv("COLI_VK_TIER_WARM"); /* 0: no warm start, the tier fills as experts pass by */
int n = pl && pe && !(warm && *warm == '0') ? vkt_plan(pl, pe, all) : 0;
if (n > 0) {
double t0 = now_s();
static const char *const part[V41_EXPERT_TENSORS] = {"w1.weight", "w1.scale", "w3.weight", "w3.scale", "w2.weight", "w2.scale"};
#pragma omp parallel for schedule(dynamic, 4)
for (int i = 0; i < n; i++) {
uint8_t *buf = malloc(expert), *at = buf, *p[V41_EXPERT_TENSORS];
if (!buf) continue;
for (int k = 0; k < V41_EXPERT_TENSORS; k++) {
char name[160];
snprintf(name, sizeof name, "layers.%d.ffn.experts.%d.%s", pl[i], pe[i], part[k]);
st_read_raw_cap(&m->S, name, at, expert_part_bytes(c, k), 0);
p[k] = at; at += expert_part_bytes(c, k);
}
VktExpertSrc src = {p[0], p[2], p[4], p[1], p[3], p[5]};
vkt_put(pl[i], pe[i], &src);
free(buf);
}
vkt_put_done();
fprintf(stderr, "[VK] tier deepseek_v41: warm start, %d experts from the history in %.1fs\n", n, now_s() - t0);
}
free(pl); free(pe);
}
#endif
int main(int argc, char **argv) {
/* Size the team to PHYSICAL cores before anything else touches the model.
* This engine issues ~720 OpenMP regions per decoded token -- three per
* expert application, 240 applications a token -- and every one of them is
* only a few thousand rows wide, so the barrier is a real share of the work
* rather than a rounding error. Left at the OpenMP default (one thread per
* logical CPU) the region cost dominates: on a 208-logical-CPU host 93% of
* all cycles land inside libgomp and the turn runs 18.7x slower than at 32
* threads and 11.7x slower than at the 104 physical cores this picks.
* colibri/inkling/kimi_k3/olmoe already call it; see
* docs/experiments/dsv41-omp-team-2026-09-15.md.
* OMP_NUM_THREADS still wins, COLI_NO_OMP_TUNE=1 still disables it. */
coli_omp_tune_threads("deepseek-v41");
const char *snap = getenv("SNAP");
if (!snap) { fprintf(stderr, "SNAP=<container dir> is required\n"); return 2; }
if (getenv("SERVE") && atoi(getenv("SERVE"))) { /* KV_SLOTS: the conversations decoded at once */
const char *ks = getenv("KV_SLOTS");
if (ks && *ks) {
char *end = NULL; long v = strtol(ks, &end, 10);
if (end == ks || *end || v < 1 || v > 16) { fprintf(stderr, "KV_SLOTS must be between 1 and 16\n"); return 2; }
g_v41_mux_slots = (int)v;
}
}
int cap = argc > 1 ? coli_arg_int(argv[1], "cache/layer") : 8;
const char *ref_path = argc > 2 ? argv[2] : NULL;
int engram_rows = getenv("V41_ENGRAM_ROWS") ? atoi(getenv("V41_ENGRAM_ROWS")) : 65536;
g_trace = getenv("V41_TRACE") ? atoi(getenv("V41_TRACE")) : 0;
Model m;
memset(&m, 0, sizeof(m));
double started = now_s();
model_load(&m, snap, cap, engram_rows);
Cfg *c = &m.c;
fprintf(stderr, "[v41] %d layers, %d experts/layer (%d routed), dim %d, hc %d, "
"window %d, engram %s — loaded in %.2fs\n",
c->n_layers, c->n_routed, c->n_activated, c->dim, c->hc_mult, c->window,
m.engram.active ? "on" : "off", now_s() - started);
#ifdef COLI_VULKAN
/* After the weights, as in glm53: the device is an option, never a requirement.
* The matrices go up on their first multiply (see vk_mul), from this thread. */
if (!g_vk_opened) { /* opened while the weights loaded when they may live on the device only */
g_vk_thread = pthread_self();
g_vk_ready = coli_vk_init_env_tier("deepseek_v41", vkt_wanted() && c->n_routed > 0);
if (g_vk_ready) v41c_fit_now(&m); /* how many layers the chain places, before any upload */
const char *keep = getenv("COLI_VK_DENSE_HOST");
if (g_vk_ready && keep && *keep && atoi(keep) != 0) {
coli_vk_dense_host_decide("deepseek_v41", v41_dense_on(&m, v41c_decide(&m)), v41_dho_bytes(&m, v41_dense_layers(&m)));
if (g_v41_fit.L > 0) coli_vk_dense_host_layers(v41_dense_layers(&m), c->n_layers);
}
}
if (g_v41_dho)
coli_vk_dense_host_placed("deepseek_v41", g_v41_dho_kept
? "the embedding, the head, the router, norms and mixers, DSpark's stages, the vision tower; and matrices the device refused"
: "the embedding, the head, the router, norms and mixers, DSpark's stages, the vision tower");
v41c_start(&m); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */
vk_tier_start(&m, snap, cap);
v41c_atexit(); /* after the tier's: the chain goes before the device */
if (g_vk_ready && !vkt_ready() && !coli_vk_dense())
coli_vk_dense_decide("deepseek_v41", 0, 1); /* no tier after all: the trunk to the device */
#endif
if (getenv("SERVE") && atoi(getenv("SERVE"))) {
char path[1024];
snprintf(path, sizeof(path), "%s/tokenizer.json", snap);
Tok tokenizer;
tok_load(&tokenizer, path);
if (g_v41_mux_slots > 1) { /* slot 0 is the Model's own state */
g_v41_mux_seq = xcalloc((size_t)g_v41_mux_slots, sizeof *g_v41_mux_seq, "conversations");
for (int i = 0; i < g_v41_mux_slots; i++) v41_seq_alloc(&m, &g_v41_mux_seq[i], i > 0);
g_v41_mux_cur = 0;
}
const char *seed = getenv("SEED");
srand(seed ? (unsigned)strtoul(seed, NULL, 10) : (unsigned)time(NULL));
serve_loop(&m, &tokenizer, snap);
vk_report();
#ifdef COLI_VULKAN
vkt_report("run", m.hits, m.miss);
vk_tier_save();
#endif
return 0;
}
if (!ref_path) {
fprintf(stderr, "usage: SNAP=<dir> %s <cap> <ref.json> (or SERVE=1 for the "
"serve protocol)\n", argv[0]);
return 2;
}
FILE *f = fopen(ref_path, "rb");
if (!f) { fprintf(stderr, "%s: %s\n", ref_path, strerror(errno)); return 1; }
fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET);
char *text = xmalloc((size_t)size + 1, "reference");
if (fread(text, 1, (size_t)size, f) != (size_t)size) { fprintf(stderr, "short read\n"); return 1; }
text[size] = 0; fclose(f);
char *arena = NULL;
jval *root = json_parse(text, &arena);
if (!root) { fprintf(stderr, "%s is not valid JSON\n", ref_path); return 1; }
int n_prompt = 0, n_expected = 0;
int *prompt = load_ids(root, "prompt_ids", &n_prompt);
int *expected = load_ids(root, "output_ids", &n_expected);
if (!prompt || !expected) { fprintf(stderr, "%s lacks prompt_ids/output_ids\n", ref_path); return 1; }
/* the vision reference, when the fixture carries one: same patches in, the
* aligner's rows out, compared as floats rather than as tokens because nothing
* downstream of it is an argmax. */
jval *vision_ref = json_get(root, "vision");
int vision_failed = 0;
if (vision_ref && vision_ref->t == J_OBJ && m.vision) {
int n_h = (int)jnum(vision_ref, "grid_h", 0), n_w = (int)jnum(vision_ref, "grid_w", 0);
jval *patch_values = json_get(vision_ref, "patches");
jval *expected_values = json_get(vision_ref, "aligned");
int patch_in = 3 * c->vision_patch * c->vision_patch;
if (patch_values && expected_values && n_h > 0 && n_w > 0 &&
patch_values->len == n_h * n_w * patch_in) {
float *patches = xmalloc((size_t)patch_values->len * sizeof(float), "patches");
for (int i = 0; i < patch_values->len; i++) patches[i] = (float)patch_values->kids[i]->num;
int rows = vision_tokens(c, n_h, n_w);
float *aligned = xmalloc((size_t)rows * c->dim * sizeof(float), "aligned");
vision_forward(&m, m.vision, patches, n_h, n_w, aligned);
double worst = 0.0;
for (int i = 0; i < rows * c->dim && i < expected_values->len; i++) {
double delta = fabs(aligned[i] - expected_values->kids[i]->num);
if (delta > worst) worst = delta;
}
/* 2e-3: the reference rounds to five decimals in the JSON and both sides
* accumulate a 32-layer tower in fp32; a real divergence is orders larger. */
vision_failed = !(rows * c->dim == expected_values->len && worst < 2e-3);
fprintf(stderr, "[v41] vision: %d x %d patches -> %d rows, max |delta| %.2e%s\n",
n_h, n_w, rows, worst, vision_failed ? " MISMATCH" : "");
free(aligned); free(patches);
}
}
float *logits = xmalloc((size_t)c->vocab * sizeof(float), "logits");
double prefill_started = now_s();
forward(&m, prompt, n_prompt, logits);
if (m.spec.active) spec_step(&m, 0, m.last_start, m.last_rows, NULL, NULL);
double prefill = now_s() - prefill_started;
/* DSpark, when the fixture describes it. Two things are checked, and they are
* different things: that the draft head produces the vendor's block (the reference
* records one per step), and that a block VERIFIED in a single forward emits the
* same tokens as decoding them one at a time. The second is what makes accepting a
* draft safe, and a tiny fixture's random draft head never proposes anything worth
* accepting, so V41_SPEC_FORCE drafts the reference's own next tokens instead:
* 1 accepts the whole block, 2 corrupts the last draft so the round is rejected
* part way and the rollback runs. */
jval *spec_ref = json_get(root, "spec");
jval *spec_drafts = spec_ref && spec_ref->t == J_OBJ ? json_get(spec_ref, "drafts") : NULL;
jval *spec_conf = spec_ref && spec_ref->t == J_OBJ ? json_get(spec_ref, "confidence") : NULL;
int force = getenv("V41_SPEC_FORCE") ? atoi(getenv("V41_SPEC_FORCE")) : 0;
int block = m.spec.active ? c->spec_block : 0;
int *draft = block ? xmalloc((size_t)(block + 1) * sizeof(int), "drafts") : NULL;
float *confidence = block ? xmalloc((size_t)block * sizeof(float), "draft confidence") : NULL;
int spec_failed = 0, spec_checked = 0, round = 0;
uint64_t forced_prop = 0, forced_acc = 0, forced_rounds = 0;
int matched = 0;
double decode_started = now_s();
for (int step = 0; step < n_expected; ) {
int token = argmax(logits, c->vocab);
printf("%s%d", step ? " " : "", token);
if (token == expected[step]) matched++;
else fprintf(stderr, "\n[mismatch] step %d: got %d, reference %d\n", step, token, expected[step]);
step++;
if (step >= n_expected) break;
int one_at_a_time = m.last_rows == 1;
int drafted = m.spec.active
? spec_step(&m, token, m.last_start, m.last_rows, draft, confidence) : 0;
/* The reference drafts after every single-token forward. Once a round has been
* accepted the engine is a step ahead of that loop -- it committed several
* positions in one forward -- and its next block is a different, equally
* correct thing. Compare only where the two loops line up. */
if (drafted && one_at_a_time && spec_drafts && round < spec_drafts->len) {
jval *want = spec_drafts->kids[round];
jval *want_conf = spec_conf && round < spec_conf->len ? spec_conf->kids[round] : NULL;
for (int i = 0; i <= drafted && i < want->len; i++) {
int reference_id = (int)want->kids[i]->num;
if (draft[i] == reference_id) continue;
fprintf(stderr, "[dspark] round %d, draft %d: got %d, reference %d\n",
round, i, draft[i], reference_id);
spec_failed = 1;
}
for (int i = 0; want_conf && i < drafted && i < want_conf->len; i++) {
double delta = fabs(confidence[i] - want_conf->kids[i]->num);
if (delta < 2e-3) continue;
fprintf(stderr, "[dspark] round %d, confidence %d: got %.5f, reference %.5f\n",
round, i, confidence[i], want_conf->kids[i]->num);
spec_failed = 1;
}
spec_checked++;
}
if (drafted > 0 && one_at_a_time) round++; /* the prefill call drafts nothing */
if (force && drafted > 0) {
if (m.spec.max_verify > 0 && drafted > m.spec.max_verify) drafted = m.spec.max_verify;
int room = n_expected - step;
if (drafted > room) drafted = room;
/* 3 keeps the head's own drafts, which is what serving does; 1 and 2 put
* the reference's tokens in their place so the verification path runs at
* full width even on a fixture whose draft head is random noise */
if (force != 3) {
for (int i = 0; i < drafted; i++) draft[1 + i] = expected[step + i];
/* 2 rejects the last draft, 4 the first -- every drafted row is
* rolled back, which is what serving at 10-30% acceptance does most
* of the time -- and 5 a different one each round */
int bad = force == 2 ? drafted : force == 4 ? 1
: force == 5 ? 1 + (int)(forced_rounds % (uint64_t)drafted) : 0;
if (bad > 0 && drafted > 0)
draft[bad] = (draft[bad] + 1) % c->vocab; /* one bad draft */
forced_rounds++;
}
} else {
drafted = 0; /* the plain oracle decodes one at a time */
}
if (drafted > 0) {
draft[0] = token;
int accepted = spec_verify(&m, draft, drafted + 1, logits, 0.0f, 1.0f);
forced_prop += (uint64_t)drafted;
forced_acc += (uint64_t)accepted;
for (int i = 0; i < accepted && step < n_expected; i++) {
int drafted_token = draft[1 + i];
printf(" %d", drafted_token);
if (drafted_token == expected[step]) matched++;
else fprintf(stderr, "\n[mismatch] step %d: accepted draft %d, reference %d\n",
step, drafted_token, expected[step]);
step++;
}
} else {
forward(&m, &token, 1, logits);
}
}
double decode = now_s() - decode_started;
printf("\n");
fprintf(stderr, "Matching tokens: %d/%d\n", matched, n_expected);
fprintf(stderr, "[v41] prefill %d tokens in %.2fs | decode %d in %.2fs (%.2f tok/s)\n",
n_prompt, prefill, n_expected, decode, n_expected / (decode > 0 ? decode : 1));
fprintf(stderr, "[v41] experts: %llu hits, %llu misses, %.1f MB read | disk %.2fs, "
"expert matmul %.2fs, attention %.2fs, engram %.2fs\n",
(unsigned long long)m.hits, (unsigned long long)m.miss,
m.expert_bytes / 1e6, m.t_disk, m.t_expert, m.t_attn, m.t_engram);
mirror_report(NULL, NULL);
if (m.engram.active)
for (int t = 0; t < m.engram.n_layers; t++)
fprintf(stderr, "[v41] engram table %d (layer %d): %lld rows, %llu cache hits, "
"%llu reads\n", t, m.engram.layer_of[t],
(long long)m.engram.table[t].rows,
(unsigned long long)m.engram.table[t].hits,
(unsigned long long)m.engram.table[t].misses);
if (spec_checked)
fprintf(stderr, "[v41] DSpark: checked %d draft round%s against the reference%s\n",
spec_checked, spec_checked == 1 ? "" : "s", spec_failed ? " MISMATCH" : "");
if (forced_prop)
fprintf(stderr, "[v41] DSpark: %llu of %llu forced drafts accepted, %d forwards "
"for %d tokens\n", (unsigned long long)forced_acc,
(unsigned long long)forced_prop, (int)m.forwards, n_expected);
if (force && !forced_prop) {
fprintf(stderr, "[v41] V41_SPEC_FORCE is set but nothing was drafted: the "
"verification path was NOT exercised\n");
spec_failed = 1;
}
vk_report();
#ifdef COLI_VULKAN
vkt_report("run", m.hits, m.miss);
vk_tier_save();
#endif
free(confidence); free(draft);
free(logits); free(prompt); free(expected); json_free(root); free(arena); free(text);
return (matched == n_expected && !vision_failed && !spec_failed) ? 0 : 1;
}