Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

inkling: ring-buffer KV cache for sliding-window layers

Davide Panelli7fefa8ccda00
diff --git a/c/inkling.c b/c/inkling.c
index 52fba9ff..62732d6f 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -141,7 +141,7 @@ typedef struct {
uint64_t clock, hits, miss;
uint64_t ereq, euse; /* routed richiesti (topk) vs usati dopo TOPP */
double t_fill, t_expert, t_shared, t_attn, t_route; /* phase timers */
- float **K, **V; int kv_len, max_t; /* per-layer [kv][max_t][hd] */
+ float **K, **V; int kv_len, max_t; /* per-layer [kv][kv_ring_rows][hd]; sliding layers are a t%window ring */
float **cs[4]; /* conv states, [n_layers][C*(K-1)] */
double dense_load_s;
/* KV prefix reuse: what the current K/V and conv states were built from.
@@ -1058,11 +1058,23 @@ static int usage_save(Model *m, const char *snap) {
return rt_save(up, 1);
}
+/* KV rows actually kept per layer: sliding layers only ever attend to the last
+ * `window` positions (t0 clamp in attention), so their cache is a ring of
+ * `window` rows instead of max_t — at 32k context that is a ~64x cut on the
+ * 5-of-6 sliding layers. Global layers keep the full max_t. Must be computed
+ * from the SAME max_t the buffers were allocated with (m->max_t). */
+static inline int kv_ring_rows(const Cfg *c, int li, int max_t) {
+ return (c->local[li] && c->window > 0 && c->window < max_t) ? c->window : max_t;
+}
+
/* ---------- attention (GQA + sliding/global + relative bias + K/V sconv) ---------- */
static void attention(Model *m, Layer *l, int li, float *x, int S, int pos0, float *out) {
Cfg *c = &m->c;
int D = c->hidden, H = L_HEADS(c,li), KV = L_KV(c,li), hd = L_HD(c,li), ext = L_EXT(c,li);
int local = c->local[li];
+ /* the ring made an over-run silent (t%win wraps instead of writing OOB), so
+ * fail fast here: every caller sizes the cache via kv_alloc before stepping */
+ if (pos0 + S > m->max_t) { fprintf(stderr, "attention: pos %d+%d exceeds kv alloc %d\n", pos0, S, m->max_t); exit(1); }
int qdim = H*hd, kvdim = KV*hd, group = H/KV;
float *q = falloc((int64_t)S*qdim);
float *k = falloc((int64_t)S*kvdim);
@@ -1080,12 +1092,12 @@ static void attention(Model *m, Layer *l, int li, float *x, int S, int pos0, flo
for (int h = 0; h < H; h++) rmsnorm_row(q + (int64_t)s*qdim + h*hd, q + (int64_t)s*qdim + h*hd, l->qn, hd, c->eps);
for (int h = 0; h < KV; h++) rmsnorm_row(k + (int64_t)s*kvdim + h*hd, k + (int64_t)s*kvdim + h*hd, l->kn, hd, c->eps);
}
- /* append K,V to the cache */
- for (int s = 0; s < S; s++) for (int h = 0; h < KV; h++) {
- int t = pos0 + s;
- memcpy(m->K[li] + ((int64_t)h*m->max_t + t)*hd, k + (int64_t)s*kvdim + h*hd, hd*sizeof(float));
- memcpy(m->V[li] + ((int64_t)h*m->max_t + t)*hd, vv + (int64_t)s*kvdim + h*hd, hd*sizeof(float));
- }
+ /* Rows for positions inside this batch are read from the k/vv scratch, not
+ * the cache: with a ring, appending the whole batch up front could overwrite
+ * history rows that earlier queries in the batch still need. The scratch
+ * holds exactly what the cache would (post-sconv, post-rmsnorm), so the
+ * arithmetic is unchanged; the cache is appended after scoring. */
+ int win = kv_ring_rows(c, li, m->max_t);
float scale = 1.f / (float)hd;
float *ctx = falloc((int64_t)S*qdim);
#pragma omp parallel
@@ -1097,6 +1109,7 @@ static void attention(Model *m, Layer *l, int li, float *x, int S, int pos0, flo
for (int s = 0; s < S; s++) {
int qpos = pos0 + s;
int t0 = local && qpos - c->window + 1 > 0 ? qpos - c->window + 1 : 0;
+ int tb = pos0 > t0 ? pos0 : t0; /* first row served by the scratch */
/* mix the relative-bias bank for this (token, head): rl[dist] */
const float *rv = rr + (int64_t)s*H*c->d_rel + h*c->d_rel;
for (int e = 0; e < ext; e++) {
@@ -1111,9 +1124,11 @@ static void attention(Model *m, Layer *l, int li, float *x, int S, int pos0, flo
if (en > 1.0) tau = 1.f + c->log_alpha * (float)log(en);
}
const float *qv = q + (int64_t)s*qdim + h*hd;
- const float *Kh = m->K[li] + ((int64_t)(h/group)*m->max_t)*hd;
+ const float *Kh = m->K[li] + ((int64_t)(h/group)*win)*hd;
+ const float *Kb = k + (int64_t)(h/group)*hd;
for (int t = t0; t <= qpos; t++) {
- const float *kv = Kh + (int64_t)t*hd;
+ const float *kv = t < tb ? Kh + (int64_t)(t % win)*hd
+ : Kb + (int64_t)(t - pos0)*kvdim;
float acc = 0.f;
for (int d = 0; d < hd; d++) acc += qv[d]*kv[d];
int dist = qpos - t;
@@ -1123,9 +1138,11 @@ static void attention(Model *m, Layer *l, int li, float *x, int S, int pos0, flo
softmax_row(sc, n);
float *cx = ctx + (int64_t)s*qdim + h*hd;
for (int d = 0; d < hd; d++) cx[d] = 0.f;
- const float *Vh = m->V[li] + ((int64_t)(h/group)*m->max_t)*hd;
+ const float *Vh = m->V[li] + ((int64_t)(h/group)*win)*hd;
+ const float *Vb = vv + (int64_t)(h/group)*hd;
for (int t = t0; t <= qpos; t++) {
- const float *vrow = Vh + (int64_t)t*hd;
+ const float *vrow = t < tb ? Vh + (int64_t)(t % win)*hd
+ : Vb + (int64_t)(t - pos0)*kvdim;
float a = sc[t - t0];
for (int d = 0; d < hd; d++) cx[d] += a * vrow[d];
}
@@ -1133,6 +1150,14 @@ static void attention(Model *m, Layer *l, int li, float *x, int S, int pos0, flo
}
free(rl); free(sc);
}
+ /* append K,V to the cache (ring on sliding layers); rows the ring would
+ * overwrite within this same batch are skipped, they can never be read */
+ int s0 = S - win > 0 ? S - win : 0;
+ for (int s = s0; s < S; s++) for (int h = 0; h < KV; h++) {
+ int t = pos0 + s;
+ memcpy(m->K[li] + ((int64_t)h*win + t % win)*hd, k + (int64_t)s*kvdim + h*hd, hd*sizeof(float));
+ memcpy(m->V[li] + ((int64_t)h*win + t % win)*hd, vv + (int64_t)s*kvdim + h*hd, hd*sizeof(float));
+ }
matmul_w(out, ctx, l->o, S, qdim, D);
free(q); free(k); free(vv); free(rr); free(ctx);
}
@@ -1575,29 +1600,52 @@ static void kv_alloc(Model *m, int max_t) {
* for a larger max_t every turn, so the state was thrown away just before
* the point of using it.
*
- * K/V are laid out [kv_head][max_t][hd], so a larger max_t changes the
- * stride: the old contents cannot be realloc'd, they have to be re-laid-out
- * head by head. That copy costs a memcpy of what is already computed, which
- * is nothing beside re-running the prefill that produced it. */
+ * K/V are laid out [kv_head][rows][hd] with rows = kv_ring_rows(), so a
+ * larger max_t changes the stride wherever rows follows max_t: those
+ * contents cannot be realloc'd, they have to be re-laid-out head by head.
+ * That copy costs a memcpy of what is already computed, which is nothing
+ * beside re-running the prefill that produced it.
+ *
+ * Sliding layers whose ring is already window rows are the exception in
+ * BOTH directions: their size does not depend on max_t (nothing to grow),
+ * and a ring that has wrapped is not linear (position t lives at row
+ * t % window), so the linear copy below would silently rotate it. Steal
+ * the buffer instead — the slot map is unchanged, so it stays valid.
+ * Every layer that does reach the copy IS linear: rows != old_rows only
+ * happens when old_rows == old_max (a wrapped ring keeps rows == window
+ * forever), and keep <= old_max <= rows there, so the copy fits. */
float **oldK = m->K, **oldV = m->V;
int old_max = m->max_t;
int keep = (m->K && m->kv_len > 0 && m->kv_len <= max_t) ? m->kv_len : 0;
m->max_t = max_t;
m->K = calloc(c->n_layers, sizeof(float*)); m->V = calloc(c->n_layers, sizeof(float*));
+ int64_t ring = 0, full = 0;
for (int i = 0; i < c->n_layers; i++) {
int kv = L_KV(c,i), hd = L_HD(c,i);
- m->K[i] = falloc((int64_t)kv * max_t * hd);
- m->V[i] = falloc((int64_t)kv * max_t * hd);
- for (int h = 0; h < kv && keep; h++) {
- memcpy(m->K[i] + (int64_t)h * max_t * hd,
- oldK[i] + (int64_t)h * old_max * hd, (size_t)keep * hd * sizeof(float));
- memcpy(m->V[i] + (int64_t)h * max_t * hd,
- oldV[i] + (int64_t)h * old_max * hd, (size_t)keep * hd * sizeof(float));
+ int64_t rows = kv_ring_rows(c, i, max_t);
+ int64_t old_rows = oldK ? kv_ring_rows(c, i, old_max) : 0;
+ if (oldK && rows == old_rows) {
+ m->K[i] = oldK[i]; m->V[i] = oldV[i];
+ oldK[i] = NULL; oldV[i] = NULL;
+ } else {
+ m->K[i] = falloc((int64_t)kv * rows * hd);
+ m->V[i] = falloc((int64_t)kv * rows * hd);
+ for (int h = 0; h < kv && keep; h++) {
+ memcpy(m->K[i] + (int64_t)h * rows * hd,
+ oldK[i] + (int64_t)h * old_max * hd, (size_t)keep * hd * sizeof(float));
+ memcpy(m->V[i] + (int64_t)h * rows * hd,
+ oldV[i] + (int64_t)h * old_max * hd, (size_t)keep * hd * sizeof(float));
+ }
}
+ ring += 2 * (int64_t)kv * rows * hd * (int64_t)sizeof(float);
+ full += 2 * (int64_t)kv * max_t * hd * (int64_t)sizeof(float);
}
if (oldK) for (int i = 0; i < c->n_layers; i++) { free(oldK[i]); free(oldV[i]); }
free(oldK); free(oldV);
+ if (ring < full)
+ fprintf(stderr, "[kv] %.1f MiB (ring buffers on sliding layers; full cache would be %.1f MiB)\n",
+ ring/1048576.0, full/1048576.0);
/* the record describes those same positions, so it survives with them --
* unless its own allocation fails, in which case reuse simply stops. */