Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

inkling: the per-position heads run over blocks of positions

JustVugg8b59118e5ace
diff --git a/c/inkling.c b/c/inkling.c
index effd1a62..f5fcb55f 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -2074,24 +2074,49 @@ static float *step_mm(Model *m, const int *ids, int S, int pos0, int *tf_out,
/* Lettura del prefill: un passaggio di lm_head per posizione, pagato solo
* da chi ha chiesto il canale. La posizione p predice il token p+1; il
* primo token fresco e predetto dalla fotografia. */
- if (g_echo_k > 0 && g_echo_id && S > 0) {
- if (g_pin_use_logit && g_pin_logit)
- ink_echo(g_echo_id, pos0, ids[0], g_pin_logit, c->unpad_vocab, g_echo_k);
- for (int p = 0; p + 1 < S; p++) {
- rmsnorm_row(last, x + (int64_t)p*D, m->final_norm, D, c->eps);
- for (int d = 0; d < D; d++) last[d] /= c->mup;
- matmul_w(logit, last, m->lm_head, 1, D, c->unpad_vocab);
- ink_echo(g_echo_id, pos0 + p + 1, ids[p+1], logit, c->unpad_vocab, g_echo_k);
+ /* The per-position heads (the logprobs channel, teacher forcing) run over a
+ * block of positions at a time: one S-row lm_head call (a device GEMM) instead
+ * of one per position. Every kernel matmul_w uses computes a row as the one-row
+ * call does, so the logits are the same bits. */
+ int hb = (int)((64LL << 20) / ((int64_t)c->unpad_vocab * (int64_t)sizeof(float)));
+ hb = hb < 1 ? 1 : hb > 64 ? 64 : hb;
+ if ((g_echo_k > 0 && g_echo_id && S > 0) || tf_out) {
+ float *normed = falloc((int64_t)hb * D), *logits = falloc((int64_t)hb * c->unpad_vocab);
+ if (g_echo_k > 0 && g_echo_id && S > 0) {
+ if (g_pin_use_logit && g_pin_logit)
+ ink_echo(g_echo_id, pos0, ids[0], g_pin_logit, c->unpad_vocab, g_echo_k);
+ for (int base = 0; base + 1 < S; base += hb) {
+ int rows = S - 1 - base < hb ? S - 1 - base : hb;
+ for (int r = 0; r < rows; r++) {
+ float *nr = normed + (int64_t)r*D;
+ rmsnorm_row(nr, x + (int64_t)(base + r)*D, m->final_norm, D, c->eps);
+ for (int d = 0; d < D; d++) nr[d] /= c->mup;
+ }
+ matmul_w(logits, normed, m->lm_head, rows, D, c->unpad_vocab);
+ for (int r = 0; r < rows; r++) {
+ int p = base + r;
+ ink_echo(g_echo_id, pos0 + p + 1, ids[p+1], logits + (int64_t)r * c->unpad_vocab,
+ c->unpad_vocab, g_echo_k);
+ }
+ }
}
- }
- if (tf_out) {
- for (int s = 0; s < S; s++) {
- rmsnorm_row(last, x + (int64_t)s*D, m->final_norm, D, c->eps);
- for (int d = 0; d < D; d++) last[d] /= c->mup;
- matmul_w(logit, last, m->lm_head, 1, D, c->unpad_vocab);
- int best = 0; for (int i = 1; i < c->unpad_vocab; i++) if (logit[i] > logit[best]) best = i;
- tf_out[pos0 + s] = best;
+ if (tf_out) {
+ for (int base = 0; base < S; base += hb) {
+ int rows = S - base < hb ? S - base : hb;
+ for (int r = 0; r < rows; r++) {
+ float *nr = normed + (int64_t)r*D;
+ rmsnorm_row(nr, x + (int64_t)(base + r)*D, m->final_norm, D, c->eps);
+ for (int d = 0; d < D; d++) nr[d] /= c->mup;
+ }
+ matmul_w(logits, normed, m->lm_head, rows, D, c->unpad_vocab);
+ for (int r = 0; r < rows; r++) {
+ const float *lr = logits + (int64_t)r * c->unpad_vocab;
+ int best = 0; for (int i = 1; i < c->unpad_vocab; i++) if (lr[i] > lr[best]) best = i;
+ tf_out[pos0 + base + r] = best;
+ }
+ }
}
+ free(normed); free(logits);
}
rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps);
for (int d = 0; d < D; d++) last[d] /= c->mup;