Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

inkling: overlap shared experts with the last GPU round

Kyle Kelleycda2e4ee03d8
diff --git a/c/inkling.c b/c/inkling.c
index c698d587..48147e34 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -1128,6 +1128,28 @@ static void dense_mlp(Model *m, Layer *l, float *x, int S, float *out) {
* (2) fill ALL missing experts in one parallel burst (the NVMe wants queue
* depth — during prefill this batches the whole sequence's misses), then
* (3) compute. */
+/* shared experts for all S positions: gamma inside (before down_proj is
+ * linear, so applied at the end). Factored out so the Metal path can run it
+ * on the CPU while the last routed-expert round is in flight on the GPU. */
+static void shared_experts_cpu(Model *m, Layer *l, const float *x, int S,
+ float *out, const float *wgt,
+ float *g, float *u, float *hh) {
+ Cfg *c = &m->c;
+ int D = c->hidden, K = c->topk, I = c->moe_inter, ns = c->n_shared;
+ for (int s = 0; s < S; s++) {
+ const float *xs = x + (int64_t)s*D;
+ float *os = out + (int64_t)s*D;
+ const float *w = wgt + (int64_t)s*(K+ns);
+ for (int j = 0; j < ns; j++) {
+ matmul_w(g, xs, wt_off_i(l->sh_g, (int64_t)j*I*D, D), 1, D, I);
+ matmul_w(u, xs, wt_off_i(l->sh_u, (int64_t)j*I*D, D), 1, D, I);
+ for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];
+ matmul_w(hh, g, wt_off_i(l->sh_d, (int64_t)j*D*I, I), 1, I, D);
+ for (int d = 0; d < D; d++) os[d] += w[K+j] * hh[d];
+ }
+ }
+}
+
static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
Cfg *c = &m->c;
int D = c->hidden, E = c->n_experts, K = c->topk, I = c->moe_inter, ns = c->n_shared;
@@ -1201,6 +1223,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
int cap = m->cache[layer].cap; if (cap < 1) cap = 1;
float *g = falloc(2*I), *u = g + I, *hh = falloc(D);
int q4 = m->xq && m->rb13*2 == D; /* packed int4 vs int8 container */
+ int shared_done = 0; /* set when overlapped with the last GPU round */
int64_t npair = (int64_t)S*K;
#ifdef COLI_METAL
/* per-round scratch for the batched GPU submit: pairs grouped by expert,
@@ -1279,6 +1302,28 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
mgp[j] = e->p13; mup[j] = e->p13 + (int64_t)I*m->rb13; mdp[j] = e->p2;
mgs[j] = e->s13; mus[j] = e->s13 + I; mds[j] = e->s2;
}
+ /* Last round: submit async and run the shared experts on the
+ * CPU while the GPU computes the routed ones — they are
+ * independent (both read x, both accumulate into out, and the
+ * GPU only touches out in _end's scatter-add, after the wait).
+ * On a GPU fault _end returns 0 and the CPU loop below redoes
+ * the round; shared_done stays set either way. */
+ if (base + cap >= npair) {
+ ColiMetalMoeHandle *h = coli_metal_moe_block_begin(
+ nb, D, I, q4 ? 2 : 1, mgp, mup, mdp, mgs, mus, mds,
+ mxg, mxoff, mnr, mrows, mrw);
+ if (h) {
+ double ts = now_s();
+ shared_experts_cpu(m, l, x, S, out, wgt, g, u, hh);
+ double sh = now_s() - ts;
+ m->t_shared += sh;
+ shared_done = 1;
+ int ok = coli_metal_moe_block_end(h, out);
+ m->t_expert += (now_s() - te) - sh;
+ if (ok) continue; /* round done on the GPU */
+ te = now_s(); /* fault: CPU redo below */
+ }
+ }
if (coli_metal_moe_block(nb, D, I, q4 ? 2 : 1, mgp, mup, mdp, mgs, mus, mds,
mxg, mxoff, mnr, mrows, mrw, out, S)) {
m->t_expert += now_s() - te;
@@ -1317,20 +1362,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
}
m->t_expert += now_s() - te;
}
- /* shared experts: una volta per token, fuori dai giri (non usano la cache) */
- for (int s = 0; s < S; s++) {
- const float *xs = x + (int64_t)s*D;
- float *os = out + (int64_t)s*D;
- float *w = wgt + (int64_t)s*(K+ns);
+ /* shared experts: una volta per token, fuori dai giri (non usano la cache).
+ * Under Metal the last routed round overlaps this on the CPU (see the
+ * begin/end path above) and sets shared_done. */
+ if (!shared_done) {
double ts = now_s();
- /* gamma inside (before down_proj is linear, so applied at the end) */
- for (int j = 0; j < ns; j++) {
- matmul_w(g, xs, wt_off_i(l->sh_g, (int64_t)j*I*D, D), 1, D, I);
- matmul_w(u, xs, wt_off_i(l->sh_u, (int64_t)j*I*D, D), 1, D, I);
- for (int i = 0; i < I; i++) g[i] = siluf(g[i]) * u[i];
- matmul_w(hh, g, wt_off_i(l->sh_d, (int64_t)j*D*I, I), 1, I, D);
- for (int d = 0; d < D; d++) os[d] += w[K+j] * hh[d];
- }
+ shared_experts_cpu(m, l, x, S, out, wgt, g, u, hh);
m->t_shared += now_s() - ts;
}
free(logits); free(idx); free(keff); free(wgt); free(use); free(fill); free(fl);