Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vk_tier: the routed-expert tier the MoE engines share

JustVugg53310db844b4
diff --git a/c/Makefile b/c/Makefile
index 90cf23d5..2a91104c 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -527,7 +527,7 @@ TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_frami
test_edge_adapters_registration test_edge_adapters_real \
test_xdna_execution test_xdna_failure \
test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_st_f16_bf16_simd_sse41 test_qwen38_tier_engine \
- test_e8x4g64_loader test_qwenimage_vae \
+ test_e8x4g64_loader test_qwenimage_vae test_vk_tier \
test_i4_grouped_sse41 test_i4_grouped_sse41_o1 test_i4_grouped_sse41_no_contract
TEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))
ifneq (,$(LINUX))
@@ -703,7 +703,9 @@ LDFLAGS += -Wl,-Bdynamic -lvulkan-1 -Wl,-Bstatic
else
LDFLAGS += -lvulkan
endif
-VK_OBJ = backend_vulkan.o
+# vk_tier.o: the routed-expert tier the MoE engines share (vk_tier.h); an engine
+# that does not call it links nothing from it.
+VK_OBJ = backend_vulkan.o vk_tier.o
VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv \
shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv shaders/expert_act.spv
endif
@@ -839,6 +841,8 @@ colibri$(EXE): colibri.c $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) $(XDNA_OBJ
# Vulkan backend object (plain C + vulkan headers) and its SPIR-V shaders.
backend_vulkan.o: backend_vulkan.c .build-config
$(CC) $(CFLAGS) -c backend_vulkan.c -o $@
+vk_tier.o: vk_tier.c .build-config
+ $(CC) $(CFLAGS) -c vk_tier.c -o $@
shaders/%.spv: shaders/%.comp
@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }
$(GLSLC) --target-env=vulkan1.2 $< -o $@
@@ -1942,6 +1946,17 @@ tests/test_expert_ffn$(EXE): tests/test_expert_ffn.c
tests/test_vk_alloc$(EXE): tests/test_vk_alloc.c
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+# The Vulkan routed-expert tier (vk_tier.c) against a CPU reference: every source
+# format, warm start, eviction, partial batches. It needs VK=1 and a Vulkan device
+# (CI: Lavapipe), so it stays out of `check` (TEST_EXCLUDE) and runs from
+# vk-tier-check.
+tests/test_vk_tier$(EXE): tests/test_vk_tier.c $(VK_OBJ) $(VK_SPV)
+ @[ "$(VK)" = 1 ] || { echo "tests/test_vk_tier needs VK=1" >&2; exit 1; }
+ $(CC) $(CFLAGS) $< $(VK_OBJ) -o $@ $(LDFLAGS)
+.PHONY: vk-tier-check
+vk-tier-check: tests/test_vk_tier$(EXE)
+ ./tests/test_vk_tier$(EXE) shaders/qmatmul.spv
+
tests/test_rans$(EXE): tests/test_rans.c
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
@@ -2728,7 +2743,7 @@ AUTODEP_OWN_FLAGS = bench_cuda_resident_batch bench_load_tq_peak_rss \
bench_indexer_allocations
ENGINE_RULES := $(foreach n,$(RULE_NAMES),$(if $(findstring /,$(n)),,$(if $(wildcard $(n).c),$(n))))
AUTODEP_TESTS := $(patsubst tests/%,%,$(filter tests/%,$(RULE_NAMES)))
-AUTODEP_OBJS = backend_vulkan.o backend_loader.o backend_xdna.o qwen36_tier.o \
+AUTODEP_OBJS = backend_vulkan.o vk_tier.o backend_loader.o backend_xdna.o qwen36_tier.o \
tests/backend_xdna_lane.o tests/edge_runtime.o tests/edge_runtime_nocuda.o \
tests/segment_runtime.o tests/segment_runtime_nocuda.o \
tests/segment_conformance_fixtures.o tests/deepseek_v4_native_quant.o \
diff --git a/c/tests/test_vk_tier.c b/c/tests/test_vk_tier.c
new file mode 100644
index 00000000..37d7b6f8
--- /dev/null
+++ b/c/tests/test_vk_tier.c
@@ -0,0 +1,342 @@
+/* The Vulkan routed-expert tier (vk_tier.c) against a CPU reference, on any Vulkan
+ * device (CI: Lavapipe). A small synthetic MoE -- L layers of E experts, top-K --
+ * whose weights are random in every source format the tier accepts, decoded here
+ * on their own, independently of the tier and of the backend. Gates:
+ * formats every VktSrc kind and both activations: each row the device returns
+ * equals the reference expert output, and the rank-order sum of a mixed
+ * step (device and CPU rows) equals the all-CPU sum, within 2e-3;
+ * warm a history fills the budget in heat order (vkt_plan / vkt_put);
+ * adapt a budget of a third of the experts and a hot set that moves halfway:
+ * experts are evicted, the new hot set becomes resident and its steps
+ * go to the device;
+ * partial a step with more assignments than max_rows: the device takes
+ * max_rows of them, the CPU the rest, the sum is still right;
+ * books device + CPU = routed, resident <= budget, no failed upload.
+ *
+ * make tests/test_vk_tier VK=1 && VK_ICD_FILENAMES=.../lvp_icd.json ./tests/test_vk_tier */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include "../backend_vulkan.h"
+#include "../vk_tier.h"
+
+static int fails;
+#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL %s:%d: ", __FILE__, __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
+
+enum { L = 2, E = 10, H = 192, F = 128, K = 3 };
+static unsigned rng = 7;
+static unsigned rnd(void) { rng = rng * 1103515245u + 12345u; return rng >> 8; }
+static float frnd(void) { return (float)((int)(rnd() % 2001) - 1000) / 1000.0f; }
+
+/* ---- one matrix in a source format, and its own decode ------------------------- */
+typedef struct { VktFmt f; int I, O; uint8_t *codes; void *scales; float *w; } Mat;
+static size_t row_bytes(VktSrc k, int I) {
+ switch (k) {
+ case VKT_SRC_I8_ROW: case VKT_SRC_I8_GS: case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I8_AS_I4_GS:
+ case VKT_SRC_FP8_GS: case VKT_SRC_FP8_BLOCK: return (size_t)I;
+ case VKT_SRC_I3_G64: return ((size_t)I + 63) / 64 * 24;
+ case VKT_SRC_BF16: return (size_t)I * 2;
+ case VKT_SRC_F32: return (size_t)I * 4;
+ default: return ((size_t)I + 1) / 2;
+ }
+}
+static float e4m3(uint8_t b) {
+ int e = (b >> 3) & 15, m = b & 7;
+ float v = e == 0 ? m / 512.0f : ldexpf(1.0f + m / 8.0f, e - 7);
+ return (b & 0x80) ? -v : v;
+}
+static int nib(const uint8_t *row, int i) { return (row[i >> 1] >> ((i & 1) * 4)) & 15; }
+/* the weight value (o, i) before scales, as the format defines it */
+static float code(const Mat *m, int o, int i) {
+ const uint8_t *row = m->codes + (size_t)o * row_bytes(m->f.kind, m->I);
+ switch (m->f.kind) {
+ case VKT_SRC_I8_ROW: case VKT_SRC_I8_GS: case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I8_AS_I4_GS:
+ return (float)(int8_t)row[i];
+ case VKT_SRC_I4S_PAIRS_ROW: case VKT_SRC_I4S_PAIRS_GS: { int v = nib(row, i); return (float)(v >= 8 ? v - 16 : v); }
+ case VKT_SRC_I4U_PAIRS_ROW: case VKT_SRC_I4U_PAIRS_GS: return (float)(nib(row, i) - 8);
+ case VKT_SRC_I4U_PLANAR64: {
+ int full = m->I / 64 * 64;
+ if (i >= full) return (float)(nib(row + full / 2, i - full) - 8);
+ const uint8_t *blk = row + (size_t)(i / 64) * 32; int k = i % 64;
+ return (float)((k < 32 ? (blk[k] & 15) : (blk[k - 32] >> 4)) - 8);
+ }
+ case VKT_SRC_I3_G64: {
+ const uint8_t *lo = row + (size_t)(i / 64) * 24, *hi = lo + 16; int j = i % 64;
+ return (float)((int)(((lo[j >> 2] >> ((j & 3) * 2)) & 3u) | (((hi[j >> 3] >> (j & 7)) & 1u) << 2)) - 4);
+ }
+ case VKT_SRC_MXFP4_F32: case VKT_SRC_MXFP4_E8M0: {
+ static const float lut[8] = {0, .5f, 1, 1.5f, 2, 3, 4, 6};
+ int v = nib(row, i); return (v & 8) ? -lut[v & 7] : lut[v & 7];
+ }
+ case VKT_SRC_FP8_GS: case VKT_SRC_FP8_BLOCK: return e4m3(row[i]);
+ case VKT_SRC_BF16: { uint16_t h; memcpy(&h, row + 2 * i, 2); uint32_t u = (uint32_t)h << 16; float f; memcpy(&f, &u, 4); return f; }
+ case VKT_SRC_F32: { float f; memcpy(&f, row + 4 * i, 4); return f; }
+ default: return 0;
+ }
+}
+static int per_row(VktSrc k) { return k == VKT_SRC_I8_ROW || k == VKT_SRC_I8_AS_I4_ROW || k == VKT_SRC_I4S_PAIRS_ROW || k == VKT_SRC_I4U_PAIRS_ROW; }
+static float scale(const Mat *m, int o, int i) {
+ VktSrc k = m->f.kind;
+ if (k == VKT_SRC_BF16 || k == VKT_SRC_F32) return 1.0f;
+ if (per_row(k)) return ((const float *)m->scales)[o];
+ int gs = k == VKT_SRC_I4U_PLANAR64 || k == VKT_SRC_I3_G64 ? 64 : m->f.gs, ng = (m->I + gs - 1) / gs;
+ if (k == VKT_SRC_MXFP4_E8M0) return ldexpf(1.0f, ((const uint8_t *)m->scales)[(size_t)o * ng + i / gs] - 127);
+ if (k == VKT_SRC_FP8_BLOCK) return ((const float *)m->scales)[(size_t)(o / gs) * ng + i / gs];
+ return ((const float *)m->scales)[(size_t)o * ng + i / gs];
+}
+static void mat_make(Mat *m, VktFmt f, int I, int O) {
+ m->f = f; m->I = I; m->O = O;
+ size_t rb = row_bytes(f.kind, I);
+ m->codes = malloc(rb * O);
+ for (size_t i = 0; i < rb * O; i++) m->codes[i] = (uint8_t)rnd();
+ if (f.kind == VKT_SRC_I8_AS_I4_ROW || f.kind == VKT_SRC_I8_AS_I4_GS) /* an unpacked int4: [-8, 7] */
+ for (size_t i = 0; i < rb * O; i++) m->codes[i] = (uint8_t)(int8_t)((int)(m->codes[i] % 16) - 8);
+ if (f.kind == VKT_SRC_FP8_GS || f.kind == VKT_SRC_FP8_BLOCK)
+ for (size_t i = 0; i < rb * O; i++) if ((m->codes[i] & 0x7f) == 0x7f) m->codes[i] ^= 1;
+ if (f.kind == VKT_SRC_BF16 || f.kind == VKT_SRC_F32)
+ for (int i = 0; i < I * O; i++) {
+ float v = frnd() * 0.1f;
+ if (f.kind == VKT_SRC_F32) memcpy(m->codes + 4 * (size_t)i, &v, 4);
+ else { uint32_t u; memcpy(&u, &v, 4); uint16_t h = (uint16_t)(u >> 16); memcpy(m->codes + 2 * (size_t)i, &h, 2); }
+ }
+ int gs = f.kind == VKT_SRC_I4U_PLANAR64 || f.kind == VKT_SRC_I3_G64 ? 64 : f.gs, ng = gs ? (I + gs - 1) / gs : 1;
+ size_t ns = per_row(f.kind) ? (size_t)O : f.kind == VKT_SRC_FP8_BLOCK ? (size_t)((O + gs - 1) / gs) * ng : (size_t)O * ng;
+ if (f.kind == VKT_SRC_MXFP4_E8M0) {
+ uint8_t *s = malloc(ns); for (size_t i = 0; i < ns; i++) s[i] = (uint8_t)(120 + rnd() % 4); m->scales = s;
+ } else if (f.kind == VKT_SRC_BF16 || f.kind == VKT_SRC_F32) m->scales = NULL;
+ else { float *s = malloc(ns * 4); for (size_t i = 0; i < ns; i++) s[i] = 0.004f + (rnd() % 100) / 25000.0f; m->scales = s; }
+ m->w = malloc((size_t)I * O * sizeof(float));
+ for (int o = 0; o < O; o++) for (int i = 0; i < I; i++) m->w[(size_t)o * I + i] = code(m, o, i) * scale(m, o, i);
+}
+static void mat_free(Mat *m) { free(m->codes); free(m->scales); free(m->w); }
+
+/* ---- the synthetic model ---------------------------------------------------------- */
+typedef struct { Mat g, u, d; } Ex;
+static Ex ex[L][E];
+static int g_act; static float g_limit;
+static void expert_ref(const Ex *e, const float *x, float *y) {
+ float h[F];
+ for (int o = 0; o < F; o++) {
+ double a = 0, b = 0;
+ for (int i = 0; i < H; i++) { a += (double)e->g.w[(size_t)o * H + i] * x[i]; b += (double)e->u.w[(size_t)o * H + i] * x[i]; }
+ float g = (float)a, u = (float)b;
+ if (g_act == VKT_ACT_SITU) h[o] = 4.f * tanhf(g / 4.f) * (1.f / (1.f + expf(-g))) * (25.f * tanhf(u / 25.f));
+ else {
+ if (g_limit > 0) { if (g > g_limit) g = g_limit; if (u > g_limit) u = g_limit; if (u < -g_limit) u = -g_limit; }
+ h[o] = g / (1.f + expf(-g)) * u;
+ }
+ }
+ for (int o = 0; o < H; o++) { double a = 0; for (int i = 0; i < F; i++) a += (double)e->d.w[(size_t)o * F + i] * h[i]; y[o] = (float)a; }
+}
+static VktExpertSrc src_of(const Ex *e) {
+ return (VktExpertSrc){e->g.codes, e->u.codes, e->d.codes, e->g.scales, e->u.scales, e->d.scales};
+}
+static void model_make(VktFmt gu, VktFmt dn) {
+ for (int l = 0; l < L; l++) for (int e = 0; e < E; e++) {
+ mat_make(&ex[l][e].g, gu, H, F); mat_make(&ex[l][e].u, gu, H, F); mat_make(&ex[l][e].d, dn, F, H);
+ }
+}
+static void model_free(void) {
+ for (int l = 0; l < L; l++) for (int e = 0; e < E; e++) { mat_free(&ex[l][e].g); mat_free(&ex[l][e].u); mat_free(&ex[l][e].d); }
+}
+static double rel(const float *a, const float *b, int n) {
+ double num = 0, den = 0;
+ for (int i = 0; i < n; i++) { double d = (double)a[i] - b[i]; num += d * d; den += (double)b[i] * b[i]; }
+ return den > 0 ? sqrt(num / den) : sqrt(num);
+}
+
+/* One layer step as an engine runs it (vk_tier.h step 3): issue, CPU pairs with a
+ * note each, join, rank-order sum. Returns the worst relative error of a device row
+ * against the reference and of the sum against the all-CPU sum. */
+typedef struct { unsigned long long routed, dev; } Books;
+static double step(int layer, int S, const int *idx, const float *w, Books *bk) {
+ float *x = calloc((size_t)S * H, sizeof(float)), *cpu = calloc((size_t)S * K * H, sizeof(float));
+ float *out = calloc((size_t)S * H, sizeof(float)), *ref = calloc((size_t)S * H, sizeof(float)), *y = malloc(sizeof(float) * H);
+ uint8_t *taken = malloc((size_t)S * K); const float **dev = malloc(sizeof(*dev) * S * K);
+ for (int i = 0; i < S * H; i++) x[i] = frnd();
+ int n = vkt_issue(layer, x, S, K, idx, taken);
+ for (int i = 0; i < S * K; i++) {
+ if (taken[i]) continue;
+ expert_ref(&ex[layer][idx[i]], x + (size_t)(i / K) * H, cpu + (size_t)i * H);
+ VktExpertSrc s = src_of(&ex[layer][idx[i]]); vkt_note(layer, idx[i], &s);
+ }
+ int joined = n ? vkt_join(dev) : 1;
+ CHECK(joined, "join failed");
+ double worst = 0;
+ for (int s = 0; s < S; s++)
+ for (int k = 0; k < K; k++) {
+ int i = s * K + k;
+ expert_ref(&ex[layer][idx[i]], x + (size_t)s * H, y);
+ for (int d = 0; d < H; d++) ref[(size_t)s * H + d] += w[i] * y[d];
+ const float *c = taken[i] && joined ? dev[i] : cpu + (size_t)i * H;
+ if (taken[i] && joined) { double r = rel(c, y, H); if (r > worst) worst = r; }
+ for (int d = 0; d < H; d++) out[(size_t)s * H + d] += w[i] * c[d];
+ bk->routed++; bk->dev += taken[i] != 0;
+ }
+ double r = rel(out, ref, S * H);
+ if (r > worst) worst = r;
+ free(x); free(cpu); free(out); free(ref); free(y); free(taken); free(dev);
+ return worst;
+}
+/* K distinct experts per row from a skewed draw over [lo, lo+span) */
+static void route(int S, int lo, int span, int *idx, float *w) {
+ for (int s = 0; s < S; s++)
+ for (int k = 0; k < K; k++) {
+ int e, dup;
+ do { unsigned r = rnd() % 100; e = lo + (r < 60 ? (int)(r % 2) : (int)(r % span)) % span; e %= E;
+ dup = 0; for (int j = 0; j < k; j++) dup |= idx[s * K + j] == e; } while (dup);
+ idx[s * K + k] = e; w[s * K + k] = 0.2f + (rnd() % 100) / 200.0f;
+ }
+}
+
+static VktConfig cfg_of(VktFmt gu, VktFmt dn, int act, float limit) {
+ return (VktConfig){.engine = "test", .layers = L, .experts = E, .hidden = H, .inter = F, .topk = K,
+ .gate_up = gu, .down = dn, .act = act, .act_limit = limit, .act_a = 4.f, .act_b = 25.f,
+ .max_rows = 1 << 20};
+}
+static void set_budget(int experts, VktFmt gu, VktFmt dn) {
+ char b[64]; snprintf(b, sizeof b, "%.12f", (experts + 0.5) * vkt_expert_bytes(H, F, gu, dn) / 1073741824.0);
+ setenv("COLI_VK_TIER_GB", b, 1);
+}
+
+/* every format: the whole model fits, experts are promoted as they pass by */
+static void formats(void) {
+ struct { VktFmt gu, dn; int act; float limit; const char *name; } cs[] = {
+ {{VKT_SRC_I8_ROW, 0}, {VKT_SRC_I8_ROW, 0}, VKT_ACT_SWIGLU, 0, "int8 per row"},
+ {{VKT_SRC_I8_GS, 32}, {VKT_SRC_I8_GS, 32}, VKT_ACT_SWIGLU, 0, "int8 gs32"},
+ {{VKT_SRC_I8_AS_I4_ROW, 0}, {VKT_SRC_I8_AS_I4_ROW, 0}, VKT_ACT_SWIGLU, 0, "unpacked int4 per row"},
+ {{VKT_SRC_I8_AS_I4_GS, 64}, {VKT_SRC_I8_GS, 32}, VKT_ACT_SWIGLU, 0, "unpacked int4 gs64, int8 down"},
+ {{VKT_SRC_I4S_PAIRS_ROW, 0}, {VKT_SRC_I4S_PAIRS_ROW, 0}, VKT_ACT_SWIGLU, 0, "signed int4 pairs per row"},
+ {{VKT_SRC_I4S_PAIRS_GS, 32}, {VKT_SRC_I4S_PAIRS_GS, 32}, VKT_ACT_SWIGLU, 0, "signed int4 pairs gs32"},
+ {{VKT_SRC_I4U_PAIRS_ROW, 0}, {VKT_SRC_I4U_PAIRS_ROW, 0}, VKT_ACT_SWIGLU, 0, "int4 v+8 per row"},
+ {{VKT_SRC_I4U_PAIRS_GS, 64}, {VKT_SRC_I4U_PAIRS_GS, 64}, VKT_ACT_SWIGLU, 2.5f, "int4 v+8 gs64, SwiGLU limit"},
+ {{VKT_SRC_I4U_PLANAR64, 64}, {VKT_SRC_I4U_PLANAR64, 64}, VKT_ACT_SWIGLU, 0, "planar int4-g64"},
+ {{VKT_SRC_I3_G64, 0}, {VKT_SRC_I3_G64, 0}, VKT_ACT_SWIGLU, 0, "int3-g64"},
+ {{VKT_SRC_MXFP4_F32, 32}, {VKT_SRC_MXFP4_F32, 32}, VKT_ACT_SWIGLU, 0, "MXFP4 f32 scales"},
+ {{VKT_SRC_MXFP4_E8M0, 32}, {VKT_SRC_MXFP4_E8M0, 32}, VKT_ACT_SITU, 0, "MXFP4 ue8m0, SiTU-GLU"},
+ {{VKT_SRC_FP8_GS, 32}, {VKT_SRC_FP8_GS, 32}, VKT_ACT_SWIGLU, 0, "fp8 gs32"},
+ {{VKT_SRC_FP8_BLOCK, 128}, {VKT_SRC_FP8_BLOCK, 128}, VKT_ACT_SWIGLU, 0, "fp8 128x128 blocks"},
+ {{VKT_SRC_BF16, 0}, {VKT_SRC_BF16, 0}, VKT_ACT_SWIGLU, 0, "bf16"},
+ {{VKT_SRC_F32, 0}, {VKT_SRC_F32, 0}, VKT_ACT_SWIGLU, 0, "f32"},
+ };
+ for (size_t c = 0; c < sizeof cs / sizeof *cs; c++) {
+ model_make(cs[c].gu, cs[c].dn);
+ g_act = cs[c].act; g_limit = cs[c].limit;
+ set_budget(L * E, cs[c].gu, cs[c].dn);
+ setenv("COLI_VK_TIER_RATE", "64", 1);
+ VktConfig vc = cfg_of(cs[c].gu, cs[c].dn, cs[c].act, cs[c].limit);
+ int on = vkt_init(&vc, NULL);
+ CHECK(on, "%s: the tier did not start", cs[c].name);
+ if (!on) { model_free(); continue; }
+ Books bk = {0, 0}; double worst = 0;
+ static const int Ss[] = {1, 3, 20, 1, 2, 24};
+ for (int t = 0; t < 6; t++)
+ for (int l = 0; l < L; l++) {
+ int idx[24 * K]; float w[24 * K];
+ route(Ss[t], 0, E, idx, w);
+ double r = step(l, Ss[t], idx, w, &bk);
+ if (r > worst) worst = r;
+ }
+ printf(" %-34s device %3llu of %3llu assignments, worst relative error %.2e\n", cs[c].name, bk.dev, bk.routed, worst);
+ CHECK(bk.dev > 0, "%s: nothing ran on the device", cs[c].name);
+ CHECK(worst < 2e-3, "%s: device rows off the reference (%.3g)", cs[c].name, worst);
+ vkt_shutdown(); model_free();
+ }
+}
+
+/* a history fills the budget in heat order */
+static void warm(void) {
+ VktFmt f = {VKT_SRC_I4U_PAIRS_GS, 64};
+ model_make(f, f); g_act = VKT_ACT_SWIGLU; g_limit = 0;
+ set_budget(7, f, f);
+ uint32_t hist[L][E], *rows[L];
+ for (int l = 0; l < L; l++) { rows[l] = hist[l]; for (int e = 0; e < E; e++) hist[l][e] = (uint32_t)(1000 - 37 * (l * E + e) % 700); }
+ VktConfig vc = cfg_of(f, f, VKT_ACT_SWIGLU, 0);
+ CHECK(vkt_init(&vc, rows), "warm: the tier did not start");
+ int pl[L * E], pe[L * E];
+ int n = vkt_plan(pl, pe, L * E);
+ CHECK(n == 7, "warm: planned %d experts for a budget of 7", n);
+ /* the plan is the 7 hottest, hottest first */
+ int ok = 1;
+ for (int i = 1; i < n; i++) ok &= hist[pl[i - 1]][pe[i - 1]] >= hist[pl[i]][pe[i]];
+ for (int l = 0; l < L; l++) for (int e = 0; e < E; e++) {
+ int planned = 0; for (int i = 0; i < n; i++) planned |= pl[i] == l && pe[i] == e;
+ if (!planned) ok &= hist[l][e] <= hist[pl[n - 1]][pe[n - 1]];
+ }
+ CHECK(ok, "warm: the plan is not the hottest experts in heat order");
+ for (int i = 0; i < n; i++) { VktExpertSrc s = src_of(&ex[pl[i]][pe[i]]); vkt_put(pl[i], pe[i], &s); }
+ vkt_put_done();
+ int res = 0; for (int i = 0; i < n; i++) res += vkt_resident(pl[i], pe[i]);
+ CHECK(res == n, "warm: %d of %d planned experts resident", res, n);
+ Books bk = {0, 0};
+ int idx[K] = {pe[0], (pe[0] + 1) % E, (pe[0] + 2) % E}; float w[K] = {0.5f, 0.3f, 0.2f};
+ double r = step(pl[0], 1, idx, w, &bk);
+ CHECK(bk.dev >= 1 && r < 2e-3, "warm: the hottest expert is not served from the device (%llu, %.3g)", bk.dev, r);
+ printf(" warm start: %d experts planned in heat order and resident, the hottest served by the device\n", n);
+ vkt_shutdown(); model_free();
+}
+
+/* a third of the experts fit; the hot set moves halfway */
+static void adapt(void) {
+ VktFmt f = {VKT_SRC_FP8_GS, 32};
+ model_make(f, f); g_act = VKT_ACT_SWIGLU; g_limit = 0;
+ set_budget(L * E / 3, f, f);
+ setenv("COLI_VK_TIER_RATE", "4", 1);
+ VktConfig vc = cfg_of(f, f, VKT_ACT_SWIGLU, 0);
+ CHECK(vkt_init(&vc, NULL), "adapt: the tier did not start");
+ Books a = {0, 0}, b1 = {0, 0}, b2 = {0, 0}; double worst = 0;
+ int idx[4 * K]; float w[4 * K];
+ /* LFRU: frequency first, halved every 1024 tokens, so an established hot set
+ * yields over a few hundred steps of a new one, not at its first appearance */
+ for (int t = 0; t < 100; t++) /* phase A: experts 0..4 */
+ for (int l = 0; l < L; l++) { route(4, 0, 5, idx, w); double r = step(l, 4, idx, w, &a); if (r > worst) worst = r; }
+ int resA = 0; for (int l = 0; l < L; l++) for (int e = 0; e < 5; e++) resA += vkt_resident(l, e);
+ for (int t = 0; t < 400; t++) /* phase B: experts 5..9 */
+ for (int l = 0; l < L; l++) { route(4, 5, 5, idx, w); double r = step(l, 4, idx, w, t < 200 ? &b1 : &b2); if (r > worst) worst = r; }
+ int resB = 0, resA2 = 0, res = 0;
+ for (int l = 0; l < L; l++) for (int e = 0; e < E; e++) { int v = vkt_resident(l, e); res += v; if (e >= 5) resB += v; else resA2 += v; }
+ printf(" adapt: phase A device %.0f%%, resident A %d; phase B device %.0f%% then %.0f%%, resident B %d, A %d, all %d (budget %d)\n",
+ 100.0 * a.dev / a.routed, resA, 100.0 * b1.dev / b1.routed, 100.0 * b2.dev / b2.routed, resB, resA2, res, L * E / 3);
+ vkt_report("test", 0, 0);
+ CHECK(resA > 0 && a.dev > 0, "adapt: phase A promoted nothing");
+ CHECK(resB > resA2, "adapt: the new hot set did not take the device (B %d, A %d)", resB, resA2);
+ CHECK(res <= L * E / 3, "adapt: %d resident over a budget of %d", res, L * E / 3);
+ CHECK(b2.dev * b1.routed > b1.dev * b2.routed, "adapt: the device share did not rise in phase B");
+ CHECK(worst < 2e-3, "adapt: relative error %.3g", worst);
+ vkt_shutdown(); model_free();
+}
+
+/* more assignments than max_rows: the rest stays on the CPU */
+static void partial(void) {
+ VktFmt f = {VKT_SRC_I8_ROW, 0};
+ model_make(f, f); g_act = VKT_ACT_SWIGLU; g_limit = 0;
+ set_budget(L * E, f, f);
+ VktConfig vc = cfg_of(f, f, VKT_ACT_SWIGLU, 0); vc.max_rows = 5;
+ setenv("COLI_VK_TIER_RATE", "64", 1); /* every expert resident after a few steps */
+ CHECK(vkt_init(&vc, NULL), "partial: the tier did not start");
+ Books bk = {0, 0}; int idx[8 * K]; float w[8 * K];
+ for (int t = 0; t < 8; t++) for (int l = 0; l < L; l++) { route(8, 0, E, idx, w); step(l, 8, idx, w, &bk); }
+ Books one = {0, 0};
+ route(8, 0, E, idx, w);
+ double r = step(0, 8, idx, w, &one);
+ printf(" partial: a step of %llu assignments, the device took %llu (max_rows 5), relative error %.2e\n", one.routed, one.dev, r);
+ CHECK(one.dev > 0 && one.dev <= 5, "partial: the device took %llu of a 5-row cap", one.dev);
+ CHECK(r < 2e-3, "partial: relative error %.3g", r);
+ vkt_shutdown(); model_free();
+}
+
+int main(int argc, char **argv) {
+ char buf[1024];
+ const char *spv = argc > 1 ? argv[1] : coli_vk_shader_path(buf, sizeof buf);
+ if (!coli_vk_init(spv)) { printf("FAIL: no Vulkan device (shaders %s)\n", spv); return 1; }
+ setenv("COLI_VK_TIER_RESERVE_GB", "0", 1);
+ printf("formats:\n"); formats();
+ printf("warm:\n"); warm();
+ printf("adapt:\n"); adapt();
+ printf("partial:\n"); partial();
+ ColiVkPoolStats ps; coli_vk_pool_stats(1, &ps);
+ CHECK(ps.live == 0, "%d tier ranges still live after every shutdown", ps.live);
+ printf(fails ? "FAIL (%d)\n" : "PASS\n", fails);
+ return fails != 0;
+}
diff --git a/c/vk_tier.c b/c/vk_tier.c
new file mode 100644
index 00000000..9c269126
--- /dev/null
+++ b/c/vk_tier.c
@@ -0,0 +1,714 @@
+/* vk_tier.c -- the Vulkan routed-expert tier shared by the MoE engines. See
+ * vk_tier.h for what it does and how an engine calls it.
+ *
+ * Threads. Everything runs on the engine thread except two things: the uploader
+ * thread, which turns staged copies of promoted experts into device tensors, and
+ * vkt_put, which a warm start may call from many loader threads. Neither touches
+ * a slot: they hand finished tensors back through the `done` list, and the engine
+ * thread makes them resident at a quiescent point (no batch in flight), where it
+ * also frees evicted experts. So the slot table, the counters and every Vulkan
+ * call but the pool's allocation are single-threaded; the lock guards the queue,
+ * the done list and the room-made signal only. */
+#ifdef COLI_VULKAN
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+#include <time.h>
+#include <errno.h>
+#include "vk_tier.h"
+#include "backend_vulkan.h"
+#include "tier.h"
+
+#define VKT_QCAP 32 /* promotions staged and waiting for the uploader */
+#define VKT_CAND 64 /* victim candidates kept per refresh */
+#define VKT_DECAY_TOKENS 1024u /* tier.h's decay period, in tokens */
+
+enum { VS_NONE = 0, VS_QUEUED, VS_RESIDENT, VS_EVICT };
+
+typedef struct {
+ ColiVkExpert *ex;
+ uint32_t heat, last;
+ uint8_t state;
+} VSlot;
+typedef struct { int layer, eid; uint8_t *buf; } VQ;
+typedef struct { int layer, eid, ok; ColiVkTensor *g, *u, *d; } VDone;
+
+static struct {
+ int on;
+ VktConfig c;
+ char engine[32];
+ int gu_fmt, gu_gs, dn_fmt, dn_gs;
+ size_t gu_codes, gu_scales, d_codes, d_scales, stage_bytes; /* one expert as the engine holds it */
+ size_t exp_bytes, budget;
+ int max_resident, resident, queued, rate, uma;
+ VSlot *s;
+ uint32_t tick, decay_at;
+ int last_layer, promos;
+ /* uploader */
+ pthread_t th; int th_on, stop;
+ pthread_mutex_t mx; pthread_cond_t cv, cv_room;
+ VQ q[VKT_QCAP]; int qh, qn;
+ VDone *done; int ndone, cdone;
+ unsigned long room_gen;
+ /* evicted, freed at the next quiescent point */
+ int *evict; int nevict, cevict;
+ /* the coldest residents, refreshed once per forward */
+ int cand[VKT_CAND]; uint64_t cand_score[VKT_CAND]; int ncand;
+ /* the step in flight */
+ int inflight, S, K;
+ int *map; int cmap;
+ int *grp; int *touched; /* expert -> group of this step, and the groups' experts */
+ ColiVkExpert **bex; int *brows; int *bfirst; int cb;
+ const float **bx, **by; int *rowsrc; int cbx;
+ int max_dev_rows;
+ double t_issued;
+ /* accounting */
+ unsigned long long routed, served, steps, uploads, upload_bytes, evictions, qfull, rated, refused, failed, warm;
+ double dev_ms, cpu_ms, wait_ms;
+ unsigned long long rep_routed, rep_served;
+} T;
+
+/* "2.31 GiB", "35.0 MiB", "9.5 KiB": the budget lines read on a 24 GB card and a tiny fixture alike */
+static const char *human(double b, char *buf, size_t n) {
+ if (b >= 1073741824.0) snprintf(buf, n, "%.2f GiB", b / 1073741824.0);
+ else if (b >= 1048576.0) snprintf(buf, n, "%.1f MiB", b / 1048576.0);
+ else snprintf(buf, n, "%.1f KiB", b / 1024.0);
+ return buf;
+}
+static double vkt_now_ms(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec * 1e3 + t.tv_nsec / 1e6; }
+static VSlot *slot(int layer, int eid) { return &T.s[(size_t)layer * T.c.experts + eid]; }
+
+/* ---- source formats ---------------------------------------------------------- */
+static int src_per_row(VktSrc k) {
+ return k == VKT_SRC_I8_ROW || k == VKT_SRC_I8_AS_I4_ROW || k == VKT_SRC_I4S_PAIRS_ROW || k == VKT_SRC_I4U_PAIRS_ROW;
+}
+static size_t src_row_bytes(VktSrc k, int I) {
+ switch (k) {
+ case VKT_SRC_I8_ROW: case VKT_SRC_I8_GS: case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I8_AS_I4_GS:
+ case VKT_SRC_FP8_GS: case VKT_SRC_FP8_BLOCK: return (size_t)I;
+ case VKT_SRC_I3_G64: return ((size_t)I + 63) / 64 * 24;
+ case VKT_SRC_BF16: return (size_t)I * 2;
+ case VKT_SRC_F32: return (size_t)I * 4;
+ default: return ((size_t)I + 1) / 2; /* the int4 and fp4 kinds */
+ }
+}
+static size_t src_scale_bytes(VktFmt f, int I, int O) {
+ if (f.kind == VKT_SRC_BF16 || f.kind == VKT_SRC_F32) return 0;
+ if (src_per_row(f.kind)) return (size_t)O * 4;
+ if (f.kind == VKT_SRC_I4U_PLANAR64 || f.kind == VKT_SRC_I3_G64) return (size_t)O * (((size_t)I + 63) / 64) * 4;
+ size_t ng = ((size_t)I + f.gs - 1) / f.gs;
+ if (f.kind == VKT_SRC_MXFP4_E8M0) return (size_t)O * ng;
+ if (f.kind == VKT_SRC_FP8_BLOCK) return (((size_t)O + f.gs - 1) / f.gs) * ng * 4;
+ return (size_t)O * ng * 4;
+}
+/* The backend format a source kind lands in; 0 when the combination is not one. */
+static int dev_fmt(VktFmt f, int *fmt, int *gs) {
+ int g = f.gs;
+ switch (f.kind) {
+ case VKT_SRC_I8_ROW: *fmt = 1; *gs = 0; return 1;
+ case VKT_SRC_I8_GS: *fmt = 13; *gs = g; return g >= 4 && g % 4 == 0;
+ case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I4S_PAIRS_ROW: case VKT_SRC_I4U_PAIRS_ROW: *fmt = 2; *gs = 0; return 1;
+ case VKT_SRC_I8_AS_I4_GS: case VKT_SRC_I4S_PAIRS_GS: case VKT_SRC_I4U_PAIRS_GS: *fmt = 4; *gs = g; return g >= 8 && g % 8 == 0;
+ case VKT_SRC_I4U_PLANAR64: *fmt = 4; *gs = 64; return 1;
+ case VKT_SRC_I3_G64: *fmt = 5; *gs = 0; return 1;
+ case VKT_SRC_MXFP4_F32: case VKT_SRC_MXFP4_E8M0: *fmt = 7; *gs = g; return g >= 8 && g % 8 == 0;
+ case VKT_SRC_FP8_GS: case VKT_SRC_FP8_BLOCK: *fmt = 12; *gs = g; return g >= 4 && g % 4 == 0;
+ case VKT_SRC_BF16: *fmt = 11; *gs = 0; return 1;
+ case VKT_SRC_F32: *fmt = 10; *gs = 0; return 1;
+ default: return 0;
+ }
+}
+
+/* One matrix [O x I] from its RAM form into a device tensor's mapping: rows at
+ * `stride` bytes (already zeroed past the row), scales as the backend reads them.
+ * Pure byte work; the values are never requantized. */
+static void convert(VktFmt f, int I, int O, const uint8_t *codes, const void *scales,
+ uint8_t *rows, size_t stride, float *sc) {
+ size_t rb = src_row_bytes(f.kind, I);
+ switch (f.kind) {
+ case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I8_AS_I4_GS: /* int8 in [-8,7] -> v+8 nibble pairs */
+ for (int o = 0; o < O; o++) {
+ const int8_t *src = (const int8_t *)codes + (size_t)o * I;
+ uint8_t *dst = rows + (size_t)o * stride;
+ for (int j = 0; j < I / 2; j++)
+ dst[j] = (uint8_t)(((src[2 * j] + 8) & 15) | (((src[2 * j + 1] + 8) & 15) << 4));
+ if (I & 1) dst[I / 2] = (uint8_t)((src[I - 1] + 8) & 15);
+ }
+ break;
+ case VKT_SRC_I4S_PAIRS_ROW: case VKT_SRC_I4S_PAIRS_GS: /* two's complement nibble -> v+8: xor 8 */
+ for (int o = 0; o < O; o++) {
+ const uint8_t *src = codes + (size_t)o * rb;
+ uint8_t *dst = rows + (size_t)o * stride;
+ for (size_t j = 0; j < rb; j++) dst[j] = src[j] ^ 0x88u;
+ }
+ break;
+ case VKT_SRC_I4U_PLANAR64: /* planar blocks (byte k = elements k, k+32) -> pairs (2j, 2j+1) */
+ for (int o = 0; o < O; o++) {
+ const uint8_t *src = codes + (size_t)o * rb;
+ uint8_t *dst = rows + (size_t)o * stride;
+ int nb = I / 64;
+ for (int b = 0; b < nb; b++) {
+ const uint8_t *blk = src + (size_t)b * 32;
+ for (int p = 0; p < 32; p++) {
+ int e0 = 2 * p, e1 = 2 * p + 1;
+ unsigned a = e0 < 32 ? (blk[e0] & 15u) : (unsigned)(blk[e0 - 32] >> 4);
+ unsigned c = e1 < 32 ? (blk[e1] & 15u) : (unsigned)(blk[e1 - 32] >> 4);
+ dst[(size_t)b * 32 + p] = (uint8_t)(a | (c << 4));
+ }
+ }
+ memcpy(dst + (size_t)nb * 32, src + (size_t)nb * 32, rb - (size_t)nb * 32); /* tail: pairs already */
+ }
+ break;
+ default: /* the same bytes */
+ for (int o = 0; o < O; o++) memcpy(rows + (size_t)o * stride, codes + (size_t)o * rb, rb);
+ break;
+ }
+ if (f.kind == VKT_SRC_BF16 || f.kind == VKT_SRC_F32) { sc[0] = 1.0f; return; }
+ if (f.kind == VKT_SRC_MXFP4_E8M0) { /* ue8m0 -> f32 2^(s-127), quant.h's mx4_scale */
+ size_t n = (size_t)O * (((size_t)I + f.gs - 1) / f.gs);
+ const uint8_t *e = scales;
+ for (size_t i = 0; i < n; i++) { uint32_t u = (uint32_t)e[i] << 23; memcpy(&sc[i], &u, 4); }
+ return;
+ }
+ if (f.kind == VKT_SRC_FP8_BLOCK) { /* [O/bs][I/bs] blocks -> one row each */
+ size_t nb = ((size_t)I + f.gs - 1) / f.gs;
+ const float *b = scales;
+ for (int o = 0; o < O; o++) memcpy(sc + (size_t)o * nb, b + (size_t)(o / f.gs) * nb, nb * 4);
+ return;
+ }
+ memcpy(sc, scales, src_scale_bytes(f, I, O));
+}
+
+/* Upload one expert from its RAM form (any thread). 0 when the pool refused. */
+static int upload(const uint8_t *g, const uint8_t *u, const uint8_t *d,
+ const void *gs, const void *us, const void *ds, ColiVkTensor *t[3]) {
+ const int H = T.c.hidden, F = T.c.inter;
+ const uint8_t *codes[3] = {g, u, d}; const void *sc[3] = {gs, us, ds};
+ t[0] = t[1] = t[2] = NULL;
+ for (int k = 0; k < 3; k++) {
+ VktFmt f = k < 2 ? T.c.gate_up : T.c.down;
+ int fmt = k < 2 ? T.gu_fmt : T.dn_fmt, fgs = k < 2 ? T.gu_gs : T.dn_gs;
+ int I = k < 2 ? H : F, O = k < 2 ? F : H;
+ uint8_t *rows; size_t stride; float *scales;
+ if (!coli_vk_tier_tensor(&t[k], fmt, I, O, fgs, &rows, &stride, &scales)) {
+ for (int j = 0; j < k; j++) { coli_vk_tensor_free(t[j]); t[j] = NULL; }
+ return 0;
+ }
+ convert(f, I, O, codes[k], sc[k], rows, stride, scales);
+ }
+ return 1;
+}
+
+size_t vkt_expert_bytes(int H, int F, VktFmt gu, VktFmt dn) {
+ int gf, gg, df, dg;
+ if (!dev_fmt(gu, &gf, &gg) || !dev_fmt(dn, &df, &dg)) return 0;
+ size_t b = 0;
+ for (int k = 0; k < 3; k++) {
+ int fmt = k < 2 ? gf : df, gs = k < 2 ? gg : dg, I = k < 2 ? H : F, O = k < 2 ? F : H;
+ size_t rows = (coli_vk_tensor_row_bytes(fmt, I) + 3) / 4 * 4 * (size_t)O;
+ size_t sc = coli_vk_tensor_scale_count(fmt, I, O, gs) * 4;
+ b += (rows + 255) / 256 * 256 + (sc + 255) / 256 * 256 + 512; /* two ranges, aligned, with slack */
+ }
+ return b;
+}
+
+/* ---- the uploader -------------------------------------------------------------- */
+static void push_done(int layer, int eid, int ok, ColiVkTensor *t[3]) {
+ pthread_mutex_lock(&T.mx);
+ if (T.ndone == T.cdone) {
+ int nc = T.cdone ? 2 * T.cdone : 64;
+ VDone *n = realloc(T.done, (size_t)nc * sizeof(*n));
+ if (!n) { /* cannot even record it: the slot stays queued and is never used */
+ pthread_mutex_unlock(&T.mx);
+ for (int k = 0; k < 3; k++) if (t[k]) coli_vk_tensor_free(t[k]);
+ return;
+ }
+ T.done = n; T.cdone = nc;
+ }
+ T.done[T.ndone++] = (VDone){layer, eid, ok, t[0], t[1], t[2]};
+ pthread_mutex_unlock(&T.mx);
+}
+
+static void *uploader(void *arg) {
+ (void)arg;
+ for (;;) {
+ pthread_mutex_lock(&T.mx);
+ while (!T.qn && !T.stop) pthread_cond_wait(&T.cv, &T.mx);
+ if (T.stop) { pthread_mutex_unlock(&T.mx); return NULL; }
+ VQ e = T.q[T.qh]; T.qh = (T.qh + 1) % VKT_QCAP; T.qn--;
+ pthread_mutex_unlock(&T.mx);
+ const uint8_t *b = e.buf;
+ const uint8_t *g = b, *u = g + T.gu_codes, *d = u + T.gu_codes;
+ const uint8_t *gs = d + T.d_codes, *us = gs + T.gu_scales, *ds = us + T.gu_scales;
+ ColiVkTensor *t[3];
+ int ok = 0;
+ /* Refused: the pool is at its budget until the engine thread frees the victim
+ * this promotion displaced, at its next join. Wait for that, a bounded number
+ * of times: a pool that stays full is full for real (fragmentation). */
+ for (int tries = 0; tries < 20 && !ok; tries++) {
+ ok = upload(g, u, d, gs, us, ds, t);
+ if (ok) break;
+ pthread_mutex_lock(&T.mx);
+ unsigned long gen = T.room_gen;
+ struct timespec until; clock_gettime(CLOCK_REALTIME, &until);
+ until.tv_nsec += 50 * 1000000L; if (until.tv_nsec >= 1000000000L) { until.tv_sec++; until.tv_nsec -= 1000000000L; }
+ while (T.room_gen == gen && !T.stop && pthread_cond_timedwait(&T.cv_room, &T.mx, &until) != ETIMEDOUT) {}
+ int stop = T.stop;
+ pthread_mutex_unlock(&T.mx);
+ if (stop) break;
+ }
+ free(e.buf);
+ if (!ok) t[0] = t[1] = t[2] = NULL;
+ push_done(e.layer, e.eid, ok, t);
+ }
+}
+
+/* ---- quiescent points: nothing in flight ---------------------------------------- */
+static void quiesce(void) {
+ if (T.inflight) return;
+ int freed = 0;
+ for (int i = 0; i < T.nevict; i++) {
+ VSlot *v = &T.s[T.evict[i]];
+ if (v->ex) { coli_vk_xb_expert_free(v->ex); v->ex = NULL; freed = 1; }
+ v->state = VS_NONE;
+ }
+ T.nevict = 0;
+ if (freed) {
+ pthread_mutex_lock(&T.mx); T.room_gen++; pthread_cond_broadcast(&T.cv_room); pthread_mutex_unlock(&T.mx);
+ }
+ /* finished uploads become resident */
+ pthread_mutex_lock(&T.mx);
+ int n = T.ndone; VDone *d = n ? malloc((size_t)n * sizeof(*d)) : NULL;
+ if (d) { memcpy(d, T.done, (size_t)n * sizeof(*d)); T.ndone = 0; } else n = 0;
+ pthread_mutex_unlock(&T.mx);
+ for (int i = 0; i < n; i++) {
+ VSlot *v = slot(d[i].layer, d[i].eid);
+ T.queued--;
+ ColiVkExpert *e = d[i].ok ? coli_vk_xb_expert(d[i].g, d[i].u, d[i].d) : NULL;
+ if (e) {
+ v->ex = e; v->state = VS_RESIDENT; T.resident++;
+ T.uploads++; T.upload_bytes += coli_vk_tensor_bytes(d[i].g) + coli_vk_tensor_bytes(d[i].u) + coli_vk_tensor_bytes(d[i].d);
+ } else {
+ for (int k = 0; k < 3; k++) { ColiVkTensor *t = k == 0 ? d[i].g : k == 1 ? d[i].u : d[i].d; if (t) coli_vk_tensor_free(t); }
+ v->state = VS_NONE; T.failed++;
+ /* the budget holds fewer than the count said (fragmentation, or the device
+ * ran out first): stop planning past what is there */
+ if (T.max_resident > T.resident + T.queued) T.max_resident = T.resident + T.queued;
+ }
+ }
+ free(d);
+}
+
+/* ---- placement ------------------------------------------------------------------ */
+static uint64_t score(const VSlot *v) { return tier_lfru_score(v->heat, v->last, T.tick); }
+
+/* The coldest residents, ascending, once per forward. */
+static void refresh_candidates(void) {
+ T.ncand = 0;
+ size_t n = (size_t)T.c.layers * T.c.experts;
+ for (size_t i = 0; i < n; i++) {
+ VSlot *v = &T.s[i];
+ if (v->state != VS_RESIDENT) continue;
+ uint64_t sc = score(v);
+ if (T.ncand == VKT_CAND && sc >= T.cand_score[VKT_CAND - 1]) continue;
+ int at = T.ncand < VKT_CAND ? T.ncand++ : VKT_CAND - 1;
+ while (at > 0 && T.cand_score[at - 1] > sc) { T.cand[at] = T.cand[at - 1]; T.cand_score[at] = T.cand_score[at - 1]; at--; }
+ T.cand[at] = (int)i; T.cand_score[at] = sc;
+ }
+}
+/* The resident a newcomer of score hs displaces, or -1: the coldest candidate still
+ * resident, when the newcomer beats it by tier.h's LFRU margin (25% + 4 counts). */
+static int pick_victim(uint64_t hs) {
+ while (T.ncand) {
+ int i = T.cand[0];
+ VSlot *v = &T.s[i];
+ if (v->state != VS_RESIDENT) { memmove(T.cand, T.cand + 1, (size_t)--T.ncand * sizeof(int));
+ memmove(T.cand_score, T.cand_score + 1, (size_t)T.ncand * sizeof(uint64_t)); continue; }
+ uint64_t cs = score(v);
+ if (hs <= cs + (cs >> 2) + (4u << 8)) return -1;
+ memmove(T.cand, T.cand + 1, (size_t)--T.ncand * sizeof(int));
+ memmove(T.cand_score, T.cand_score + 1, (size_t)T.ncand * sizeof(uint64_t));
+ return i;
+ }
+ return -1;
+}
+
+static void new_forward(int S) {
+ T.tick += (uint32_t)(S > 0 ? S : 1);
+ T.promos = 0;
+ if (T.tick >= T.decay_at) {
+ size_t n = (size_t)T.c.layers * T.c.experts;
+ for (size_t i = 0; i < n; i++) T.s[i].heat = tier_decay_value(T.s[i].heat);
+ T.decay_at = T.tick + VKT_DECAY_TOKENS;
+ }
+ refresh_candidates();
+}
+
+/* The bytes of an expert the CPU just computed pass by: promote it when there is
+ * room, or when it is hot enough to displace the coldest resident. The copy is the
+ * only cost on this thread; the uploader does the rest. */
+void vkt_note(int layer, int eid, const VktExpertSrc *src) {
+ if (!T.on || !src || layer < 0 || layer >= T.c.layers || eid < 0 || eid >= T.c.experts) return;
+ VSlot *v = slot(layer, eid);
+ if (v->state != VS_NONE || !src->g || !src->u || !src->d) return;
+ if (T.promos >= T.rate) { T.rated++; return; }
+ int victim = -1;
+ if (T.resident + T.queued >= T.max_resident) {
+ if (!v->heat) return;
+ victim = pick_victim(score(v));
+ if (victim < 0) return;
+ }
+ if (T.qn >= VKT_QCAP) { T.qfull++; return; }
+ if (T.gu_scales && (!src->gs || !src->us || !src->ds)) return;
+ uint8_t *buf = malloc(T.stage_bytes);
+ if (!buf) return;
+ uint8_t *p = buf;
+ memcpy(p, src->g, T.gu_codes); p += T.gu_codes;
+ memcpy(p, src->u, T.gu_codes); p += T.gu_codes;
+ memcpy(p, src->d, T.d_codes); p += T.d_codes;
+ if (T.gu_scales) { memcpy(p, src->gs, T.gu_scales); p += T.gu_scales; memcpy(p, src->us, T.gu_scales); p += T.gu_scales; }
+ if (T.d_scales) memcpy(p, src->ds, T.d_scales);
+ if (victim >= 0) {
+ VSlot *w = &T.s[victim];
+ w->state = VS_EVICT; T.resident--; T.evictions++;
+ if (T.nevict == T.cevict) {
+ int nc = T.cevict ? 2 * T.cevict : 64;
+ int *n = realloc(T.evict, (size_t)nc * sizeof(int));
+ if (!n) { w->state = VS_RESIDENT; T.resident++; T.evictions--; free(buf); return; }
+ T.evict = n; T.cevict = nc;
+ }
+ T.evict[T.nevict++] = victim;
+ quiesce(); /* nothing in flight: free it right away */
+ }
+ v->state = VS_QUEUED; T.queued++; T.promos++;
+ pthread_mutex_lock(&T.mx);
+ T.q[(T.qh + T.qn) % VKT_QCAP] = (VQ){layer, eid, buf}; T.qn++;
+ pthread_cond_signal(&T.cv);
+ pthread_mutex_unlock(&T.mx);
+}
+
+/* ---- warm start ------------------------------------------------------------------ */
+static const uint32_t *const *g_plan_heat;
+static int g_plan_E;
+static int plan_cmp(const void *a, const void *b) {
+ int x = *(const int *)a, y = *(const int *)b;
+ uint32_t hx = g_plan_heat[x / g_plan_E][x % g_plan_E], hy = g_plan_heat[y / g_plan_E][y % g_plan_E];
+ return hx < hy ? 1 : hx > hy ? -1 : (x > y) - (x < y);
+}
+static uint32_t *const *g_heat_hist;
+int vkt_plan(int *layers, int *eids, int max) {
+ if (!T.on || !g_heat_hist) return 0;
+ int L = T.c.layers, E = T.c.experts, n = 0;
+ int *order = malloc((size_t)L * E * sizeof(int));
+ if (!order) return 0;
+ for (int l = 0; l < L; l++)
+ if (g_heat_hist[l]) for (int e = 0; e < E; e++) if (g_heat_hist[l][e]) order[n++] = l * E + e;
+ g_plan_heat = (const uint32_t *const *)g_heat_hist; g_plan_E = E;
+ qsort(order, (size_t)n, sizeof(int), plan_cmp);
+ int k = 0;
+ for (int i = 0; i < n && k < max && T.resident + T.queued < T.max_resident; i++) {
+ VSlot *v = &T.s[order[i]];
+ if (v->state != VS_NONE) continue;
+ v->state = VS_QUEUED; T.queued++;
+ layers[k] = order[i] / E; eids[k] = order[i] % E; k++;
+ }
+ free(order);
+ return k;
+}
+int vkt_put(int layer, int eid, const VktExpertSrc *src) {
+ if (!T.on || layer < 0 || layer >= T.c.layers || eid < 0 || eid >= T.c.experts) return 0;
+ ColiVkTensor *t[3] = {NULL, NULL, NULL};
+ int ok = src && src->g && src->u && src->d &&
+ upload(src->g, src->u, src->d, src->gs, src->us, src->ds, t);
+ push_done(layer, eid, ok, t);
+ if (ok) __atomic_add_fetch(&T.warm, 1, __ATOMIC_RELAXED);
+ return ok;
+}
+void vkt_put_done(void) { if (T.on) quiesce(); }
+
+/* ---- one layer step -------------------------------------------------------------- */
+/* The per-step arrays: assignments (map), groups (bex, brows, bfirst, touched) and
+ * rows (bx, by), each family grown together. */
+static int grow_to(int need, int *cap) {
+ int nc = *cap ? *cap : 64;
+ while (nc < need) nc *= 2;
+ return nc;
+}
+static int grow_map(int need) {
+ if (need <= T.cmap) return 1;
+ int nc = grow_to(need, &T.cmap);
+ int *m = realloc(T.map, (size_t)nc * sizeof(int));
+ if (!m) return 0;
+ T.map = m; T.cmap = nc;
+ return 1;
+}
+static int grow_groups(int need) {
+ if (need <= T.cb) return 1;
+ int nc = grow_to(need, &T.cb);
+ ColiVkExpert **a = realloc(T.bex, (size_t)nc * sizeof(*a)); if (a) T.bex = a;
+ int *b = realloc(T.brows, (size_t)nc * sizeof(int)); if (b) T.brows = b;
+ int *c = realloc(T.bfirst, (size_t)nc * sizeof(int)); if (c) T.bfirst = c;
+ int *d = realloc(T.touched, (size_t)nc * sizeof(int)); if (d) T.touched = d;
+ if (!a || !b || !c || !d) return 0;
+ T.cb = nc;
+ return 1;
+}
+static int grow_rows(int need) {
+ if (need <= T.cbx) return 1;
+ int nc = grow_to(need, &T.cbx);
+ const float **a = realloc(T.bx, (size_t)nc * sizeof(*a)); if (a) T.bx = a;
+ const float **b = realloc(T.by, (size_t)nc * sizeof(*b)); if (b) T.by = b;
+ if (!a || !b) return 0;
+ T.cbx = nc;
+ return 1;
+}
+
+int vkt_issue(int layer, const float *x, int S, int K, const int *idx, uint8_t *taken) {
+ if (S > 0 && K > 0) memset(taken, 0, (size_t)S * K);
+ if (!T.on || T.inflight || layer < 0 || layer >= T.c.layers || S < 1 || K < 1) return 0;
+ if (!coli_vk_xb_ready()) {
+ if (T.on) { fprintf(stderr, "[VK] tier %s: the device stopped answering, the experts stay on the CPU\n", T.engine); T.on = 0; }
+ return 0;
+ }
+ if (layer <= T.last_layer) new_forward(S);
+ T.last_layer = layer;
+ quiesce();
+ int E = T.c.experts, H = T.c.hidden, n = S * K;
+ if (!grow_map(n)) return 0;
+ /* heat for every routed expert, groups for the resident ones (first seen first) */
+ int ng = 0, total = 0;
+ for (int i = 0; i < n; i++) {
+ T.map[i] = -1;
+ int e = idx[i];
+ if (e < 0 || e >= E) continue;
+ T.routed++;
+ VSlot *v = slot(layer, e);
+ if (v->heat < 0xFFFFFFFFu) v->heat++;
+ v->last = T.tick;
+ if (v->state != VS_RESIDENT) continue;
+ int g = T.grp[e];
+ if (g < 0) {
+ if (total >= T.max_dev_rows || !grow_groups(ng + 1)) continue; /* the batch is as large as it gets */
+ g = ng++; T.grp[e] = g; T.touched[g] = e; T.bex[g] = v->ex; T.brows[g] = 0;
+ } else if (total >= T.max_dev_rows) continue;
+ T.brows[g]++; total++;
+ T.map[i] = g; /* group for now, row index below */
+ }
+ if (!ng) return 0;
+ /* rows expert by expert, each expert's in routing order */
+ if (!grow_rows(total)) {
+ for (int g = 0; g < ng; g++) T.grp[T.touched[g]] = -1;
+ for (int i = 0; i < n; i++) T.map[i] = -1;
+ return 0;
+ }
+ int at = 0;
+ for (int g = 0; g < ng; g++) { T.bfirst[g] = at; at += T.brows[g]; T.brows[g] = 0; }
+ for (int i = 0; i < n; i++) {
+ int g = T.map[i];
+ if (g < 0) continue;
+ int j = T.bfirst[g] + T.brows[g]++;
+ T.bx[j] = x + (size_t)(i / K) * H;
+ T.map[i] = j;
+ }
+ for (int g = 0; g < ng; g++) T.grp[T.touched[g]] = -1;
+ if (!coli_vk_xb_issue(T.bex, T.brows, ng, T.bx)) {
+ for (int i = 0; i < n; i++) T.map[i] = -1;
+ return 0;
+ }
+ for (int i = 0; i < n; i++) if (T.map[i] >= 0) taken[i] = 1;
+ T.inflight = 1; T.S = S; T.K = K; T.steps++; T.served += (unsigned long long)total;
+ T.t_issued = vkt_now_ms();
+ return total;
+}
+
+int vkt_join(const float **rows) {
+ if (!T.inflight) return 0;
+ double t0 = vkt_now_ms(), dms = 0;
+ T.cpu_ms += t0 - T.t_issued;
+ int ok = coli_vk_xb_join(T.by, &dms);
+ T.wait_ms += vkt_now_ms() - t0;
+ T.dev_ms += dms;
+ T.inflight = 0;
+ int n = T.S * T.K;
+ for (int i = 0; i < n; i++) rows[i] = ok && T.map[i] >= 0 ? T.by[T.map[i]] : NULL;
+ if (!ok) {
+ fprintf(stderr, "[VK] tier %s: a batch failed, its experts are recomputed on the CPU and the tier stops\n", T.engine);
+ T.on = 0;
+ return 0;
+ }
+ quiesce();
+ return 1;
+}
+
+int vkt_resident(int layer, int eid) {
+ return T.on && layer >= 0 && layer < T.c.layers && eid >= 0 && eid < T.c.experts &&
+ slot(layer, eid)->state == VS_RESIDENT;
+}
+int vkt_ready(void) { return T.on; }
+
+/* ---- budget ---------------------------------------------------------------------- */
+static size_t mem_available(void) {
+#ifdef __linux__
+ FILE *f = fopen("/proc/meminfo", "r");
+ if (!f) return 0;
+ char line[256]; size_t kb = 0;
+ while (fgets(line, sizeof line, f)) if (sscanf(line, "MemAvailable: %zu kB", &kb) == 1) break;
+ fclose(f);
+ return kb * 1024;
+#else
+ return 0;
+#endif
+}
+static double env_gb(const char *name, double def) {
+ const char *e = getenv(name);
+ return e && *e ? atof(e) : def;
+}
+
+int vkt_init(const VktConfig *cfg, uint32_t *const *heat) {
+ if (T.on || !cfg) return 0;
+ const char *on = getenv("COLI_VK_TIER");
+ if (on && *on == '0') return 0;
+ if (!coli_vk_available()) return 0;
+ const char *eng = cfg->engine ? cfg->engine : "engine";
+ memset(&T, 0, sizeof T);
+ T.c = *cfg;
+ snprintf(T.engine, sizeof T.engine, "%s", eng);
+ if (T.c.layers < 1 || T.c.experts < 1 || T.c.hidden < 1 || T.c.inter < 1 || T.c.topk < 1) return 0;
+ if (!dev_fmt(T.c.gate_up, &T.gu_fmt, &T.gu_gs) || !dev_fmt(T.c.down, &T.dn_fmt, &T.dn_gs)) {
+ fprintf(stderr, "[VK] tier %s: expert format %d/%d (gs %d/%d) has no device form, the experts stay on the CPU\n",
+ eng, T.c.gate_up.kind, T.c.down.kind, T.c.gate_up.gs, T.c.down.gs);
+ return 0;
+ }
+ int act = T.c.act == VKT_ACT_SITU ? COLI_VK_ACT_SITU : COLI_VK_ACT_SWIGLU;
+ if (!coli_vk_xb_init(T.c.hidden, T.c.inter, act, T.c.act_limit, T.c.act_a, T.c.act_b)) {
+ fprintf(stderr, "[VK] tier %s: no expert batch on this device (shaders?), the experts stay on the CPU\n", eng);
+ return 0;
+ }
+ int H = T.c.hidden, F = T.c.inter;
+ T.gu_codes = src_row_bytes(T.c.gate_up.kind, H) * (size_t)F;
+ T.d_codes = src_row_bytes(T.c.down.kind, F) * (size_t)H;
+ T.gu_scales = src_scale_bytes(T.c.gate_up, H, F);
+ T.d_scales = src_scale_bytes(T.c.down, F, H);
+ T.stage_bytes = 2 * T.gu_codes + T.d_codes + 2 * T.gu_scales + T.d_scales;
+ T.exp_bytes = vkt_expert_bytes(H, F, T.c.gate_up, T.c.down);
+
+ /* The budget. Discrete: what VK_EXT_memory_budget says is free on the device,
+ * minus a reserve (scratch, KV mirrors, driver) and the dense weights still to
+ * come. Integrated (device memory is host RAM): a quarter of what the RAM has
+ * left after the engine's expert cache has grown to its size and the dense
+ * weights are placed, so the tier never takes what the CPU cache needs.
+ * COLI_VK_TIER_GB replaces either, within what the device can hold. */
+ const double GiB = 1073741824.0;
+ double used = 0, bud = 0;
+ int have = coli_vk_mem_budget(&used, &bud);
+ double heap_free = have ? (bud - used) * 1e9 : (double)coli_vk_device_local_bytes() * 0.8;
+ T.uma = coli_vk_device_shares_ram();
+ double reserve = env_gb("COLI_VK_TIER_RESERVE_GB", 1.0) * GiB;
+ double room = heap_free - reserve - (double)T.c.dense_bytes;
+ double want;
+ const char *cap = getenv("COLI_VK_TIER_GB");
+ if (cap && *cap) want = atof(cap) * GiB;
+ else if (T.uma) {
+ double avail = (double)mem_available();
+ double ram_room = avail > 0 ? avail - (double)T.c.ram_reserve - (double)T.c.dense_bytes - 2.0 * GiB : room;
+ want = ram_room / 4;
+ } else want = room;
+ if (want > room) want = room;
+ if (want < 0) want = 0;
+ /* whole experts per pool block, so the count matches what the blocks can hold */
+ size_t blk = (size_t)256 << 20;
+ if ((size_t)want < blk) blk = (size_t)want;
+ long long fit = 0;
+ if (T.exp_bytes && blk >= T.exp_bytes) {
+ size_t full = (size_t)want / blk, rest = (size_t)want - full * blk;
+ fit = (long long)full * (long long)(blk / T.exp_bytes) + (long long)(rest / T.exp_bytes);
+ }
+ long long all = (long long)T.c.layers * T.c.experts;
+ if (fit > all) fit = all;
+ if (fit < 2) {
+ char hb[32], he[32];
+ fprintf(stderr, "[VK] tier %s: no room for experts (budget %s, %s each; COLI_VK_TIER_GB sets it), the experts stay on the CPU\n",
+ eng, human(want, hb, sizeof hb), human((double)T.exp_bytes, he, sizeof he));
+ return 0;
+ }
+ T.max_resident = (int)fit;
+ T.budget = (size_t)want;
+ coli_vk_tier_pool_limit(T.budget);
+ const char *r = getenv("COLI_VK_TIER_RATE");
+ T.rate = r && *r ? atoi(r) : 16;
+ if (T.rate < 0) T.rate = 0;
+ T.s = calloc((size_t)all, sizeof(VSlot));
+ T.grp = malloc((size_t)T.c.experts * sizeof(int));
+ if (!T.s || !T.grp) { free(T.s); free(T.grp); return 0; }
+ for (int e = 0; e < T.c.experts; e++) T.grp[e] = -1;
+ T.max_dev_rows = T.c.max_rows > 0 ? T.c.max_rows : 1 << 20;
+ T.last_layer = 1 << 30; T.decay_at = VKT_DECAY_TOKENS;
+ /* The history, scaled: the hottest expert of a layer starts at 32 and the rest in
+ * proportion, so a few minutes of a new workload can displace it (raw counts
+ * from a long history would hold the tier for hours). */
+ g_heat_hist = heat;
+ if (heat)
+ for (int l = 0; l < T.c.layers; l++) {
+ if (!heat[l]) continue;
+ uint32_t mx = 0;
+ for (int e = 0; e < T.c.experts; e++) if (heat[l][e] > mx) mx = heat[l][e];
+ if (mx) for (int e = 0; e < T.c.experts; e++)
+ if (heat[l][e]) T.s[(size_t)l * T.c.experts + e].heat = 1 + (uint32_t)(31.0 * heat[l][e] / mx);
+ }
+ if (pthread_mutex_init(&T.mx, NULL) || pthread_cond_init(&T.cv, NULL) || pthread_cond_init(&T.cv_room, NULL)) return 0;
+ if (pthread_create(&T.th, NULL, uploader, NULL)) return 0;
+ T.th_on = 1;
+ T.on = 1;
+ char hb[32], he[32];
+ fprintf(stderr, "[VK] tier %s: on, %s, budget %s = %d experts of %s (fmt %d", eng,
+ coli_vk_device_name(), human((double)T.budget, hb, sizeof hb), T.max_resident,
+ human((double)T.exp_bytes, he, sizeof he), T.gu_fmt);
+ if (T.gu_gs) fprintf(stderr, " gs %d", T.gu_gs);
+ fprintf(stderr, ", down fmt %d", T.dn_fmt);
+ if (T.dn_gs) fprintf(stderr, " gs %d", T.dn_gs);
+ fprintf(stderr, "), %s, %s queue, up to %d promotions per forward\n",
+ T.uma ? (cap && *cap ? "shared RAM (COLI_VK_TIER_GB)" : "shared RAM: a quarter of what the expert cache leaves")
+ : "device memory", coli_vk_xb_queue_shared() ? "shared" : "own", T.rate);
+ return 1;
+}
+
+void vkt_report(const char *scope, unsigned long long ram_hits, unsigned long long disk_loads) {
+ if (!T.s) return;
+ ColiVkPoolStats ps; coli_vk_pool_stats(1, &ps);
+ unsigned long long r = T.routed - T.rep_routed, sv = T.served - T.rep_served;
+ T.rep_routed = T.routed; T.rep_served = T.served;
+ double hidden = T.dev_ms > T.wait_ms ? T.dev_ms - T.wait_ms : 0;
+ char hu[32], hb[32], hl[32];
+ fprintf(stderr, "[VK] tier %s %s: device %llu of %llu routed experts (%.1f%%; this %s %llu of %llu) | "
+ "CPU RAM hits %llu, disk loads %llu | resident %d (budget %d, %s of %s, %d blocks, frag %.2f) | "
+ "uploads %llu (%s, %llu warm), evictions %llu, skipped %llu queue + %llu rate, failed %llu | "
+ "device %.1f ms, CPU share %.1f ms, waited %.1f ms (%.0f%% of device time hidden)\n",
+ T.engine, scope, T.served, T.routed, T.routed ? 100.0 * T.served / T.routed : 0.0, scope, sv, r,
+ ram_hits, disk_loads, T.resident, T.max_resident, human((double)ps.used, hu, sizeof hu),
+ human((double)T.budget, hb, sizeof hb), ps.blocks, ps.frag, T.uploads,
+ human((double)T.upload_bytes, hl, sizeof hl), T.warm, T.evictions, T.qfull, T.rated, T.failed,
+ T.dev_ms, T.cpu_ms, T.wait_ms, T.dev_ms > 0 ? 100.0 * hidden / T.dev_ms : 0.0);
+}
+
+void vkt_shutdown(void) {
+ if (!T.s) return;
+ if (T.th_on) {
+ pthread_mutex_lock(&T.mx); T.stop = 1; pthread_cond_broadcast(&T.cv); pthread_cond_broadcast(&T.cv_room); pthread_mutex_unlock(&T.mx);
+ pthread_join(T.th, NULL);
+ T.th_on = 0;
+ }
+ if (T.inflight) { const float *dummy[1]; (void)dummy; coli_vk_xb_join(T.by, NULL); T.inflight = 0; }
+ for (int i = 0; i < T.qn; i++) free(T.q[(T.qh + i) % VKT_QCAP].buf);
+ T.qn = 0;
+ if (coli_vk_available()) {
+ quiesce();
+ size_t n = (size_t)T.c.layers * T.c.experts;
+ for (size_t i = 0; i < n; i++) if (T.s[i].ex) { coli_vk_xb_expert_free(T.s[i].ex); T.s[i].ex = NULL; }
+ }
+ pthread_mutex_destroy(&T.mx); pthread_cond_destroy(&T.cv); pthread_cond_destroy(&T.cv_room);
+ free(T.s); free(T.grp); free(T.map); free(T.touched); free(T.bex); free(T.brows); free(T.bfirst);
+ free(T.bx); free(T.by); free(T.evict); free(T.done);
+ memset(&T, 0, sizeof T);
+}
+#endif /* COLI_VULKAN */
diff --git a/c/vk_tier.h b/c/vk_tier.h
new file mode 100644
index 00000000..e7d87f2d
--- /dev/null
+++ b/c/vk_tier.h
@@ -0,0 +1,160 @@
+/* vk_tier.h -- the Vulkan routed-expert tier shared by the MoE engines.
+ *
+ * What it does, for a GPU a Vulkan driver can see (COLI_VULKAN=1):
+ * - holds a cache of routed experts in device memory, under a budget taken from
+ * VK_EXT_memory_budget (minus a reserve and the dense trunk the engine will
+ * place there), capped by COLI_VK_TIER_GB;
+ * - fills it at startup from the expert history (.coli_usage, route_trace.h) and
+ * keeps adapting while the user chats: an expert the CPU computes is a candidate,
+ * admitted when there is room or when it is hotter than the coldest resident by
+ * tier.h's LFRU margin, which is then evicted; uploads run on a thread of their
+ * own and never hold up a layer;
+ * - computes, per layer step, every resident expert the routing picked as ONE
+ * asynchronous submit (backend_vulkan.c's expert batch) while the engine computes
+ * the rest on the CPU, then hands back one output row per (row, rank) so the
+ * engine adds them in its own routing order;
+ * - on an integrated GPU, whose device memory is the CPU's RAM, defaults to a
+ * budget that leaves the RAM the engine's expert cache will need;
+ * - accounts device / CPU hits, uploads, evictions and how much device time the
+ * CPU work hid, in one "[VK] tier <engine>" line per run and serve turn.
+ * With COLI_VULKAN unset, COLI_VK_TIER=0, a CUDA tier active, or a build without
+ * VK=1 (the inline stubs below), nothing here runs and the engine is unchanged.
+ *
+ * ---- integrating an engine (the phase-2 guide) -------------------------------
+ * 1. Describe the experts once, after the model and the dense trunk are loaded and
+ * after the engine's history is read (rt_load), before the first token:
+ *
+ * VktConfig c = {.engine = "glm53", .layers = L, .experts = E,
+ * .hidden = H, .inter = F, .topk = K,
+ * .gate_up = {VKT_SRC_I4U_PAIRS_GS, 64}, // how RAM holds them
+ * .down = {VKT_SRC_I4U_PAIRS_GS, 64},
+ * .act = VKT_ACT_SWIGLU, .act_limit = swiglu_limit,
+ * .max_rows = prefill_rows * K,
+ * .ram_reserve = bytes the RAM expert cache may still grow by,
+ * .dense_bytes = bytes of dense weights the engine will put
+ * on the device after this call};
+ * if (vkt_init(&c, rt_counts_all())) atexit(vkt_shutdown);
+ *
+ * The heat table is [layers][experts] counts (route_trace.h's rows; NULL rows are
+ * layers without experts). Pick .gate_up/.down from the table below; gate and up
+ * must share one kind. A layer the tier should not serve (an MTP layer in another
+ * format) is simply never passed to vkt_issue: index layers 0..layers-1 only.
+ *
+ * 2. Warm start (optional, recommended when there is history): the tier plans, the
+ * engine reads each planned expert from disk (any number of threads) and hands
+ * it over, then publishes on the engine thread:
+ *
+ * int n = vkt_plan(pl, pe, max);
+ * #pragma omp parallel for
+ * for (i < n) { read expert (pl[i], pe[i]) into a private buffer;
+ * vkt_put(pl[i], pe[i], &src); }
+ * vkt_put_done();
+ *
+ * 3. Every MoE layer step, rows x[S][hidden] routed to idx[S][K] (rank order, -1 =
+ * unused slot):
+ *
+ * uint8_t taken[S*K]; const float *dev[S*K];
+ * int n = vkt_issue(layer, x, S, K, idx, taken); // returns at once
+ * for every (s,k) with !taken: compute on the CPU into its own buffer,
+ * and when the expert's bytes are in RAM: vkt_note(layer, e, &src);
+ * shared experts, other CPU work ...
+ * if (n && !vkt_join(dev)) compute the taken (s,k) on the CPU too (device lost);
+ * for s: for k in rank order: out[s] += w[s][k] * (taken ? dev[s*K+k] : cpu[s][k]);
+ *
+ * vkt_issue returns how many (s,k) the device took; join only when it took any.
+ *
+ * Add in rank order, device or not: then the sum's order never depends on which
+ * experts happened to be resident (it is the CPU run's order, the values differ
+ * by the device's summation order only). Never call the dense Vulkan matmuls
+ * between vkt_issue and vkt_join from a parallel region (backend rule), and
+ * keep one step in flight: join before the next layer's issue.
+ *
+ * 4. Report: vkt_report("run" or "turn", ram_hits, disk_loads) where the engine
+ * prints its "[VK] <engine>" line; EMAP's tier bits: vkt_resident(l, e) -> 2.
+ *
+ * Source kinds (VktSrc) and what the device holds -- the conversion runs on the
+ * uploader thread, nothing changes in the engine's RAM:
+ * VKT_SRC_I8_ROW int8, f32 per row -> fmt 1
+ * VKT_SRC_I8_GS int8, f32 per (row, gs) -> fmt 13
+ * VKT_SRC_I8_AS_I4_ROW int8 holding int4 values [-8,7], per row -> fmt 2 (repacked: half the bytes)
+ * VKT_SRC_I8_AS_I4_GS same, f32 per (row, gs) -> fmt 4
+ * VKT_SRC_I4S_PAIRS_ROW signed int4, byte j = elements 2j (low), 2j+1, per row -> fmt 2
+ * VKT_SRC_I4S_PAIRS_GS same, per (row, gs) -> fmt 4
+ * VKT_SRC_I4U_PAIRS_ROW int4 as v+8, pairs, per row -> fmt 2
+ * VKT_SRC_I4U_PAIRS_GS int4 as v+8, pairs, per (row, gs) -> fmt 4 (GLM, glm53)
+ * VKT_SRC_I4U_PLANAR64 expert_ffn.h planar blocks of 64, v+8, f32 per 64 -> fmt 4 gs 64
+ * VKT_SRC_I3_G64 int3-g64 planes -> fmt 5
+ * VKT_SRC_MXFP4_F32 e2m1 pairs, f32 per gs -> fmt 7
+ * VKT_SRC_MXFP4_E8M0 e2m1 pairs, ue8m0 byte per gs -> fmt 7 (kimi_k3, mimo, deepseek_v41)
+ * VKT_SRC_FP8_GS e4m3, f32 per (row, gs) -> fmt 12
+ * VKT_SRC_FP8_BLOCK e4m3, f32 per gs x gs block -> fmt 12 (qwen38 native)
+ * VKT_SRC_BF16, VKT_SRC_F32 -> fmt 11, 10
+ * Activations (VktConfig.act): VKT_ACT_SWIGLU silu(g)*u, act_limit > 0 clamps the
+ * gate from above and up to [-limit, limit] first (GLM-5.3, DeepSeek V4/V4.1);
+ * VKT_ACT_SITU a*tanh(g/a)*sigmoid(g) * b*tanh(u/b) with act_a, act_b (Kimi K3). */
+#ifndef COLI_VK_TIER_H
+#define COLI_VK_TIER_H
+#include <stdint.h>
+#include <stddef.h>
+
+typedef enum {
+ VKT_SRC_NONE = 0,
+ VKT_SRC_I8_ROW, VKT_SRC_I8_GS, VKT_SRC_I8_AS_I4_ROW, VKT_SRC_I8_AS_I4_GS,
+ VKT_SRC_I4S_PAIRS_ROW, VKT_SRC_I4S_PAIRS_GS, VKT_SRC_I4U_PAIRS_ROW, VKT_SRC_I4U_PAIRS_GS,
+ VKT_SRC_I4U_PLANAR64, VKT_SRC_I3_G64, VKT_SRC_MXFP4_F32, VKT_SRC_MXFP4_E8M0,
+ VKT_SRC_FP8_GS, VKT_SRC_FP8_BLOCK, VKT_SRC_BF16, VKT_SRC_F32
+} VktSrc;
+typedef struct { VktSrc kind; int gs; } VktFmt; /* gs: group (block) size, 0 = per row */
+
+#define VKT_ACT_SWIGLU 0
+#define VKT_ACT_SITU 1
+
+typedef struct {
+ const char *engine; /* names the [VK] tier lines */
+ int layers, experts, hidden, inter, topk;
+ VktFmt gate_up, down;
+ int act; float act_limit, act_a, act_b;
+ int max_rows; /* most (row, expert) assignments one vkt_issue carries */
+ size_t ram_reserve; /* bytes the engine's RAM caches may still grow by */
+ size_t dense_bytes; /* dense weights the engine puts on the device after vkt_init */
+} VktConfig;
+
+/* One expert as it sits in RAM: codes and scales of gate, up, down (float scales,
+ * or ue8m0 bytes for VKT_SRC_MXFP4_E8M0; NULL for bf16/f32). Read during the call
+ * only: the tier copies what it keeps. */
+typedef struct {
+ const void *g, *u, *d;
+ const void *gs, *us, *ds;
+} VktExpertSrc;
+
+#ifdef COLI_VULKAN
+/* 1 = the tier is on. heat: [layers][experts] routing counts, or NULL. */
+int vkt_init(const VktConfig *cfg, uint32_t *const *heat);
+int vkt_ready(void);
+void vkt_shutdown(void);
+int vkt_plan(int *layers, int *eids, int max);
+int vkt_put(int layer, int eid, const VktExpertSrc *src); /* any thread */
+void vkt_put_done(void);
+void vkt_note(int layer, int eid, const VktExpertSrc *src);
+int vkt_issue(int layer, const float *x, int S, int K, const int *idx, uint8_t *taken);
+int vkt_join(const float **rows);
+int vkt_resident(int layer, int eid);
+void vkt_report(const char *scope, unsigned long long ram_hits, unsigned long long disk_loads);
+/* Sizing helpers for engines: bytes one expert takes on the device in a source format. */
+size_t vkt_expert_bytes(int hidden, int inter, VktFmt gate_up, VktFmt down);
+#else
+static inline int vkt_init(const VktConfig *c, uint32_t *const *h){(void)c;(void)h;return 0;}
+static inline int vkt_ready(void){return 0;}
+static inline void vkt_shutdown(void){}
+static inline int vkt_plan(int *l,int *e,int m){(void)l;(void)e;(void)m;return 0;}
+static inline int vkt_put(int l,int e,const VktExpertSrc *s){(void)l;(void)e;(void)s;return 0;}
+static inline void vkt_put_done(void){}
+static inline void vkt_note(int l,int e,const VktExpertSrc *s){(void)l;(void)e;(void)s;}
+static inline int vkt_issue(int l,const float *x,int S,int K,const int *i,uint8_t *t){(void)l;(void)x;(void)S;(void)K;(void)i;(void)t;return 0;}
+static inline int vkt_join(const float **r){(void)r;return 0;}
+static inline int vkt_resident(int l,int e){(void)l;(void)e;return 0;}
+static inline void vkt_report(const char *s,unsigned long long r,unsigned long long d){(void)s;(void)r;(void)d;}
+static inline size_t vkt_expert_bytes(int h,int i,VktFmt a,VktFmt b){(void)h;(void)i;(void)a;(void)b;return 0;}
+#endif
+
+#endif