Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vk_tier: the routed-expert tier the MoE engines share
JustVugg53310db844b4
diff --git a/c/Makefile b/c/Makefileindex 90cf23d5..2a91104c 100644--- a/c/Makefile+++ b/c/Makefile@@ -527,7 +527,7 @@ TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framitest_edge_adapters_registration test_edge_adapters_real \test_xdna_execution test_xdna_failure \test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_st_f16_bf16_simd_sse41 test_qwen38_tier_engine \- test_e8x4g64_loader test_qwenimage_vae \+ test_e8x4g64_loader test_qwenimage_vae test_vk_tier \test_i4_grouped_sse41 test_i4_grouped_sse41_o1 test_i4_grouped_sse41_no_contractTEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))ifneq (,$(LINUX))@@ -703,7 +703,9 @@ LDFLAGS += -Wl,-Bdynamic -lvulkan-1 -Wl,-BstaticelseLDFLAGS += -lvulkanendif-VK_OBJ = backend_vulkan.o+# vk_tier.o: the routed-expert tier the MoE engines share (vk_tier.h); an engine+# that does not call it links nothing from it.+VK_OBJ = backend_vulkan.o vk_tier.oVK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv \shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv shaders/expert_act.spvendif@@ -839,6 +841,8 @@ colibri$(EXE): colibri.c $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) $(XDNA_OBJ# Vulkan backend object (plain C + vulkan headers) and its SPIR-V shaders.backend_vulkan.o: backend_vulkan.c .build-config$(CC) $(CFLAGS) -c backend_vulkan.c -o $@+vk_tier.o: vk_tier.c .build-config+ $(CC) $(CFLAGS) -c vk_tier.c -o $@shaders/%.spv: shaders/%.comp@command -v $(GLSLC) >/dev/null 2>&1 || { echo "glslc not found: install shaderc (for VK=1)" >&2; exit 1; }$(GLSLC) --target-env=vulkan1.2 $< -o $@@@ -1942,6 +1946,17 @@ tests/test_expert_ffn$(EXE): tests/test_expert_ffn.ctests/test_vk_alloc$(EXE): tests/test_vk_alloc.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# The Vulkan routed-expert tier (vk_tier.c) against a CPU reference: every source+# format, warm start, eviction, partial batches. It needs VK=1 and a Vulkan device+# (CI: Lavapipe), so it stays out of `check` (TEST_EXCLUDE) and runs from+# vk-tier-check.+tests/test_vk_tier$(EXE): tests/test_vk_tier.c $(VK_OBJ) $(VK_SPV)+ @[ "$(VK)" = 1 ] || { echo "tests/test_vk_tier needs VK=1" >&2; exit 1; }+ $(CC) $(CFLAGS) $< $(VK_OBJ) -o $@ $(LDFLAGS)+.PHONY: vk-tier-check+vk-tier-check: tests/test_vk_tier$(EXE)+ ./tests/test_vk_tier$(EXE) shaders/qmatmul.spv+tests/test_rans$(EXE): tests/test_rans.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)@@ -2728,7 +2743,7 @@ AUTODEP_OWN_FLAGS = bench_cuda_resident_batch bench_load_tq_peak_rss \bench_indexer_allocationsENGINE_RULES := $(foreach n,$(RULE_NAMES),$(if $(findstring /,$(n)),,$(if $(wildcard $(n).c),$(n))))AUTODEP_TESTS := $(patsubst tests/%,%,$(filter tests/%,$(RULE_NAMES)))-AUTODEP_OBJS = backend_vulkan.o backend_loader.o backend_xdna.o qwen36_tier.o \+AUTODEP_OBJS = backend_vulkan.o vk_tier.o backend_loader.o backend_xdna.o qwen36_tier.o \tests/backend_xdna_lane.o tests/edge_runtime.o tests/edge_runtime_nocuda.o \tests/segment_runtime.o tests/segment_runtime_nocuda.o \tests/segment_conformance_fixtures.o tests/deepseek_v4_native_quant.o \diff --git a/c/tests/test_vk_tier.c b/c/tests/test_vk_tier.cnew file mode 100644index 00000000..37d7b6f8--- /dev/null+++ b/c/tests/test_vk_tier.c@@ -0,0 +1,342 @@+/* The Vulkan routed-expert tier (vk_tier.c) against a CPU reference, on any Vulkan+ * device (CI: Lavapipe). A small synthetic MoE -- L layers of E experts, top-K --+ * whose weights are random in every source format the tier accepts, decoded here+ * on their own, independently of the tier and of the backend. Gates:+ * formats every VktSrc kind and both activations: each row the device returns+ * equals the reference expert output, and the rank-order sum of a mixed+ * step (device and CPU rows) equals the all-CPU sum, within 2e-3;+ * warm a history fills the budget in heat order (vkt_plan / vkt_put);+ * adapt a budget of a third of the experts and a hot set that moves halfway:+ * experts are evicted, the new hot set becomes resident and its steps+ * go to the device;+ * partial a step with more assignments than max_rows: the device takes+ * max_rows of them, the CPU the rest, the sum is still right;+ * books device + CPU = routed, resident <= budget, no failed upload.+ *+ * make tests/test_vk_tier VK=1 && VK_ICD_FILENAMES=.../lvp_icd.json ./tests/test_vk_tier */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <math.h>+#include "../backend_vulkan.h"+#include "../vk_tier.h"++static int fails;+#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL %s:%d: ", __FILE__, __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)++enum { L = 2, E = 10, H = 192, F = 128, K = 3 };+static unsigned rng = 7;+static unsigned rnd(void) { rng = rng * 1103515245u + 12345u; return rng >> 8; }+static float frnd(void) { return (float)((int)(rnd() % 2001) - 1000) / 1000.0f; }++/* ---- one matrix in a source format, and its own decode ------------------------- */+typedef struct { VktFmt f; int I, O; uint8_t *codes; void *scales; float *w; } Mat;+static size_t row_bytes(VktSrc k, int I) {+ switch (k) {+ case VKT_SRC_I8_ROW: case VKT_SRC_I8_GS: case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I8_AS_I4_GS:+ case VKT_SRC_FP8_GS: case VKT_SRC_FP8_BLOCK: return (size_t)I;+ case VKT_SRC_I3_G64: return ((size_t)I + 63) / 64 * 24;+ case VKT_SRC_BF16: return (size_t)I * 2;+ case VKT_SRC_F32: return (size_t)I * 4;+ default: return ((size_t)I + 1) / 2;+ }+}+static float e4m3(uint8_t b) {+ int e = (b >> 3) & 15, m = b & 7;+ float v = e == 0 ? m / 512.0f : ldexpf(1.0f + m / 8.0f, e - 7);+ return (b & 0x80) ? -v : v;+}+static int nib(const uint8_t *row, int i) { return (row[i >> 1] >> ((i & 1) * 4)) & 15; }+/* the weight value (o, i) before scales, as the format defines it */+static float code(const Mat *m, int o, int i) {+ const uint8_t *row = m->codes + (size_t)o * row_bytes(m->f.kind, m->I);+ switch (m->f.kind) {+ case VKT_SRC_I8_ROW: case VKT_SRC_I8_GS: case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I8_AS_I4_GS:+ return (float)(int8_t)row[i];+ case VKT_SRC_I4S_PAIRS_ROW: case VKT_SRC_I4S_PAIRS_GS: { int v = nib(row, i); return (float)(v >= 8 ? v - 16 : v); }+ case VKT_SRC_I4U_PAIRS_ROW: case VKT_SRC_I4U_PAIRS_GS: return (float)(nib(row, i) - 8);+ case VKT_SRC_I4U_PLANAR64: {+ int full = m->I / 64 * 64;+ if (i >= full) return (float)(nib(row + full / 2, i - full) - 8);+ const uint8_t *blk = row + (size_t)(i / 64) * 32; int k = i % 64;+ return (float)((k < 32 ? (blk[k] & 15) : (blk[k - 32] >> 4)) - 8);+ }+ case VKT_SRC_I3_G64: {+ const uint8_t *lo = row + (size_t)(i / 64) * 24, *hi = lo + 16; int j = i % 64;+ return (float)((int)(((lo[j >> 2] >> ((j & 3) * 2)) & 3u) | (((hi[j >> 3] >> (j & 7)) & 1u) << 2)) - 4);+ }+ case VKT_SRC_MXFP4_F32: case VKT_SRC_MXFP4_E8M0: {+ static const float lut[8] = {0, .5f, 1, 1.5f, 2, 3, 4, 6};+ int v = nib(row, i); return (v & 8) ? -lut[v & 7] : lut[v & 7];+ }+ case VKT_SRC_FP8_GS: case VKT_SRC_FP8_BLOCK: return e4m3(row[i]);+ case VKT_SRC_BF16: { uint16_t h; memcpy(&h, row + 2 * i, 2); uint32_t u = (uint32_t)h << 16; float f; memcpy(&f, &u, 4); return f; }+ case VKT_SRC_F32: { float f; memcpy(&f, row + 4 * i, 4); return f; }+ default: return 0;+ }+}+static int per_row(VktSrc k) { return k == VKT_SRC_I8_ROW || k == VKT_SRC_I8_AS_I4_ROW || k == VKT_SRC_I4S_PAIRS_ROW || k == VKT_SRC_I4U_PAIRS_ROW; }+static float scale(const Mat *m, int o, int i) {+ VktSrc k = m->f.kind;+ if (k == VKT_SRC_BF16 || k == VKT_SRC_F32) return 1.0f;+ if (per_row(k)) return ((const float *)m->scales)[o];+ int gs = k == VKT_SRC_I4U_PLANAR64 || k == VKT_SRC_I3_G64 ? 64 : m->f.gs, ng = (m->I + gs - 1) / gs;+ if (k == VKT_SRC_MXFP4_E8M0) return ldexpf(1.0f, ((const uint8_t *)m->scales)[(size_t)o * ng + i / gs] - 127);+ if (k == VKT_SRC_FP8_BLOCK) return ((const float *)m->scales)[(size_t)(o / gs) * ng + i / gs];+ return ((const float *)m->scales)[(size_t)o * ng + i / gs];+}+static void mat_make(Mat *m, VktFmt f, int I, int O) {+ m->f = f; m->I = I; m->O = O;+ size_t rb = row_bytes(f.kind, I);+ m->codes = malloc(rb * O);+ for (size_t i = 0; i < rb * O; i++) m->codes[i] = (uint8_t)rnd();+ if (f.kind == VKT_SRC_I8_AS_I4_ROW || f.kind == VKT_SRC_I8_AS_I4_GS) /* an unpacked int4: [-8, 7] */+ for (size_t i = 0; i < rb * O; i++) m->codes[i] = (uint8_t)(int8_t)((int)(m->codes[i] % 16) - 8);+ if (f.kind == VKT_SRC_FP8_GS || f.kind == VKT_SRC_FP8_BLOCK)+ for (size_t i = 0; i < rb * O; i++) if ((m->codes[i] & 0x7f) == 0x7f) m->codes[i] ^= 1;+ if (f.kind == VKT_SRC_BF16 || f.kind == VKT_SRC_F32)+ for (int i = 0; i < I * O; i++) {+ float v = frnd() * 0.1f;+ if (f.kind == VKT_SRC_F32) memcpy(m->codes + 4 * (size_t)i, &v, 4);+ else { uint32_t u; memcpy(&u, &v, 4); uint16_t h = (uint16_t)(u >> 16); memcpy(m->codes + 2 * (size_t)i, &h, 2); }+ }+ int gs = f.kind == VKT_SRC_I4U_PLANAR64 || f.kind == VKT_SRC_I3_G64 ? 64 : f.gs, ng = gs ? (I + gs - 1) / gs : 1;+ size_t ns = per_row(f.kind) ? (size_t)O : f.kind == VKT_SRC_FP8_BLOCK ? (size_t)((O + gs - 1) / gs) * ng : (size_t)O * ng;+ if (f.kind == VKT_SRC_MXFP4_E8M0) {+ uint8_t *s = malloc(ns); for (size_t i = 0; i < ns; i++) s[i] = (uint8_t)(120 + rnd() % 4); m->scales = s;+ } else if (f.kind == VKT_SRC_BF16 || f.kind == VKT_SRC_F32) m->scales = NULL;+ else { float *s = malloc(ns * 4); for (size_t i = 0; i < ns; i++) s[i] = 0.004f + (rnd() % 100) / 25000.0f; m->scales = s; }+ m->w = malloc((size_t)I * O * sizeof(float));+ for (int o = 0; o < O; o++) for (int i = 0; i < I; i++) m->w[(size_t)o * I + i] = code(m, o, i) * scale(m, o, i);+}+static void mat_free(Mat *m) { free(m->codes); free(m->scales); free(m->w); }++/* ---- the synthetic model ---------------------------------------------------------- */+typedef struct { Mat g, u, d; } Ex;+static Ex ex[L][E];+static int g_act; static float g_limit;+static void expert_ref(const Ex *e, const float *x, float *y) {+ float h[F];+ for (int o = 0; o < F; o++) {+ double a = 0, b = 0;+ for (int i = 0; i < H; i++) { a += (double)e->g.w[(size_t)o * H + i] * x[i]; b += (double)e->u.w[(size_t)o * H + i] * x[i]; }+ float g = (float)a, u = (float)b;+ if (g_act == VKT_ACT_SITU) h[o] = 4.f * tanhf(g / 4.f) * (1.f / (1.f + expf(-g))) * (25.f * tanhf(u / 25.f));+ else {+ if (g_limit > 0) { if (g > g_limit) g = g_limit; if (u > g_limit) u = g_limit; if (u < -g_limit) u = -g_limit; }+ h[o] = g / (1.f + expf(-g)) * u;+ }+ }+ for (int o = 0; o < H; o++) { double a = 0; for (int i = 0; i < F; i++) a += (double)e->d.w[(size_t)o * F + i] * h[i]; y[o] = (float)a; }+}+static VktExpertSrc src_of(const Ex *e) {+ return (VktExpertSrc){e->g.codes, e->u.codes, e->d.codes, e->g.scales, e->u.scales, e->d.scales};+}+static void model_make(VktFmt gu, VktFmt dn) {+ for (int l = 0; l < L; l++) for (int e = 0; e < E; e++) {+ mat_make(&ex[l][e].g, gu, H, F); mat_make(&ex[l][e].u, gu, H, F); mat_make(&ex[l][e].d, dn, F, H);+ }+}+static void model_free(void) {+ for (int l = 0; l < L; l++) for (int e = 0; e < E; e++) { mat_free(&ex[l][e].g); mat_free(&ex[l][e].u); mat_free(&ex[l][e].d); }+}+static double rel(const float *a, const float *b, int n) {+ double num = 0, den = 0;+ for (int i = 0; i < n; i++) { double d = (double)a[i] - b[i]; num += d * d; den += (double)b[i] * b[i]; }+ return den > 0 ? sqrt(num / den) : sqrt(num);+}++/* One layer step as an engine runs it (vk_tier.h step 3): issue, CPU pairs with a+ * note each, join, rank-order sum. Returns the worst relative error of a device row+ * against the reference and of the sum against the all-CPU sum. */+typedef struct { unsigned long long routed, dev; } Books;+static double step(int layer, int S, const int *idx, const float *w, Books *bk) {+ float *x = calloc((size_t)S * H, sizeof(float)), *cpu = calloc((size_t)S * K * H, sizeof(float));+ float *out = calloc((size_t)S * H, sizeof(float)), *ref = calloc((size_t)S * H, sizeof(float)), *y = malloc(sizeof(float) * H);+ uint8_t *taken = malloc((size_t)S * K); const float **dev = malloc(sizeof(*dev) * S * K);+ for (int i = 0; i < S * H; i++) x[i] = frnd();+ int n = vkt_issue(layer, x, S, K, idx, taken);+ for (int i = 0; i < S * K; i++) {+ if (taken[i]) continue;+ expert_ref(&ex[layer][idx[i]], x + (size_t)(i / K) * H, cpu + (size_t)i * H);+ VktExpertSrc s = src_of(&ex[layer][idx[i]]); vkt_note(layer, idx[i], &s);+ }+ int joined = n ? vkt_join(dev) : 1;+ CHECK(joined, "join failed");+ double worst = 0;+ for (int s = 0; s < S; s++)+ for (int k = 0; k < K; k++) {+ int i = s * K + k;+ expert_ref(&ex[layer][idx[i]], x + (size_t)s * H, y);+ for (int d = 0; d < H; d++) ref[(size_t)s * H + d] += w[i] * y[d];+ const float *c = taken[i] && joined ? dev[i] : cpu + (size_t)i * H;+ if (taken[i] && joined) { double r = rel(c, y, H); if (r > worst) worst = r; }+ for (int d = 0; d < H; d++) out[(size_t)s * H + d] += w[i] * c[d];+ bk->routed++; bk->dev += taken[i] != 0;+ }+ double r = rel(out, ref, S * H);+ if (r > worst) worst = r;+ free(x); free(cpu); free(out); free(ref); free(y); free(taken); free(dev);+ return worst;+}+/* K distinct experts per row from a skewed draw over [lo, lo+span) */+static void route(int S, int lo, int span, int *idx, float *w) {+ for (int s = 0; s < S; s++)+ for (int k = 0; k < K; k++) {+ int e, dup;+ do { unsigned r = rnd() % 100; e = lo + (r < 60 ? (int)(r % 2) : (int)(r % span)) % span; e %= E;+ dup = 0; for (int j = 0; j < k; j++) dup |= idx[s * K + j] == e; } while (dup);+ idx[s * K + k] = e; w[s * K + k] = 0.2f + (rnd() % 100) / 200.0f;+ }+}++static VktConfig cfg_of(VktFmt gu, VktFmt dn, int act, float limit) {+ return (VktConfig){.engine = "test", .layers = L, .experts = E, .hidden = H, .inter = F, .topk = K,+ .gate_up = gu, .down = dn, .act = act, .act_limit = limit, .act_a = 4.f, .act_b = 25.f,+ .max_rows = 1 << 20};+}+static void set_budget(int experts, VktFmt gu, VktFmt dn) {+ char b[64]; snprintf(b, sizeof b, "%.12f", (experts + 0.5) * vkt_expert_bytes(H, F, gu, dn) / 1073741824.0);+ setenv("COLI_VK_TIER_GB", b, 1);+}++/* every format: the whole model fits, experts are promoted as they pass by */+static void formats(void) {+ struct { VktFmt gu, dn; int act; float limit; const char *name; } cs[] = {+ {{VKT_SRC_I8_ROW, 0}, {VKT_SRC_I8_ROW, 0}, VKT_ACT_SWIGLU, 0, "int8 per row"},+ {{VKT_SRC_I8_GS, 32}, {VKT_SRC_I8_GS, 32}, VKT_ACT_SWIGLU, 0, "int8 gs32"},+ {{VKT_SRC_I8_AS_I4_ROW, 0}, {VKT_SRC_I8_AS_I4_ROW, 0}, VKT_ACT_SWIGLU, 0, "unpacked int4 per row"},+ {{VKT_SRC_I8_AS_I4_GS, 64}, {VKT_SRC_I8_GS, 32}, VKT_ACT_SWIGLU, 0, "unpacked int4 gs64, int8 down"},+ {{VKT_SRC_I4S_PAIRS_ROW, 0}, {VKT_SRC_I4S_PAIRS_ROW, 0}, VKT_ACT_SWIGLU, 0, "signed int4 pairs per row"},+ {{VKT_SRC_I4S_PAIRS_GS, 32}, {VKT_SRC_I4S_PAIRS_GS, 32}, VKT_ACT_SWIGLU, 0, "signed int4 pairs gs32"},+ {{VKT_SRC_I4U_PAIRS_ROW, 0}, {VKT_SRC_I4U_PAIRS_ROW, 0}, VKT_ACT_SWIGLU, 0, "int4 v+8 per row"},+ {{VKT_SRC_I4U_PAIRS_GS, 64}, {VKT_SRC_I4U_PAIRS_GS, 64}, VKT_ACT_SWIGLU, 2.5f, "int4 v+8 gs64, SwiGLU limit"},+ {{VKT_SRC_I4U_PLANAR64, 64}, {VKT_SRC_I4U_PLANAR64, 64}, VKT_ACT_SWIGLU, 0, "planar int4-g64"},+ {{VKT_SRC_I3_G64, 0}, {VKT_SRC_I3_G64, 0}, VKT_ACT_SWIGLU, 0, "int3-g64"},+ {{VKT_SRC_MXFP4_F32, 32}, {VKT_SRC_MXFP4_F32, 32}, VKT_ACT_SWIGLU, 0, "MXFP4 f32 scales"},+ {{VKT_SRC_MXFP4_E8M0, 32}, {VKT_SRC_MXFP4_E8M0, 32}, VKT_ACT_SITU, 0, "MXFP4 ue8m0, SiTU-GLU"},+ {{VKT_SRC_FP8_GS, 32}, {VKT_SRC_FP8_GS, 32}, VKT_ACT_SWIGLU, 0, "fp8 gs32"},+ {{VKT_SRC_FP8_BLOCK, 128}, {VKT_SRC_FP8_BLOCK, 128}, VKT_ACT_SWIGLU, 0, "fp8 128x128 blocks"},+ {{VKT_SRC_BF16, 0}, {VKT_SRC_BF16, 0}, VKT_ACT_SWIGLU, 0, "bf16"},+ {{VKT_SRC_F32, 0}, {VKT_SRC_F32, 0}, VKT_ACT_SWIGLU, 0, "f32"},+ };+ for (size_t c = 0; c < sizeof cs / sizeof *cs; c++) {+ model_make(cs[c].gu, cs[c].dn);+ g_act = cs[c].act; g_limit = cs[c].limit;+ set_budget(L * E, cs[c].gu, cs[c].dn);+ setenv("COLI_VK_TIER_RATE", "64", 1);+ VktConfig vc = cfg_of(cs[c].gu, cs[c].dn, cs[c].act, cs[c].limit);+ int on = vkt_init(&vc, NULL);+ CHECK(on, "%s: the tier did not start", cs[c].name);+ if (!on) { model_free(); continue; }+ Books bk = {0, 0}; double worst = 0;+ static const int Ss[] = {1, 3, 20, 1, 2, 24};+ for (int t = 0; t < 6; t++)+ for (int l = 0; l < L; l++) {+ int idx[24 * K]; float w[24 * K];+ route(Ss[t], 0, E, idx, w);+ double r = step(l, Ss[t], idx, w, &bk);+ if (r > worst) worst = r;+ }+ printf(" %-34s device %3llu of %3llu assignments, worst relative error %.2e\n", cs[c].name, bk.dev, bk.routed, worst);+ CHECK(bk.dev > 0, "%s: nothing ran on the device", cs[c].name);+ CHECK(worst < 2e-3, "%s: device rows off the reference (%.3g)", cs[c].name, worst);+ vkt_shutdown(); model_free();+ }+}++/* a history fills the budget in heat order */+static void warm(void) {+ VktFmt f = {VKT_SRC_I4U_PAIRS_GS, 64};+ model_make(f, f); g_act = VKT_ACT_SWIGLU; g_limit = 0;+ set_budget(7, f, f);+ uint32_t hist[L][E], *rows[L];+ for (int l = 0; l < L; l++) { rows[l] = hist[l]; for (int e = 0; e < E; e++) hist[l][e] = (uint32_t)(1000 - 37 * (l * E + e) % 700); }+ VktConfig vc = cfg_of(f, f, VKT_ACT_SWIGLU, 0);+ CHECK(vkt_init(&vc, rows), "warm: the tier did not start");+ int pl[L * E], pe[L * E];+ int n = vkt_plan(pl, pe, L * E);+ CHECK(n == 7, "warm: planned %d experts for a budget of 7", n);+ /* the plan is the 7 hottest, hottest first */+ int ok = 1;+ for (int i = 1; i < n; i++) ok &= hist[pl[i - 1]][pe[i - 1]] >= hist[pl[i]][pe[i]];+ for (int l = 0; l < L; l++) for (int e = 0; e < E; e++) {+ int planned = 0; for (int i = 0; i < n; i++) planned |= pl[i] == l && pe[i] == e;+ if (!planned) ok &= hist[l][e] <= hist[pl[n - 1]][pe[n - 1]];+ }+ CHECK(ok, "warm: the plan is not the hottest experts in heat order");+ for (int i = 0; i < n; i++) { VktExpertSrc s = src_of(&ex[pl[i]][pe[i]]); vkt_put(pl[i], pe[i], &s); }+ vkt_put_done();+ int res = 0; for (int i = 0; i < n; i++) res += vkt_resident(pl[i], pe[i]);+ CHECK(res == n, "warm: %d of %d planned experts resident", res, n);+ Books bk = {0, 0};+ int idx[K] = {pe[0], (pe[0] + 1) % E, (pe[0] + 2) % E}; float w[K] = {0.5f, 0.3f, 0.2f};+ double r = step(pl[0], 1, idx, w, &bk);+ CHECK(bk.dev >= 1 && r < 2e-3, "warm: the hottest expert is not served from the device (%llu, %.3g)", bk.dev, r);+ printf(" warm start: %d experts planned in heat order and resident, the hottest served by the device\n", n);+ vkt_shutdown(); model_free();+}++/* a third of the experts fit; the hot set moves halfway */+static void adapt(void) {+ VktFmt f = {VKT_SRC_FP8_GS, 32};+ model_make(f, f); g_act = VKT_ACT_SWIGLU; g_limit = 0;+ set_budget(L * E / 3, f, f);+ setenv("COLI_VK_TIER_RATE", "4", 1);+ VktConfig vc = cfg_of(f, f, VKT_ACT_SWIGLU, 0);+ CHECK(vkt_init(&vc, NULL), "adapt: the tier did not start");+ Books a = {0, 0}, b1 = {0, 0}, b2 = {0, 0}; double worst = 0;+ int idx[4 * K]; float w[4 * K];+ /* LFRU: frequency first, halved every 1024 tokens, so an established hot set+ * yields over a few hundred steps of a new one, not at its first appearance */+ for (int t = 0; t < 100; t++) /* phase A: experts 0..4 */+ for (int l = 0; l < L; l++) { route(4, 0, 5, idx, w); double r = step(l, 4, idx, w, &a); if (r > worst) worst = r; }+ int resA = 0; for (int l = 0; l < L; l++) for (int e = 0; e < 5; e++) resA += vkt_resident(l, e);+ for (int t = 0; t < 400; t++) /* phase B: experts 5..9 */+ for (int l = 0; l < L; l++) { route(4, 5, 5, idx, w); double r = step(l, 4, idx, w, t < 200 ? &b1 : &b2); if (r > worst) worst = r; }+ int resB = 0, resA2 = 0, res = 0;+ for (int l = 0; l < L; l++) for (int e = 0; e < E; e++) { int v = vkt_resident(l, e); res += v; if (e >= 5) resB += v; else resA2 += v; }+ printf(" adapt: phase A device %.0f%%, resident A %d; phase B device %.0f%% then %.0f%%, resident B %d, A %d, all %d (budget %d)\n",+ 100.0 * a.dev / a.routed, resA, 100.0 * b1.dev / b1.routed, 100.0 * b2.dev / b2.routed, resB, resA2, res, L * E / 3);+ vkt_report("test", 0, 0);+ CHECK(resA > 0 && a.dev > 0, "adapt: phase A promoted nothing");+ CHECK(resB > resA2, "adapt: the new hot set did not take the device (B %d, A %d)", resB, resA2);+ CHECK(res <= L * E / 3, "adapt: %d resident over a budget of %d", res, L * E / 3);+ CHECK(b2.dev * b1.routed > b1.dev * b2.routed, "adapt: the device share did not rise in phase B");+ CHECK(worst < 2e-3, "adapt: relative error %.3g", worst);+ vkt_shutdown(); model_free();+}++/* more assignments than max_rows: the rest stays on the CPU */+static void partial(void) {+ VktFmt f = {VKT_SRC_I8_ROW, 0};+ model_make(f, f); g_act = VKT_ACT_SWIGLU; g_limit = 0;+ set_budget(L * E, f, f);+ VktConfig vc = cfg_of(f, f, VKT_ACT_SWIGLU, 0); vc.max_rows = 5;+ setenv("COLI_VK_TIER_RATE", "64", 1); /* every expert resident after a few steps */+ CHECK(vkt_init(&vc, NULL), "partial: the tier did not start");+ Books bk = {0, 0}; int idx[8 * K]; float w[8 * K];+ for (int t = 0; t < 8; t++) for (int l = 0; l < L; l++) { route(8, 0, E, idx, w); step(l, 8, idx, w, &bk); }+ Books one = {0, 0};+ route(8, 0, E, idx, w);+ double r = step(0, 8, idx, w, &one);+ printf(" partial: a step of %llu assignments, the device took %llu (max_rows 5), relative error %.2e\n", one.routed, one.dev, r);+ CHECK(one.dev > 0 && one.dev <= 5, "partial: the device took %llu of a 5-row cap", one.dev);+ CHECK(r < 2e-3, "partial: relative error %.3g", r);+ vkt_shutdown(); model_free();+}++int main(int argc, char **argv) {+ char buf[1024];+ const char *spv = argc > 1 ? argv[1] : coli_vk_shader_path(buf, sizeof buf);+ if (!coli_vk_init(spv)) { printf("FAIL: no Vulkan device (shaders %s)\n", spv); return 1; }+ setenv("COLI_VK_TIER_RESERVE_GB", "0", 1);+ printf("formats:\n"); formats();+ printf("warm:\n"); warm();+ printf("adapt:\n"); adapt();+ printf("partial:\n"); partial();+ ColiVkPoolStats ps; coli_vk_pool_stats(1, &ps);+ CHECK(ps.live == 0, "%d tier ranges still live after every shutdown", ps.live);+ printf(fails ? "FAIL (%d)\n" : "PASS\n", fails);+ return fails != 0;+}diff --git a/c/vk_tier.c b/c/vk_tier.cnew file mode 100644index 00000000..9c269126--- /dev/null+++ b/c/vk_tier.c@@ -0,0 +1,714 @@+/* vk_tier.c -- the Vulkan routed-expert tier shared by the MoE engines. See+ * vk_tier.h for what it does and how an engine calls it.+ *+ * Threads. Everything runs on the engine thread except two things: the uploader+ * thread, which turns staged copies of promoted experts into device tensors, and+ * vkt_put, which a warm start may call from many loader threads. Neither touches+ * a slot: they hand finished tensors back through the `done` list, and the engine+ * thread makes them resident at a quiescent point (no batch in flight), where it+ * also frees evicted experts. So the slot table, the counters and every Vulkan+ * call but the pool's allocation are single-threaded; the lock guards the queue,+ * the done list and the room-made signal only. */+#ifdef COLI_VULKAN+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <pthread.h>+#include <time.h>+#include <errno.h>+#include "vk_tier.h"+#include "backend_vulkan.h"+#include "tier.h"++#define VKT_QCAP 32 /* promotions staged and waiting for the uploader */+#define VKT_CAND 64 /* victim candidates kept per refresh */+#define VKT_DECAY_TOKENS 1024u /* tier.h's decay period, in tokens */++enum { VS_NONE = 0, VS_QUEUED, VS_RESIDENT, VS_EVICT };++typedef struct {+ ColiVkExpert *ex;+ uint32_t heat, last;+ uint8_t state;+} VSlot;+typedef struct { int layer, eid; uint8_t *buf; } VQ;+typedef struct { int layer, eid, ok; ColiVkTensor *g, *u, *d; } VDone;++static struct {+ int on;+ VktConfig c;+ char engine[32];+ int gu_fmt, gu_gs, dn_fmt, dn_gs;+ size_t gu_codes, gu_scales, d_codes, d_scales, stage_bytes; /* one expert as the engine holds it */+ size_t exp_bytes, budget;+ int max_resident, resident, queued, rate, uma;+ VSlot *s;+ uint32_t tick, decay_at;+ int last_layer, promos;+ /* uploader */+ pthread_t th; int th_on, stop;+ pthread_mutex_t mx; pthread_cond_t cv, cv_room;+ VQ q[VKT_QCAP]; int qh, qn;+ VDone *done; int ndone, cdone;+ unsigned long room_gen;+ /* evicted, freed at the next quiescent point */+ int *evict; int nevict, cevict;+ /* the coldest residents, refreshed once per forward */+ int cand[VKT_CAND]; uint64_t cand_score[VKT_CAND]; int ncand;+ /* the step in flight */+ int inflight, S, K;+ int *map; int cmap;+ int *grp; int *touched; /* expert -> group of this step, and the groups' experts */+ ColiVkExpert **bex; int *brows; int *bfirst; int cb;+ const float **bx, **by; int *rowsrc; int cbx;+ int max_dev_rows;+ double t_issued;+ /* accounting */+ unsigned long long routed, served, steps, uploads, upload_bytes, evictions, qfull, rated, refused, failed, warm;+ double dev_ms, cpu_ms, wait_ms;+ unsigned long long rep_routed, rep_served;+} T;++/* "2.31 GiB", "35.0 MiB", "9.5 KiB": the budget lines read on a 24 GB card and a tiny fixture alike */+static const char *human(double b, char *buf, size_t n) {+ if (b >= 1073741824.0) snprintf(buf, n, "%.2f GiB", b / 1073741824.0);+ else if (b >= 1048576.0) snprintf(buf, n, "%.1f MiB", b / 1048576.0);+ else snprintf(buf, n, "%.1f KiB", b / 1024.0);+ return buf;+}+static double vkt_now_ms(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec * 1e3 + t.tv_nsec / 1e6; }+static VSlot *slot(int layer, int eid) { return &T.s[(size_t)layer * T.c.experts + eid]; }++/* ---- source formats ---------------------------------------------------------- */+static int src_per_row(VktSrc k) {+ return k == VKT_SRC_I8_ROW || k == VKT_SRC_I8_AS_I4_ROW || k == VKT_SRC_I4S_PAIRS_ROW || k == VKT_SRC_I4U_PAIRS_ROW;+}+static size_t src_row_bytes(VktSrc k, int I) {+ switch (k) {+ case VKT_SRC_I8_ROW: case VKT_SRC_I8_GS: case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I8_AS_I4_GS:+ case VKT_SRC_FP8_GS: case VKT_SRC_FP8_BLOCK: return (size_t)I;+ case VKT_SRC_I3_G64: return ((size_t)I + 63) / 64 * 24;+ case VKT_SRC_BF16: return (size_t)I * 2;+ case VKT_SRC_F32: return (size_t)I * 4;+ default: return ((size_t)I + 1) / 2; /* the int4 and fp4 kinds */+ }+}+static size_t src_scale_bytes(VktFmt f, int I, int O) {+ if (f.kind == VKT_SRC_BF16 || f.kind == VKT_SRC_F32) return 0;+ if (src_per_row(f.kind)) return (size_t)O * 4;+ if (f.kind == VKT_SRC_I4U_PLANAR64 || f.kind == VKT_SRC_I3_G64) return (size_t)O * (((size_t)I + 63) / 64) * 4;+ size_t ng = ((size_t)I + f.gs - 1) / f.gs;+ if (f.kind == VKT_SRC_MXFP4_E8M0) return (size_t)O * ng;+ if (f.kind == VKT_SRC_FP8_BLOCK) return (((size_t)O + f.gs - 1) / f.gs) * ng * 4;+ return (size_t)O * ng * 4;+}+/* The backend format a source kind lands in; 0 when the combination is not one. */+static int dev_fmt(VktFmt f, int *fmt, int *gs) {+ int g = f.gs;+ switch (f.kind) {+ case VKT_SRC_I8_ROW: *fmt = 1; *gs = 0; return 1;+ case VKT_SRC_I8_GS: *fmt = 13; *gs = g; return g >= 4 && g % 4 == 0;+ case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I4S_PAIRS_ROW: case VKT_SRC_I4U_PAIRS_ROW: *fmt = 2; *gs = 0; return 1;+ case VKT_SRC_I8_AS_I4_GS: case VKT_SRC_I4S_PAIRS_GS: case VKT_SRC_I4U_PAIRS_GS: *fmt = 4; *gs = g; return g >= 8 && g % 8 == 0;+ case VKT_SRC_I4U_PLANAR64: *fmt = 4; *gs = 64; return 1;+ case VKT_SRC_I3_G64: *fmt = 5; *gs = 0; return 1;+ case VKT_SRC_MXFP4_F32: case VKT_SRC_MXFP4_E8M0: *fmt = 7; *gs = g; return g >= 8 && g % 8 == 0;+ case VKT_SRC_FP8_GS: case VKT_SRC_FP8_BLOCK: *fmt = 12; *gs = g; return g >= 4 && g % 4 == 0;+ case VKT_SRC_BF16: *fmt = 11; *gs = 0; return 1;+ case VKT_SRC_F32: *fmt = 10; *gs = 0; return 1;+ default: return 0;+ }+}++/* One matrix [O x I] from its RAM form into a device tensor's mapping: rows at+ * `stride` bytes (already zeroed past the row), scales as the backend reads them.+ * Pure byte work; the values are never requantized. */+static void convert(VktFmt f, int I, int O, const uint8_t *codes, const void *scales,+ uint8_t *rows, size_t stride, float *sc) {+ size_t rb = src_row_bytes(f.kind, I);+ switch (f.kind) {+ case VKT_SRC_I8_AS_I4_ROW: case VKT_SRC_I8_AS_I4_GS: /* int8 in [-8,7] -> v+8 nibble pairs */+ for (int o = 0; o < O; o++) {+ const int8_t *src = (const int8_t *)codes + (size_t)o * I;+ uint8_t *dst = rows + (size_t)o * stride;+ for (int j = 0; j < I / 2; j++)+ dst[j] = (uint8_t)(((src[2 * j] + 8) & 15) | (((src[2 * j + 1] + 8) & 15) << 4));+ if (I & 1) dst[I / 2] = (uint8_t)((src[I - 1] + 8) & 15);+ }+ break;+ case VKT_SRC_I4S_PAIRS_ROW: case VKT_SRC_I4S_PAIRS_GS: /* two's complement nibble -> v+8: xor 8 */+ for (int o = 0; o < O; o++) {+ const uint8_t *src = codes + (size_t)o * rb;+ uint8_t *dst = rows + (size_t)o * stride;+ for (size_t j = 0; j < rb; j++) dst[j] = src[j] ^ 0x88u;+ }+ break;+ case VKT_SRC_I4U_PLANAR64: /* planar blocks (byte k = elements k, k+32) -> pairs (2j, 2j+1) */+ for (int o = 0; o < O; o++) {+ const uint8_t *src = codes + (size_t)o * rb;+ uint8_t *dst = rows + (size_t)o * stride;+ int nb = I / 64;+ for (int b = 0; b < nb; b++) {+ const uint8_t *blk = src + (size_t)b * 32;+ for (int p = 0; p < 32; p++) {+ int e0 = 2 * p, e1 = 2 * p + 1;+ unsigned a = e0 < 32 ? (blk[e0] & 15u) : (unsigned)(blk[e0 - 32] >> 4);+ unsigned c = e1 < 32 ? (blk[e1] & 15u) : (unsigned)(blk[e1 - 32] >> 4);+ dst[(size_t)b * 32 + p] = (uint8_t)(a | (c << 4));+ }+ }+ memcpy(dst + (size_t)nb * 32, src + (size_t)nb * 32, rb - (size_t)nb * 32); /* tail: pairs already */+ }+ break;+ default: /* the same bytes */+ for (int o = 0; o < O; o++) memcpy(rows + (size_t)o * stride, codes + (size_t)o * rb, rb);+ break;+ }+ if (f.kind == VKT_SRC_BF16 || f.kind == VKT_SRC_F32) { sc[0] = 1.0f; return; }+ if (f.kind == VKT_SRC_MXFP4_E8M0) { /* ue8m0 -> f32 2^(s-127), quant.h's mx4_scale */+ size_t n = (size_t)O * (((size_t)I + f.gs - 1) / f.gs);+ const uint8_t *e = scales;+ for (size_t i = 0; i < n; i++) { uint32_t u = (uint32_t)e[i] << 23; memcpy(&sc[i], &u, 4); }+ return;+ }+ if (f.kind == VKT_SRC_FP8_BLOCK) { /* [O/bs][I/bs] blocks -> one row each */+ size_t nb = ((size_t)I + f.gs - 1) / f.gs;+ const float *b = scales;+ for (int o = 0; o < O; o++) memcpy(sc + (size_t)o * nb, b + (size_t)(o / f.gs) * nb, nb * 4);+ return;+ }+ memcpy(sc, scales, src_scale_bytes(f, I, O));+}++/* Upload one expert from its RAM form (any thread). 0 when the pool refused. */+static int upload(const uint8_t *g, const uint8_t *u, const uint8_t *d,+ const void *gs, const void *us, const void *ds, ColiVkTensor *t[3]) {+ const int H = T.c.hidden, F = T.c.inter;+ const uint8_t *codes[3] = {g, u, d}; const void *sc[3] = {gs, us, ds};+ t[0] = t[1] = t[2] = NULL;+ for (int k = 0; k < 3; k++) {+ VktFmt f = k < 2 ? T.c.gate_up : T.c.down;+ int fmt = k < 2 ? T.gu_fmt : T.dn_fmt, fgs = k < 2 ? T.gu_gs : T.dn_gs;+ int I = k < 2 ? H : F, O = k < 2 ? F : H;+ uint8_t *rows; size_t stride; float *scales;+ if (!coli_vk_tier_tensor(&t[k], fmt, I, O, fgs, &rows, &stride, &scales)) {+ for (int j = 0; j < k; j++) { coli_vk_tensor_free(t[j]); t[j] = NULL; }+ return 0;+ }+ convert(f, I, O, codes[k], sc[k], rows, stride, scales);+ }+ return 1;+}++size_t vkt_expert_bytes(int H, int F, VktFmt gu, VktFmt dn) {+ int gf, gg, df, dg;+ if (!dev_fmt(gu, &gf, &gg) || !dev_fmt(dn, &df, &dg)) return 0;+ size_t b = 0;+ for (int k = 0; k < 3; k++) {+ int fmt = k < 2 ? gf : df, gs = k < 2 ? gg : dg, I = k < 2 ? H : F, O = k < 2 ? F : H;+ size_t rows = (coli_vk_tensor_row_bytes(fmt, I) + 3) / 4 * 4 * (size_t)O;+ size_t sc = coli_vk_tensor_scale_count(fmt, I, O, gs) * 4;+ b += (rows + 255) / 256 * 256 + (sc + 255) / 256 * 256 + 512; /* two ranges, aligned, with slack */+ }+ return b;+}++/* ---- the uploader -------------------------------------------------------------- */+static void push_done(int layer, int eid, int ok, ColiVkTensor *t[3]) {+ pthread_mutex_lock(&T.mx);+ if (T.ndone == T.cdone) {+ int nc = T.cdone ? 2 * T.cdone : 64;+ VDone *n = realloc(T.done, (size_t)nc * sizeof(*n));+ if (!n) { /* cannot even record it: the slot stays queued and is never used */+ pthread_mutex_unlock(&T.mx);+ for (int k = 0; k < 3; k++) if (t[k]) coli_vk_tensor_free(t[k]);+ return;+ }+ T.done = n; T.cdone = nc;+ }+ T.done[T.ndone++] = (VDone){layer, eid, ok, t[0], t[1], t[2]};+ pthread_mutex_unlock(&T.mx);+}++static void *uploader(void *arg) {+ (void)arg;+ for (;;) {+ pthread_mutex_lock(&T.mx);+ while (!T.qn && !T.stop) pthread_cond_wait(&T.cv, &T.mx);+ if (T.stop) { pthread_mutex_unlock(&T.mx); return NULL; }+ VQ e = T.q[T.qh]; T.qh = (T.qh + 1) % VKT_QCAP; T.qn--;+ pthread_mutex_unlock(&T.mx);+ const uint8_t *b = e.buf;+ const uint8_t *g = b, *u = g + T.gu_codes, *d = u + T.gu_codes;+ const uint8_t *gs = d + T.d_codes, *us = gs + T.gu_scales, *ds = us + T.gu_scales;+ ColiVkTensor *t[3];+ int ok = 0;+ /* Refused: the pool is at its budget until the engine thread frees the victim+ * this promotion displaced, at its next join. Wait for that, a bounded number+ * of times: a pool that stays full is full for real (fragmentation). */+ for (int tries = 0; tries < 20 && !ok; tries++) {+ ok = upload(g, u, d, gs, us, ds, t);+ if (ok) break;+ pthread_mutex_lock(&T.mx);+ unsigned long gen = T.room_gen;+ struct timespec until; clock_gettime(CLOCK_REALTIME, &until);+ until.tv_nsec += 50 * 1000000L; if (until.tv_nsec >= 1000000000L) { until.tv_sec++; until.tv_nsec -= 1000000000L; }+ while (T.room_gen == gen && !T.stop && pthread_cond_timedwait(&T.cv_room, &T.mx, &until) != ETIMEDOUT) {}+ int stop = T.stop;+ pthread_mutex_unlock(&T.mx);+ if (stop) break;+ }+ free(e.buf);+ if (!ok) t[0] = t[1] = t[2] = NULL;+ push_done(e.layer, e.eid, ok, t);+ }+}++/* ---- quiescent points: nothing in flight ---------------------------------------- */+static void quiesce(void) {+ if (T.inflight) return;+ int freed = 0;+ for (int i = 0; i < T.nevict; i++) {+ VSlot *v = &T.s[T.evict[i]];+ if (v->ex) { coli_vk_xb_expert_free(v->ex); v->ex = NULL; freed = 1; }+ v->state = VS_NONE;+ }+ T.nevict = 0;+ if (freed) {+ pthread_mutex_lock(&T.mx); T.room_gen++; pthread_cond_broadcast(&T.cv_room); pthread_mutex_unlock(&T.mx);+ }+ /* finished uploads become resident */+ pthread_mutex_lock(&T.mx);+ int n = T.ndone; VDone *d = n ? malloc((size_t)n * sizeof(*d)) : NULL;+ if (d) { memcpy(d, T.done, (size_t)n * sizeof(*d)); T.ndone = 0; } else n = 0;+ pthread_mutex_unlock(&T.mx);+ for (int i = 0; i < n; i++) {+ VSlot *v = slot(d[i].layer, d[i].eid);+ T.queued--;+ ColiVkExpert *e = d[i].ok ? coli_vk_xb_expert(d[i].g, d[i].u, d[i].d) : NULL;+ if (e) {+ v->ex = e; v->state = VS_RESIDENT; T.resident++;+ T.uploads++; T.upload_bytes += coli_vk_tensor_bytes(d[i].g) + coli_vk_tensor_bytes(d[i].u) + coli_vk_tensor_bytes(d[i].d);+ } else {+ for (int k = 0; k < 3; k++) { ColiVkTensor *t = k == 0 ? d[i].g : k == 1 ? d[i].u : d[i].d; if (t) coli_vk_tensor_free(t); }+ v->state = VS_NONE; T.failed++;+ /* the budget holds fewer than the count said (fragmentation, or the device+ * ran out first): stop planning past what is there */+ if (T.max_resident > T.resident + T.queued) T.max_resident = T.resident + T.queued;+ }+ }+ free(d);+}++/* ---- placement ------------------------------------------------------------------ */+static uint64_t score(const VSlot *v) { return tier_lfru_score(v->heat, v->last, T.tick); }++/* The coldest residents, ascending, once per forward. */+static void refresh_candidates(void) {+ T.ncand = 0;+ size_t n = (size_t)T.c.layers * T.c.experts;+ for (size_t i = 0; i < n; i++) {+ VSlot *v = &T.s[i];+ if (v->state != VS_RESIDENT) continue;+ uint64_t sc = score(v);+ if (T.ncand == VKT_CAND && sc >= T.cand_score[VKT_CAND - 1]) continue;+ int at = T.ncand < VKT_CAND ? T.ncand++ : VKT_CAND - 1;+ while (at > 0 && T.cand_score[at - 1] > sc) { T.cand[at] = T.cand[at - 1]; T.cand_score[at] = T.cand_score[at - 1]; at--; }+ T.cand[at] = (int)i; T.cand_score[at] = sc;+ }+}+/* The resident a newcomer of score hs displaces, or -1: the coldest candidate still+ * resident, when the newcomer beats it by tier.h's LFRU margin (25% + 4 counts). */+static int pick_victim(uint64_t hs) {+ while (T.ncand) {+ int i = T.cand[0];+ VSlot *v = &T.s[i];+ if (v->state != VS_RESIDENT) { memmove(T.cand, T.cand + 1, (size_t)--T.ncand * sizeof(int));+ memmove(T.cand_score, T.cand_score + 1, (size_t)T.ncand * sizeof(uint64_t)); continue; }+ uint64_t cs = score(v);+ if (hs <= cs + (cs >> 2) + (4u << 8)) return -1;+ memmove(T.cand, T.cand + 1, (size_t)--T.ncand * sizeof(int));+ memmove(T.cand_score, T.cand_score + 1, (size_t)T.ncand * sizeof(uint64_t));+ return i;+ }+ return -1;+}++static void new_forward(int S) {+ T.tick += (uint32_t)(S > 0 ? S : 1);+ T.promos = 0;+ if (T.tick >= T.decay_at) {+ size_t n = (size_t)T.c.layers * T.c.experts;+ for (size_t i = 0; i < n; i++) T.s[i].heat = tier_decay_value(T.s[i].heat);+ T.decay_at = T.tick + VKT_DECAY_TOKENS;+ }+ refresh_candidates();+}++/* The bytes of an expert the CPU just computed pass by: promote it when there is+ * room, or when it is hot enough to displace the coldest resident. The copy is the+ * only cost on this thread; the uploader does the rest. */+void vkt_note(int layer, int eid, const VktExpertSrc *src) {+ if (!T.on || !src || layer < 0 || layer >= T.c.layers || eid < 0 || eid >= T.c.experts) return;+ VSlot *v = slot(layer, eid);+ if (v->state != VS_NONE || !src->g || !src->u || !src->d) return;+ if (T.promos >= T.rate) { T.rated++; return; }+ int victim = -1;+ if (T.resident + T.queued >= T.max_resident) {+ if (!v->heat) return;+ victim = pick_victim(score(v));+ if (victim < 0) return;+ }+ if (T.qn >= VKT_QCAP) { T.qfull++; return; }+ if (T.gu_scales && (!src->gs || !src->us || !src->ds)) return;+ uint8_t *buf = malloc(T.stage_bytes);+ if (!buf) return;+ uint8_t *p = buf;+ memcpy(p, src->g, T.gu_codes); p += T.gu_codes;+ memcpy(p, src->u, T.gu_codes); p += T.gu_codes;+ memcpy(p, src->d, T.d_codes); p += T.d_codes;+ if (T.gu_scales) { memcpy(p, src->gs, T.gu_scales); p += T.gu_scales; memcpy(p, src->us, T.gu_scales); p += T.gu_scales; }+ if (T.d_scales) memcpy(p, src->ds, T.d_scales);+ if (victim >= 0) {+ VSlot *w = &T.s[victim];+ w->state = VS_EVICT; T.resident--; T.evictions++;+ if (T.nevict == T.cevict) {+ int nc = T.cevict ? 2 * T.cevict : 64;+ int *n = realloc(T.evict, (size_t)nc * sizeof(int));+ if (!n) { w->state = VS_RESIDENT; T.resident++; T.evictions--; free(buf); return; }+ T.evict = n; T.cevict = nc;+ }+ T.evict[T.nevict++] = victim;+ quiesce(); /* nothing in flight: free it right away */+ }+ v->state = VS_QUEUED; T.queued++; T.promos++;+ pthread_mutex_lock(&T.mx);+ T.q[(T.qh + T.qn) % VKT_QCAP] = (VQ){layer, eid, buf}; T.qn++;+ pthread_cond_signal(&T.cv);+ pthread_mutex_unlock(&T.mx);+}++/* ---- warm start ------------------------------------------------------------------ */+static const uint32_t *const *g_plan_heat;+static int g_plan_E;+static int plan_cmp(const void *a, const void *b) {+ int x = *(const int *)a, y = *(const int *)b;+ uint32_t hx = g_plan_heat[x / g_plan_E][x % g_plan_E], hy = g_plan_heat[y / g_plan_E][y % g_plan_E];+ return hx < hy ? 1 : hx > hy ? -1 : (x > y) - (x < y);+}+static uint32_t *const *g_heat_hist;+int vkt_plan(int *layers, int *eids, int max) {+ if (!T.on || !g_heat_hist) return 0;+ int L = T.c.layers, E = T.c.experts, n = 0;+ int *order = malloc((size_t)L * E * sizeof(int));+ if (!order) return 0;+ for (int l = 0; l < L; l++)+ if (g_heat_hist[l]) for (int e = 0; e < E; e++) if (g_heat_hist[l][e]) order[n++] = l * E + e;+ g_plan_heat = (const uint32_t *const *)g_heat_hist; g_plan_E = E;+ qsort(order, (size_t)n, sizeof(int), plan_cmp);+ int k = 0;+ for (int i = 0; i < n && k < max && T.resident + T.queued < T.max_resident; i++) {+ VSlot *v = &T.s[order[i]];+ if (v->state != VS_NONE) continue;+ v->state = VS_QUEUED; T.queued++;+ layers[k] = order[i] / E; eids[k] = order[i] % E; k++;+ }+ free(order);+ return k;+}+int vkt_put(int layer, int eid, const VktExpertSrc *src) {+ if (!T.on || layer < 0 || layer >= T.c.layers || eid < 0 || eid >= T.c.experts) return 0;+ ColiVkTensor *t[3] = {NULL, NULL, NULL};+ int ok = src && src->g && src->u && src->d &&+ upload(src->g, src->u, src->d, src->gs, src->us, src->ds, t);+ push_done(layer, eid, ok, t);+ if (ok) __atomic_add_fetch(&T.warm, 1, __ATOMIC_RELAXED);+ return ok;+}+void vkt_put_done(void) { if (T.on) quiesce(); }++/* ---- one layer step -------------------------------------------------------------- */+/* The per-step arrays: assignments (map), groups (bex, brows, bfirst, touched) and+ * rows (bx, by), each family grown together. */+static int grow_to(int need, int *cap) {+ int nc = *cap ? *cap : 64;+ while (nc < need) nc *= 2;+ return nc;+}+static int grow_map(int need) {+ if (need <= T.cmap) return 1;+ int nc = grow_to(need, &T.cmap);+ int *m = realloc(T.map, (size_t)nc * sizeof(int));+ if (!m) return 0;+ T.map = m; T.cmap = nc;+ return 1;+}+static int grow_groups(int need) {+ if (need <= T.cb) return 1;+ int nc = grow_to(need, &T.cb);+ ColiVkExpert **a = realloc(T.bex, (size_t)nc * sizeof(*a)); if (a) T.bex = a;+ int *b = realloc(T.brows, (size_t)nc * sizeof(int)); if (b) T.brows = b;+ int *c = realloc(T.bfirst, (size_t)nc * sizeof(int)); if (c) T.bfirst = c;+ int *d = realloc(T.touched, (size_t)nc * sizeof(int)); if (d) T.touched = d;+ if (!a || !b || !c || !d) return 0;+ T.cb = nc;+ return 1;+}+static int grow_rows(int need) {+ if (need <= T.cbx) return 1;+ int nc = grow_to(need, &T.cbx);+ const float **a = realloc(T.bx, (size_t)nc * sizeof(*a)); if (a) T.bx = a;+ const float **b = realloc(T.by, (size_t)nc * sizeof(*b)); if (b) T.by = b;+ if (!a || !b) return 0;+ T.cbx = nc;+ return 1;+}++int vkt_issue(int layer, const float *x, int S, int K, const int *idx, uint8_t *taken) {+ if (S > 0 && K > 0) memset(taken, 0, (size_t)S * K);+ if (!T.on || T.inflight || layer < 0 || layer >= T.c.layers || S < 1 || K < 1) return 0;+ if (!coli_vk_xb_ready()) {+ if (T.on) { fprintf(stderr, "[VK] tier %s: the device stopped answering, the experts stay on the CPU\n", T.engine); T.on = 0; }+ return 0;+ }+ if (layer <= T.last_layer) new_forward(S);+ T.last_layer = layer;+ quiesce();+ int E = T.c.experts, H = T.c.hidden, n = S * K;+ if (!grow_map(n)) return 0;+ /* heat for every routed expert, groups for the resident ones (first seen first) */+ int ng = 0, total = 0;+ for (int i = 0; i < n; i++) {+ T.map[i] = -1;+ int e = idx[i];+ if (e < 0 || e >= E) continue;+ T.routed++;+ VSlot *v = slot(layer, e);+ if (v->heat < 0xFFFFFFFFu) v->heat++;+ v->last = T.tick;+ if (v->state != VS_RESIDENT) continue;+ int g = T.grp[e];+ if (g < 0) {+ if (total >= T.max_dev_rows || !grow_groups(ng + 1)) continue; /* the batch is as large as it gets */+ g = ng++; T.grp[e] = g; T.touched[g] = e; T.bex[g] = v->ex; T.brows[g] = 0;+ } else if (total >= T.max_dev_rows) continue;+ T.brows[g]++; total++;+ T.map[i] = g; /* group for now, row index below */+ }+ if (!ng) return 0;+ /* rows expert by expert, each expert's in routing order */+ if (!grow_rows(total)) {+ for (int g = 0; g < ng; g++) T.grp[T.touched[g]] = -1;+ for (int i = 0; i < n; i++) T.map[i] = -1;+ return 0;+ }+ int at = 0;+ for (int g = 0; g < ng; g++) { T.bfirst[g] = at; at += T.brows[g]; T.brows[g] = 0; }+ for (int i = 0; i < n; i++) {+ int g = T.map[i];+ if (g < 0) continue;+ int j = T.bfirst[g] + T.brows[g]++;+ T.bx[j] = x + (size_t)(i / K) * H;+ T.map[i] = j;+ }+ for (int g = 0; g < ng; g++) T.grp[T.touched[g]] = -1;+ if (!coli_vk_xb_issue(T.bex, T.brows, ng, T.bx)) {+ for (int i = 0; i < n; i++) T.map[i] = -1;+ return 0;+ }+ for (int i = 0; i < n; i++) if (T.map[i] >= 0) taken[i] = 1;+ T.inflight = 1; T.S = S; T.K = K; T.steps++; T.served += (unsigned long long)total;+ T.t_issued = vkt_now_ms();+ return total;+}++int vkt_join(const float **rows) {+ if (!T.inflight) return 0;+ double t0 = vkt_now_ms(), dms = 0;+ T.cpu_ms += t0 - T.t_issued;+ int ok = coli_vk_xb_join(T.by, &dms);+ T.wait_ms += vkt_now_ms() - t0;+ T.dev_ms += dms;+ T.inflight = 0;+ int n = T.S * T.K;+ for (int i = 0; i < n; i++) rows[i] = ok && T.map[i] >= 0 ? T.by[T.map[i]] : NULL;+ if (!ok) {+ fprintf(stderr, "[VK] tier %s: a batch failed, its experts are recomputed on the CPU and the tier stops\n", T.engine);+ T.on = 0;+ return 0;+ }+ quiesce();+ return 1;+}++int vkt_resident(int layer, int eid) {+ return T.on && layer >= 0 && layer < T.c.layers && eid >= 0 && eid < T.c.experts &&+ slot(layer, eid)->state == VS_RESIDENT;+}+int vkt_ready(void) { return T.on; }++/* ---- budget ---------------------------------------------------------------------- */+static size_t mem_available(void) {+#ifdef __linux__+ FILE *f = fopen("/proc/meminfo", "r");+ if (!f) return 0;+ char line[256]; size_t kb = 0;+ while (fgets(line, sizeof line, f)) if (sscanf(line, "MemAvailable: %zu kB", &kb) == 1) break;+ fclose(f);+ return kb * 1024;+#else+ return 0;+#endif+}+static double env_gb(const char *name, double def) {+ const char *e = getenv(name);+ return e && *e ? atof(e) : def;+}++int vkt_init(const VktConfig *cfg, uint32_t *const *heat) {+ if (T.on || !cfg) return 0;+ const char *on = getenv("COLI_VK_TIER");+ if (on && *on == '0') return 0;+ if (!coli_vk_available()) return 0;+ const char *eng = cfg->engine ? cfg->engine : "engine";+ memset(&T, 0, sizeof T);+ T.c = *cfg;+ snprintf(T.engine, sizeof T.engine, "%s", eng);+ if (T.c.layers < 1 || T.c.experts < 1 || T.c.hidden < 1 || T.c.inter < 1 || T.c.topk < 1) return 0;+ if (!dev_fmt(T.c.gate_up, &T.gu_fmt, &T.gu_gs) || !dev_fmt(T.c.down, &T.dn_fmt, &T.dn_gs)) {+ fprintf(stderr, "[VK] tier %s: expert format %d/%d (gs %d/%d) has no device form, the experts stay on the CPU\n",+ eng, T.c.gate_up.kind, T.c.down.kind, T.c.gate_up.gs, T.c.down.gs);+ return 0;+ }+ int act = T.c.act == VKT_ACT_SITU ? COLI_VK_ACT_SITU : COLI_VK_ACT_SWIGLU;+ if (!coli_vk_xb_init(T.c.hidden, T.c.inter, act, T.c.act_limit, T.c.act_a, T.c.act_b)) {+ fprintf(stderr, "[VK] tier %s: no expert batch on this device (shaders?), the experts stay on the CPU\n", eng);+ return 0;+ }+ int H = T.c.hidden, F = T.c.inter;+ T.gu_codes = src_row_bytes(T.c.gate_up.kind, H) * (size_t)F;+ T.d_codes = src_row_bytes(T.c.down.kind, F) * (size_t)H;+ T.gu_scales = src_scale_bytes(T.c.gate_up, H, F);+ T.d_scales = src_scale_bytes(T.c.down, F, H);+ T.stage_bytes = 2 * T.gu_codes + T.d_codes + 2 * T.gu_scales + T.d_scales;+ T.exp_bytes = vkt_expert_bytes(H, F, T.c.gate_up, T.c.down);++ /* The budget. Discrete: what VK_EXT_memory_budget says is free on the device,+ * minus a reserve (scratch, KV mirrors, driver) and the dense weights still to+ * come. Integrated (device memory is host RAM): a quarter of what the RAM has+ * left after the engine's expert cache has grown to its size and the dense+ * weights are placed, so the tier never takes what the CPU cache needs.+ * COLI_VK_TIER_GB replaces either, within what the device can hold. */+ const double GiB = 1073741824.0;+ double used = 0, bud = 0;+ int have = coli_vk_mem_budget(&used, &bud);+ double heap_free = have ? (bud - used) * 1e9 : (double)coli_vk_device_local_bytes() * 0.8;+ T.uma = coli_vk_device_shares_ram();+ double reserve = env_gb("COLI_VK_TIER_RESERVE_GB", 1.0) * GiB;+ double room = heap_free - reserve - (double)T.c.dense_bytes;+ double want;+ const char *cap = getenv("COLI_VK_TIER_GB");+ if (cap && *cap) want = atof(cap) * GiB;+ else if (T.uma) {+ double avail = (double)mem_available();+ double ram_room = avail > 0 ? avail - (double)T.c.ram_reserve - (double)T.c.dense_bytes - 2.0 * GiB : room;+ want = ram_room / 4;+ } else want = room;+ if (want > room) want = room;+ if (want < 0) want = 0;+ /* whole experts per pool block, so the count matches what the blocks can hold */+ size_t blk = (size_t)256 << 20;+ if ((size_t)want < blk) blk = (size_t)want;+ long long fit = 0;+ if (T.exp_bytes && blk >= T.exp_bytes) {+ size_t full = (size_t)want / blk, rest = (size_t)want - full * blk;+ fit = (long long)full * (long long)(blk / T.exp_bytes) + (long long)(rest / T.exp_bytes);+ }+ long long all = (long long)T.c.layers * T.c.experts;+ if (fit > all) fit = all;+ if (fit < 2) {+ char hb[32], he[32];+ fprintf(stderr, "[VK] tier %s: no room for experts (budget %s, %s each; COLI_VK_TIER_GB sets it), the experts stay on the CPU\n",+ eng, human(want, hb, sizeof hb), human((double)T.exp_bytes, he, sizeof he));+ return 0;+ }+ T.max_resident = (int)fit;+ T.budget = (size_t)want;+ coli_vk_tier_pool_limit(T.budget);+ const char *r = getenv("COLI_VK_TIER_RATE");+ T.rate = r && *r ? atoi(r) : 16;+ if (T.rate < 0) T.rate = 0;+ T.s = calloc((size_t)all, sizeof(VSlot));+ T.grp = malloc((size_t)T.c.experts * sizeof(int));+ if (!T.s || !T.grp) { free(T.s); free(T.grp); return 0; }+ for (int e = 0; e < T.c.experts; e++) T.grp[e] = -1;+ T.max_dev_rows = T.c.max_rows > 0 ? T.c.max_rows : 1 << 20;+ T.last_layer = 1 << 30; T.decay_at = VKT_DECAY_TOKENS;+ /* The history, scaled: the hottest expert of a layer starts at 32 and the rest in+ * proportion, so a few minutes of a new workload can displace it (raw counts+ * from a long history would hold the tier for hours). */+ g_heat_hist = heat;+ if (heat)+ for (int l = 0; l < T.c.layers; l++) {+ if (!heat[l]) continue;+ uint32_t mx = 0;+ for (int e = 0; e < T.c.experts; e++) if (heat[l][e] > mx) mx = heat[l][e];+ if (mx) for (int e = 0; e < T.c.experts; e++)+ if (heat[l][e]) T.s[(size_t)l * T.c.experts + e].heat = 1 + (uint32_t)(31.0 * heat[l][e] / mx);+ }+ if (pthread_mutex_init(&T.mx, NULL) || pthread_cond_init(&T.cv, NULL) || pthread_cond_init(&T.cv_room, NULL)) return 0;+ if (pthread_create(&T.th, NULL, uploader, NULL)) return 0;+ T.th_on = 1;+ T.on = 1;+ char hb[32], he[32];+ fprintf(stderr, "[VK] tier %s: on, %s, budget %s = %d experts of %s (fmt %d", eng,+ coli_vk_device_name(), human((double)T.budget, hb, sizeof hb), T.max_resident,+ human((double)T.exp_bytes, he, sizeof he), T.gu_fmt);+ if (T.gu_gs) fprintf(stderr, " gs %d", T.gu_gs);+ fprintf(stderr, ", down fmt %d", T.dn_fmt);+ if (T.dn_gs) fprintf(stderr, " gs %d", T.dn_gs);+ fprintf(stderr, "), %s, %s queue, up to %d promotions per forward\n",+ T.uma ? (cap && *cap ? "shared RAM (COLI_VK_TIER_GB)" : "shared RAM: a quarter of what the expert cache leaves")+ : "device memory", coli_vk_xb_queue_shared() ? "shared" : "own", T.rate);+ return 1;+}++void vkt_report(const char *scope, unsigned long long ram_hits, unsigned long long disk_loads) {+ if (!T.s) return;+ ColiVkPoolStats ps; coli_vk_pool_stats(1, &ps);+ unsigned long long r = T.routed - T.rep_routed, sv = T.served - T.rep_served;+ T.rep_routed = T.routed; T.rep_served = T.served;+ double hidden = T.dev_ms > T.wait_ms ? T.dev_ms - T.wait_ms : 0;+ char hu[32], hb[32], hl[32];+ fprintf(stderr, "[VK] tier %s %s: device %llu of %llu routed experts (%.1f%%; this %s %llu of %llu) | "+ "CPU RAM hits %llu, disk loads %llu | resident %d (budget %d, %s of %s, %d blocks, frag %.2f) | "+ "uploads %llu (%s, %llu warm), evictions %llu, skipped %llu queue + %llu rate, failed %llu | "+ "device %.1f ms, CPU share %.1f ms, waited %.1f ms (%.0f%% of device time hidden)\n",+ T.engine, scope, T.served, T.routed, T.routed ? 100.0 * T.served / T.routed : 0.0, scope, sv, r,+ ram_hits, disk_loads, T.resident, T.max_resident, human((double)ps.used, hu, sizeof hu),+ human((double)T.budget, hb, sizeof hb), ps.blocks, ps.frag, T.uploads,+ human((double)T.upload_bytes, hl, sizeof hl), T.warm, T.evictions, T.qfull, T.rated, T.failed,+ T.dev_ms, T.cpu_ms, T.wait_ms, T.dev_ms > 0 ? 100.0 * hidden / T.dev_ms : 0.0);+}++void vkt_shutdown(void) {+ if (!T.s) return;+ if (T.th_on) {+ pthread_mutex_lock(&T.mx); T.stop = 1; pthread_cond_broadcast(&T.cv); pthread_cond_broadcast(&T.cv_room); pthread_mutex_unlock(&T.mx);+ pthread_join(T.th, NULL);+ T.th_on = 0;+ }+ if (T.inflight) { const float *dummy[1]; (void)dummy; coli_vk_xb_join(T.by, NULL); T.inflight = 0; }+ for (int i = 0; i < T.qn; i++) free(T.q[(T.qh + i) % VKT_QCAP].buf);+ T.qn = 0;+ if (coli_vk_available()) {+ quiesce();+ size_t n = (size_t)T.c.layers * T.c.experts;+ for (size_t i = 0; i < n; i++) if (T.s[i].ex) { coli_vk_xb_expert_free(T.s[i].ex); T.s[i].ex = NULL; }+ }+ pthread_mutex_destroy(&T.mx); pthread_cond_destroy(&T.cv); pthread_cond_destroy(&T.cv_room);+ free(T.s); free(T.grp); free(T.map); free(T.touched); free(T.bex); free(T.brows); free(T.bfirst);+ free(T.bx); free(T.by); free(T.evict); free(T.done);+ memset(&T, 0, sizeof T);+}+#endif /* COLI_VULKAN */diff --git a/c/vk_tier.h b/c/vk_tier.hnew file mode 100644index 00000000..e7d87f2d--- /dev/null+++ b/c/vk_tier.h@@ -0,0 +1,160 @@+/* vk_tier.h -- the Vulkan routed-expert tier shared by the MoE engines.+ *+ * What it does, for a GPU a Vulkan driver can see (COLI_VULKAN=1):+ * - holds a cache of routed experts in device memory, under a budget taken from+ * VK_EXT_memory_budget (minus a reserve and the dense trunk the engine will+ * place there), capped by COLI_VK_TIER_GB;+ * - fills it at startup from the expert history (.coli_usage, route_trace.h) and+ * keeps adapting while the user chats: an expert the CPU computes is a candidate,+ * admitted when there is room or when it is hotter than the coldest resident by+ * tier.h's LFRU margin, which is then evicted; uploads run on a thread of their+ * own and never hold up a layer;+ * - computes, per layer step, every resident expert the routing picked as ONE+ * asynchronous submit (backend_vulkan.c's expert batch) while the engine computes+ * the rest on the CPU, then hands back one output row per (row, rank) so the+ * engine adds them in its own routing order;+ * - on an integrated GPU, whose device memory is the CPU's RAM, defaults to a+ * budget that leaves the RAM the engine's expert cache will need;+ * - accounts device / CPU hits, uploads, evictions and how much device time the+ * CPU work hid, in one "[VK] tier <engine>" line per run and serve turn.+ * With COLI_VULKAN unset, COLI_VK_TIER=0, a CUDA tier active, or a build without+ * VK=1 (the inline stubs below), nothing here runs and the engine is unchanged.+ *+ * ---- integrating an engine (the phase-2 guide) -------------------------------+ * 1. Describe the experts once, after the model and the dense trunk are loaded and+ * after the engine's history is read (rt_load), before the first token:+ *+ * VktConfig c = {.engine = "glm53", .layers = L, .experts = E,+ * .hidden = H, .inter = F, .topk = K,+ * .gate_up = {VKT_SRC_I4U_PAIRS_GS, 64}, // how RAM holds them+ * .down = {VKT_SRC_I4U_PAIRS_GS, 64},+ * .act = VKT_ACT_SWIGLU, .act_limit = swiglu_limit,+ * .max_rows = prefill_rows * K,+ * .ram_reserve = bytes the RAM expert cache may still grow by,+ * .dense_bytes = bytes of dense weights the engine will put+ * on the device after this call};+ * if (vkt_init(&c, rt_counts_all())) atexit(vkt_shutdown);+ *+ * The heat table is [layers][experts] counts (route_trace.h's rows; NULL rows are+ * layers without experts). Pick .gate_up/.down from the table below; gate and up+ * must share one kind. A layer the tier should not serve (an MTP layer in another+ * format) is simply never passed to vkt_issue: index layers 0..layers-1 only.+ *+ * 2. Warm start (optional, recommended when there is history): the tier plans, the+ * engine reads each planned expert from disk (any number of threads) and hands+ * it over, then publishes on the engine thread:+ *+ * int n = vkt_plan(pl, pe, max);+ * #pragma omp parallel for+ * for (i < n) { read expert (pl[i], pe[i]) into a private buffer;+ * vkt_put(pl[i], pe[i], &src); }+ * vkt_put_done();+ *+ * 3. Every MoE layer step, rows x[S][hidden] routed to idx[S][K] (rank order, -1 =+ * unused slot):+ *+ * uint8_t taken[S*K]; const float *dev[S*K];+ * int n = vkt_issue(layer, x, S, K, idx, taken); // returns at once+ * for every (s,k) with !taken: compute on the CPU into its own buffer,+ * and when the expert's bytes are in RAM: vkt_note(layer, e, &src);+ * shared experts, other CPU work ...+ * if (n && !vkt_join(dev)) compute the taken (s,k) on the CPU too (device lost);+ * for s: for k in rank order: out[s] += w[s][k] * (taken ? dev[s*K+k] : cpu[s][k]);+ *+ * vkt_issue returns how many (s,k) the device took; join only when it took any.+ *+ * Add in rank order, device or not: then the sum's order never depends on which+ * experts happened to be resident (it is the CPU run's order, the values differ+ * by the device's summation order only). Never call the dense Vulkan matmuls+ * between vkt_issue and vkt_join from a parallel region (backend rule), and+ * keep one step in flight: join before the next layer's issue.+ *+ * 4. Report: vkt_report("run" or "turn", ram_hits, disk_loads) where the engine+ * prints its "[VK] <engine>" line; EMAP's tier bits: vkt_resident(l, e) -> 2.+ *+ * Source kinds (VktSrc) and what the device holds -- the conversion runs on the+ * uploader thread, nothing changes in the engine's RAM:+ * VKT_SRC_I8_ROW int8, f32 per row -> fmt 1+ * VKT_SRC_I8_GS int8, f32 per (row, gs) -> fmt 13+ * VKT_SRC_I8_AS_I4_ROW int8 holding int4 values [-8,7], per row -> fmt 2 (repacked: half the bytes)+ * VKT_SRC_I8_AS_I4_GS same, f32 per (row, gs) -> fmt 4+ * VKT_SRC_I4S_PAIRS_ROW signed int4, byte j = elements 2j (low), 2j+1, per row -> fmt 2+ * VKT_SRC_I4S_PAIRS_GS same, per (row, gs) -> fmt 4+ * VKT_SRC_I4U_PAIRS_ROW int4 as v+8, pairs, per row -> fmt 2+ * VKT_SRC_I4U_PAIRS_GS int4 as v+8, pairs, per (row, gs) -> fmt 4 (GLM, glm53)+ * VKT_SRC_I4U_PLANAR64 expert_ffn.h planar blocks of 64, v+8, f32 per 64 -> fmt 4 gs 64+ * VKT_SRC_I3_G64 int3-g64 planes -> fmt 5+ * VKT_SRC_MXFP4_F32 e2m1 pairs, f32 per gs -> fmt 7+ * VKT_SRC_MXFP4_E8M0 e2m1 pairs, ue8m0 byte per gs -> fmt 7 (kimi_k3, mimo, deepseek_v41)+ * VKT_SRC_FP8_GS e4m3, f32 per (row, gs) -> fmt 12+ * VKT_SRC_FP8_BLOCK e4m3, f32 per gs x gs block -> fmt 12 (qwen38 native)+ * VKT_SRC_BF16, VKT_SRC_F32 -> fmt 11, 10+ * Activations (VktConfig.act): VKT_ACT_SWIGLU silu(g)*u, act_limit > 0 clamps the+ * gate from above and up to [-limit, limit] first (GLM-5.3, DeepSeek V4/V4.1);+ * VKT_ACT_SITU a*tanh(g/a)*sigmoid(g) * b*tanh(u/b) with act_a, act_b (Kimi K3). */+#ifndef COLI_VK_TIER_H+#define COLI_VK_TIER_H+#include <stdint.h>+#include <stddef.h>++typedef enum {+ VKT_SRC_NONE = 0,+ VKT_SRC_I8_ROW, VKT_SRC_I8_GS, VKT_SRC_I8_AS_I4_ROW, VKT_SRC_I8_AS_I4_GS,+ VKT_SRC_I4S_PAIRS_ROW, VKT_SRC_I4S_PAIRS_GS, VKT_SRC_I4U_PAIRS_ROW, VKT_SRC_I4U_PAIRS_GS,+ VKT_SRC_I4U_PLANAR64, VKT_SRC_I3_G64, VKT_SRC_MXFP4_F32, VKT_SRC_MXFP4_E8M0,+ VKT_SRC_FP8_GS, VKT_SRC_FP8_BLOCK, VKT_SRC_BF16, VKT_SRC_F32+} VktSrc;+typedef struct { VktSrc kind; int gs; } VktFmt; /* gs: group (block) size, 0 = per row */++#define VKT_ACT_SWIGLU 0+#define VKT_ACT_SITU 1++typedef struct {+ const char *engine; /* names the [VK] tier lines */+ int layers, experts, hidden, inter, topk;+ VktFmt gate_up, down;+ int act; float act_limit, act_a, act_b;+ int max_rows; /* most (row, expert) assignments one vkt_issue carries */+ size_t ram_reserve; /* bytes the engine's RAM caches may still grow by */+ size_t dense_bytes; /* dense weights the engine puts on the device after vkt_init */+} VktConfig;++/* One expert as it sits in RAM: codes and scales of gate, up, down (float scales,+ * or ue8m0 bytes for VKT_SRC_MXFP4_E8M0; NULL for bf16/f32). Read during the call+ * only: the tier copies what it keeps. */+typedef struct {+ const void *g, *u, *d;+ const void *gs, *us, *ds;+} VktExpertSrc;++#ifdef COLI_VULKAN+/* 1 = the tier is on. heat: [layers][experts] routing counts, or NULL. */+int vkt_init(const VktConfig *cfg, uint32_t *const *heat);+int vkt_ready(void);+void vkt_shutdown(void);+int vkt_plan(int *layers, int *eids, int max);+int vkt_put(int layer, int eid, const VktExpertSrc *src); /* any thread */+void vkt_put_done(void);+void vkt_note(int layer, int eid, const VktExpertSrc *src);+int vkt_issue(int layer, const float *x, int S, int K, const int *idx, uint8_t *taken);+int vkt_join(const float **rows);+int vkt_resident(int layer, int eid);+void vkt_report(const char *scope, unsigned long long ram_hits, unsigned long long disk_loads);+/* Sizing helpers for engines: bytes one expert takes on the device in a source format. */+size_t vkt_expert_bytes(int hidden, int inter, VktFmt gate_up, VktFmt down);+#else+static inline int vkt_init(const VktConfig *c, uint32_t *const *h){(void)c;(void)h;return 0;}+static inline int vkt_ready(void){return 0;}+static inline void vkt_shutdown(void){}+static inline int vkt_plan(int *l,int *e,int m){(void)l;(void)e;(void)m;return 0;}+static inline int vkt_put(int l,int e,const VktExpertSrc *s){(void)l;(void)e;(void)s;return 0;}+static inline void vkt_put_done(void){}+static inline void vkt_note(int l,int e,const VktExpertSrc *s){(void)l;(void)e;(void)s;}+static inline int vkt_issue(int l,const float *x,int S,int K,const int *i,uint8_t *t){(void)l;(void)x;(void)S;(void)K;(void)i;(void)t;return 0;}+static inline int vkt_join(const float **r){(void)r;return 0;}+static inline int vkt_resident(int l,int e){(void)l;(void)e;return 0;}+static inline void vkt_report(const char *s,unsigned long long r,unsigned long long d){(void)s;(void)r;(void)d;}+static inline size_t vkt_expert_bytes(int h,int i,VktFmt a,VktFmt b){(void)h;(void)i;(void)a;(void)b;return 0;}+#endif++#endif