Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(vulkan): the GLM fits before the chain's pipelines, coli plan cross-checked

JustVugge206e741afb1
diff --git a/c/glm53_chain.h b/c/glm53_chain.h
index 7baad590..e8b1e647 100644
--- a/c/glm53_chain.h
+++ b/c/glm53_chain.h
@@ -780,13 +780,16 @@ static void g53c_start(GModel *m) {
if (!m->has_io || m->layer_begin != 0 || m->layer_end != c->n_layers) return; /* a segment: no chain, no drop */
int on = coli_vk_chain_decide("glm53", tier_on, COLI_VK_CHAIN_UNMEASURED);
const char *no = NULL;
+ const char *why = on ? g53c_check(m) : NULL;
+ if (why) fprintf(stderr, "[VK] glm53 chain: %s; per-matrix path\n", why);
+ /* the fit before anything of the chain is on the device (its pools' first blocks are
+ * the fit's own granularity, as coli plan counts them) */
+ if (on && !why) g53c_fit_plan(m);
if (on && !(g_g53c_inited = vkc_init())) no = "the chain's pipelines did not come up";
if (on && !no && !(vkc_mla_ready() && vkc_kda_ready() && vkc_mhc_ready()))
no = "the MLA, KDA or mHC shaders are missing";
if (no) fprintf(stderr, "[VK] glm53: %s: the dense chain stays off\n", no);
- const char *why = on && !no ? g53c_check(m) : NULL;
- if (why) fprintf(stderr, "[VK] glm53 chain: %s; per-matrix path\n", why);
- if (on && !no && !why) g53c_fit_plan(m); /* before any upload */
+ if (no) g_g53c_fitted = 0; /* no chain after all: everything as before */
g53_dho_start(m, tier_on, g_g53c_fitted ? g_g53c_fit.n : -1, g_g53c_fitted ? g_g53c_fit.tail : 1);
if (!g_g53c_fitted) return;
if (g_g53c_fit.n > 0 && g53c_setup(m)) g_vk_chain = on;
diff --git a/c/glm_chain.h b/c/glm_chain.h
index 16925de0..8fd59f12 100644
--- a/c/glm_chain.h
+++ b/c/glm_chain.h
@@ -660,12 +660,15 @@ static void glmc_start(Model *m) {
#endif
if (on && !no && (g_kv8 || g_tq)) no = "a quantized KV cache (KV8, KV_TQ) stays on the CPU";
if (on && !no && g_pilot) no = "PILOT prefetch reads the residual on the host";
+ const char *why = on && !no ? glmc_check(m) : NULL;
+ if (why) fprintf(stderr, "[VK] colibri chain: %s; per-matrix path\n", why);
+ /* the fit before anything of the chain is on the device (its pools' first blocks are
+ * the fit's own granularity, as coli plan counts them) */
+ if (on && !no && !why) glmc_fit_plan(m);
if (on && !no && !(g_glmc_inited = vkc_init())) no = "the chain's pipelines did not come up";
if (on && !no && !vkc_mla_ready()) no = "the MLA shaders are missing (chain_mla, chain_hgemv, chain_dsa)";
if (no) fprintf(stderr, "[VK] colibri: %s: the dense chain stays off\n", no);
- const char *why = on && !no ? glmc_check(m) : NULL;
- if (why) fprintf(stderr, "[VK] colibri chain: %s; per-matrix path\n", why);
- if (on && !no && !why) glmc_fit_plan(m); /* before any upload */
+ if (no) g_glmc_fitted = 0; /* no chain after all: everything as before */
glm_dho_start(m);
if (!g_glmc_fitted) return;
if (g_glmc_fit.n > 0 && glmc_setup(m)) g_vk_chain = on;
diff --git a/c/tests/vulkan_partial_glm.sh b/c/tests/vulkan_partial_glm.sh
index c7d542bd..9aa692f3 100644
--- a/c/tests/vulkan_partial_glm.sh
+++ b/c/tests/vulkan_partial_glm.sh
@@ -126,9 +126,35 @@ ptl_mla() {
[ "$want" -ge "$L" ] || ptl_mla_held "$eng" vk.log "$tag"
echo " N = $want of $L, $(ptl_calc matrices "$eng" vk.log | cut -d' ' -f1) B of matrices on the device after setup"
}
+# ptl_mla_plan <tag> <model dir> <log> <env...>: coli plan's prediction (resource_plan.py,
+# vk_chain_fit) for glm53 on the same device and settings: the engine's free bytes,
+# per-layer bytes, fixed bytes and N, from the checkpoint's header and config alone.
+# colibri has no layout (its dense formats are the command line's): nothing to compare.
+ptl_mla_plan() {
+ local tag=$1 fx=$2 log=$3; shift 3
+ $PY - "$fx" "$log" "$@" <<'PY' || fail "$tag: coli plan predicts otherwise"
+import re, sys
+sys.path.insert(0, ".")
+import resource_plan as rp
+fx, log = sys.argv[1], sys.argv[2]
+env = dict(a.split("=", 1) for a in sys.argv[3:] if "=" in a)
+env.update(COLI_VULKAN="1", COLI_VK_CHAIN="1")
+lines = open(log, errors="replace").read().splitlines()
+f = [l for l in lines if l.startswith("[VK] glm53 chain fit: ")][-1]
+num = lambda key: int(re.search(key + r" (\d+) B", f)[1])
+layers = [int(x) for x in re.search(r"layers((?: \d+)*) B", f)[1].split()]
+n = int(re.findall(r"\[VK\] glm53 chain: (\d+) of \d+ layers on the device", "\n".join(lines))[0])
+fit = rp.vk_chain_fit(rp.analyze_model(fx), "glm53", env, {"type": "cpu"})
+ok = (fit["layers"], fit["fixed"], fit["free"], fit["n"]) == (layers, num("fixed"), num("free"), n)
+print(f" coli plan: N = {fit['n']} of {fit['L']} ({'as the engine' if ok else 'the engine: %d' % n}), "
+ f"layers {'= the engine' if fit['layers'] == layers else fit['layers']}, fixed {fit['fixed']} B, free {fit['free']} B")
+sys.exit(0 if ok else 1)
+PY
+}
# ptl_mla_cap <engine> <tag> <k> <env...> -- <argv...>: COLI_VK_DEVICE_CAP_MB aimed at k
# layers from a probe at 256 MiB with the same settings; the reserve holds the device's
-# own scratch beside what the tier takes
+# own scratch beside what the tier takes. glm53: coli plan predicts the same N (PTL_MODEL
+# names the checkpoint)
ptl_mla_cap() {
local eng=$1 tag=$2 k=$3; shift 3
local R=COLI_VK_TIER_RESERVE_GB=0.04 cap pred
@@ -137,6 +163,8 @@ ptl_mla_cap() {
PTL_WANT=$k ptl_mla "$eng" "$tag (cap $cap MiB)" - COLI_VK_DEVICE_CAP_MB="$cap" $R "$@"
pred=$(ptl_calc predict "$eng" vk.log)
[ "$pred" = "$k" ] || { grep -a "^\[VK\] $eng chain" vk.log; fail "$tag: the rule gives $pred from the run's own numbers"; }
+ local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done
+ if [ -n "${PTL_MODEL:-}" ]; then ptl_mla_plan "$tag" "$PTL_MODEL" vk.log COLI_VK_DEVICE_CAP_MB="$cap" $R "${envs[@]}"; fi
}
# ptl_mla_fault <engine> <tag> <k> <env...> -- <argv...>: a staged upload fails inside
# layer k's setup (every layer asked for): the chain keeps the layers before it
@@ -293,7 +321,8 @@ ptl_family_glm() {
PTL_WANT=6 ptl_mla glm53 "partial glm53 the fit, everything fitting" - GLM53_BITS=32 -- $F --ids $ids --greedy 4
ptl_mla_same glm53 "partial glm53 the fit = every layer asked for" "X=1" "COLI_VK_CHAIN_LAYERS=6" GLM53_BITS=32 -- $F --ids $ids --greedy 4
# ---- a small device; an upload failing in the middle of the setup
- for k in 1 4 0; do ptl_mla_cap glm53 "partial glm53 a device for $k layers" $k GLM53_BITS=32 -- $F --ids $ids --greedy 4; done
+ for k in 1 4 0; do PTL_MODEL=glm53_l6s-i4 ptl_mla_cap glm53 "partial glm53 a device for $k layers" $k GLM53_BITS=32 -- $F --ids $ids --greedy 4; done
+ PTL_MODEL=glm53_l6s-i4 ptl_mla_cap glm53 "partial glm53 a device for 3 layers, 4-bit trunk, chunks of 7" 3 GLM53_BITS=4 COLI_VK_CHAIN_ROWS=7 -- $F --ids $ids --greedy 4
for k in 0 2 5; do ptl_mla_fault glm53 "partial glm53 an upload failing in layer $k" $k GLM53_BITS=4 -- $F --ids $ids --greedy 4; done
# ---- the forward paths with N < L
ptl_mla glm53 "partial glm53 prefill chunks of 7, one cache slot" 3 GLM53_BITS=4 GLM53_PREFILL_CHUNK=7 GLM53_EXPERT_GB=0.000001 -- $F --ids $ids --greedy 4
@@ -307,7 +336,7 @@ ptl_family_glm() {
KV=16:4 ptl_mla glm53 "partial glm53 the KV split" 4 GLM53_BITS=32 -- $F --ids $ids --greedy 8
KV=16:4 ptl_mla glm53 "partial glm53 the KV split, chain chunks of 3" 2 GLM53_BITS=32 COLI_VK_CHAIN_ROWS=3 -- $F --ids $ids --greedy 8
FAULT_BACK=7 REBUILD=1 ptl_mla glm53 "partial glm53 device lost, the device layers' KDA state rebuilt" 3 GLM53_BITS=32 GLM53_PREFILL_CHUNK=16 -- $F --ids $ids --greedy 6
- FAULT_BACK=5 REBUILD=1 ptl_mla glm53 "partial glm53 device lost after an image" 2 GLM53_BITS=32 -- $M --greedy 4
+ FAULT_BACK=2 REBUILD=1 ptl_mla glm53 "partial glm53 device lost after an image" 2 GLM53_BITS=32 -- $M --greedy 4
ptl_mla_same glm53 "partial glm53 chunks of 1 = 7" "COLI_VK_CHAIN_ROWS=1" "COLI_VK_CHAIN_ROWS=7" COLI_VK_CHAIN_LAYERS=4 GLM53_BITS=32 -- $F --ids $ids --greedy 4
# ---- the dense weights on the device only
for k in 1 3; do ptl_mla_dho glm53 "partial glm53 device only, $k of 6 layers" $k glm53_l6s-i4 GLM53_BITS=32 -- $F --ids $ids --greedy 4; done