Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
test(vulkan): the cap helper's pools at the new cap, and coli plan beside the engines
JustVuggce741e8ba67a
diff --git a/c/tests/vulkan_partial_inkling-mimo.sh b/c/tests/vulkan_partial_inkling-mimo.shindex 8be15c80..72e87504 100644--- a/c/tests/vulkan_partial_inkling-mimo.sh+++ b/c/tests/vulkan_partial_inkling-mimo.sh@@ -87,6 +87,7 @@ ptl_cap() {local eng=$1 tag=$2 probe=$3 k=$4; shift 4local cap; cap=$(PTL_PROBE_CAP_MB=256 ptl_calc cap "$eng" "$probe" "$k") || { cat "$probe"; fail "$tag: no cap from the probe"; }PTL_CAP=$cap "$@"+ PTL_CAP_LAST=$caplocal p; p=$(ptl_calc predict "$eng" vk.log) || { cat vk.log; fail "$tag: no fit line"; }[ "$p" = "$k" ] || { grep -a "^\[VK\] $eng chain" vk.log; fail "$tag: under COLI_VK_DEVICE_CAP_MB=$cap the rule says $p layers, not $k"; }ptl_check_n "$eng" vk.log "$k" "$tag"@@ -96,6 +97,44 @@ ptl_cap() {# The reserve the capped runs keep for a chunk's scratch and the frames' staging (the tier# takes the rest of the device): 0.04 GiB.PTL_RESERVE=0.04+# ptl_plan <engine> <tag> <fixture> <log> <env...>: coli plan's prediction (resource_plan.py,+# vk_chain_fit) for the same device and settings, from the checkpoint's header and config+# alone: the engine's free bytes, per-layer bytes, fixed bytes, tail and N. The tiny+# inkling's config says model_type inkling_text, which the family registry does not+# take: the plan reads a copy that says inkling (the real checkpoint's), the tensors linked.+ptl_plan() {+ local eng=$1 tag=$2 fx=$3 log=$4; shift 4+ $PY - "$eng" "$fx" "$log" "$@" <<'PY' || fail "$tag: coli plan predicts otherwise"+import json, os, re, shutil, sys, tempfile+sys.path.insert(0, ".")+import resource_plan as rp+eng, fx, log = sys.argv[1], sys.argv[2], sys.argv[3]+env = dict(a.split("=", 1) for a in sys.argv[4:] if "=" in a)+env.update(COLI_VULKAN="1", COLI_VK_CHAIN="1")+lines = open(log, errors="replace").read().splitlines()+f = [l for l in lines if l.startswith(f"[VK] {eng} chain fit: ")][-1]+num = lambda key: int(re.search(key + r" (\d+) B", f)[1])+layers = [int(x) for x in re.search(r"layers((?: \d+)*) B", f)[1].split()]+n = int(re.findall(rf"\[VK\] {eng} chain: (\d+) of \d+ layers on the device", "\n".join(lines))[0])+with tempfile.TemporaryDirectory() as tmp:+ model = fx+ config = json.load(open(os.path.join(fx, "config.json")))+ if config.get("model_type") == "inkling_text":+ model = os.path.join(tmp, "model")+ os.mkdir(model)+ json.dump(dict(config, model_type="inkling"), open(os.path.join(model, "config.json"), "w"))+ for name in os.listdir(fx):+ if name != "config.json":+ os.symlink(os.path.abspath(os.path.join(fx, name)), os.path.join(model, name))+ fit = rp.vk_chain_fit(rp.analyze_model(model), eng, env, {"type": "cpu"})+ok = fit is not None and (fit["layers"], fit["fixed"], fit["free"], fit["n"]) == (layers, num("fixed"), num("free"), n)+if fit is None:+ sys.exit(f" coli plan: no fit for {eng}")+print(f" coli plan: N = {fit['n']} of {fit['L']} ({'as the engine' if ok else 'the engine: %d' % n}), "+ f"layers {'= the engine' if fit['layers'] == layers else fit['layers']}, fixed {fit['fixed']} B, free {fit['free']} B")+sys.exit(0 if ok else 1)+PY+}ptl_inkling() {ptl_ink_fixtures@@ -123,7 +162,22 @@ ptl_inkling() {SNAP=tiny_inkling ./inkling 8 0 $R > cpu.log 2>&1 || truePTL_CAP=256 ink_cap_run; cp vk.log ptl-probe.logptl_check_n inkling ptl-probe.log 8 "partial inkling cap probe"- for k in 1 4 7; do ptl_cap inkling "partial inkling cap for N=$k" ptl-probe.log $k ink_cap_run; echo "OK partial inkling cap for N=$k: tokens = CPU"; done+ for k in 1 4 7; do+ ptl_cap inkling "partial inkling cap for N=$k" ptl-probe.log $k ink_cap_run+ ptl_plan inkling "partial inkling cap for N=$k" tiny_inkling vk.log COLI_VK_DEVICE_CAP_MB=$PTL_CAP_LAST COLI_VK_TIER_RESERVE_GB=$PTL_RESERVE+ echo "OK partial inkling cap for N=$k: tokens = CPU, coli plan's N the engine's"+ done+ # the dense-int4g64 container's forms, and D = 6144, in the plan+ rm -f chain.usage+ env COLI_VK_DEVICE_CAP_MB=256 COLI_VK_TIER_RESERVE_GB=$PTL_RESERVE COLI_VK_CHAIN_LAYERS=0 COLI_USAGE=chain.usage COLI_VULKAN=1 \+ COLI_VK_CHAIN=1 SNAP=tiny_inkling_q ./inkling 8 0 $R > vk.log 2>&1 || true+ ptl_plan inkling "partial inkling plan, the dense-int4g64 container" tiny_inkling_q vk.log COLI_VK_DEVICE_CAP_MB=256 \+ COLI_VK_TIER_RESERVE_GB=$PTL_RESERVE COLI_VK_CHAIN_LAYERS=0+ rm -f chain.usage+ env COLI_VK_DEVICE_CAP_MB=256 COLI_VK_TIER_RESERVE_GB=$PTL_RESERVE COLI_USAGE=chain.usage COLI_VULKAN=1 COLI_VK_CHAIN=1 \+ SNAP=tiny_inkling_wide ./inkling 8 0 $W > vk.log 2>&1 || true+ ptl_plan inkling "partial inkling plan, D=6144" tiny_inkling_wide vk.log COLI_VK_DEVICE_CAP_MB=256 COLI_VK_TIER_RESERVE_GB=$PTL_RESERVE+ echo "OK partial inkling plan: coli plan's numbers the engine's (the container, D = 6144)"# ---- an upload failing inside layer k's setup (COLI_VK_STAGED_FAULT, aimed from a probe's# counts): the layers before it on the device, nothing of layer k; the full chain's setup@@ -255,7 +309,18 @@ ptl_mimo_all() {}PTL_CAP=256 mimo_cap_run; cp vk.log ptl-probe.logptl_check_n mimo ptl-probe.log 6 "partial mimo cap probe"- for k in 1 3 5; do ptl_cap mimo "partial mimo cap for N=$k" ptl-probe.log $k mimo_cap_run; echo "OK partial mimo cap for N=$k: tokens = CPU"; done+ for k in 1 3 5; do+ ptl_cap mimo "partial mimo cap for N=$k" ptl-probe.log $k mimo_cap_run+ ptl_plan mimo "partial mimo cap for N=$k" mimo_tiny vk.log COLI_VK_DEVICE_CAP_MB=$PTL_CAP_LAST COLI_VK_TIER_RESERVE_GB=$PTL_RESERVE MIMO_DENSE_BITS=32+ echo "OK partial mimo cap for N=$k: tokens = CPU, coli plan's N the engine's"+ done+ for bits in 0 8; do # the release's FP8/BF16 and int8 rows, a block of 3, in the plan+ env COLI_VK_DEVICE_CAP_MB=256 COLI_VK_TIER_RESERVE_GB=$PTL_RESERVE COLI_VK_CHAIN_LAYERS=0 MIMO_CHUNK=3 COLI_TEMP=0 COLI_VULKAN=1 \+ COLI_VK_CHAIN=1 MIMO_DENSE_BITS=$bits ./mimo mimo_tiny --ids "$P" --ngen 1 > /dev/null 2> vk.log+ ptl_plan mimo "partial mimo plan bits=$bits" mimo_tiny vk.log COLI_VK_DEVICE_CAP_MB=256 COLI_VK_TIER_RESERVE_GB=$PTL_RESERVE \+ COLI_VK_CHAIN_LAYERS=0 MIMO_CHUNK=3 MIMO_DENSE_BITS=$bits+ done+ echo "OK partial mimo plan: coli plan's numbers the engine's (MIMO_DENSE_BITS 0 and 8, blocks of 3)"# ---- an upload failing inside layer k (the full chain's setup, after the tier; the# device-only one, before it)