Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf: byte-identical CPU/startup optimizations

zh-Processorca17810802ee
diff --git a/c/Makefile b/c/Makefile
index 798d81fc..d832b2f9 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -75,6 +75,15 @@ CFLAGS += -mcpu=$(ARCH)
endif
endif
LDFLAGS = -lm $(OMPL) $(EXTRA_LDFLAGS)
+# LTO is opt-in (LTO=1), not default: cross-TU inlining can change
+# floating-point contraction (FMA formation), and colibri's token-exactness
+# oracles are validated against the default build's optimization regime
+# (#1044/#1024 showed how FMA + accumulator order changes rounding).
+# Enable only after a token-exactness A/B on your own toolchain.
+ifeq ($(LTO),1)
+CFLAGS += -flto
+LDFLAGS += -flto
+endif
EXE =
else ifneq ($(IS_WIN),)
# --- Windows 11 x86-64 (MinGW-w64 / MSYS2) ---
@@ -93,6 +102,10 @@ CFLAGS = -D_FILE_OFFSET_BITS=64 -O3 -march=$(ARCH) -fopenmp -Wall -Wextra -Wno-
# (warns), so link psapi explicitly or the build fails undefined-reference on
# modern GCC (e.g. 16.x under UCRT). Harmless where the pragma also resolves it.
LDFLAGS = -lm -fopenmp -static -lpsapi $(EXTRA_LDFLAGS)
+ifeq ($(LTO),1)
+CFLAGS += -flto
+LDFLAGS += -flto
+endif
EXE = .exe
else
ifneq (,$(PPC64))
@@ -104,6 +117,10 @@ CC = gcc
ARCH ?= native
CFLAGS = -O3 -mcpu=$(ARCH) -fopenmp -pthread -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function $(EXTRA_CFLAGS)
LDFLAGS = -lm -fopenmp -pthread $(EXTRA_LDFLAGS)
+ifeq ($(LTO),1)
+CFLAGS += -flto
+LDFLAGS += -flto
+endif
EXE =
else ifneq (,$(AARCH64))
# --- Linux / *BSD aarch64 (#631) ---
@@ -138,6 +155,10 @@ ARCHFLAG := -mcpu=$(ARCH)
endif
CFLAGS = -O3 $(ARCHFLAG) -fopenmp -pthread -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function $(EXTRA_CFLAGS)
LDFLAGS = -lm -fopenmp -pthread $(EXTRA_LDFLAGS)
+ifeq ($(LTO),1)
+CFLAGS += -flto
+LDFLAGS += -flto
+endif
EXE =
else
# --- Linux / *BSD x86-64 (percorso originale, invariato) ---
@@ -149,6 +170,10 @@ CC = gcc
ARCH ?= native
CFLAGS = -O3 -march=$(ARCH) -fopenmp -pthread -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function $(EXTRA_CFLAGS)
LDFLAGS = -lm -fopenmp -pthread $(EXTRA_LDFLAGS)
+ifeq ($(LTO),1)
+CFLAGS += -flto
+LDFLAGS += -flto
+endif
EXE =
endif
endif
@@ -932,6 +957,24 @@ tests/test_tier$(EXE): tests/test_tier.c tier.h
tests/test_grammar$(EXE): tests/test_grammar.c grammar.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+# RoPE inv_freq precompute must stay byte-identical to the old powf-per-position
+# path; drives the real rope_interleave (includes colibri.c) vs a reference.
+# -ffp-contract=off: the in-test reference is a DIFFERENT loop body than the
+# shipping one, and on FMA targets (x86-64-v3, where -ffp-contract=fast is the
+# default) the compiler contracts the two bodies differently β€” a*cs-b*sn then
+# rounds differently between reference and real function even for pre-change
+# code, a harness artifact rather than an engine regression. Engine-level
+# equivalence under the shipping flags is covered by the differential dump
+# (old vs new rope_interleave are byte-identical); this test guards the
+# source-level formula equivalence and must not depend on contraction luck.
+tests/test_rope_invfreq$(EXE): tests/test_rope_invfreq.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
+ $(CC) $(CFLAGS) -ffp-contract=off $< -o $@ $(LDFLAGS)
+
+# schema->GBNF compile cache (#7): grammar_reset must equal a fresh setup, and
+# the GrDraft.src ownership must not leak/double-free. Includes colibri.c.
+tests/test_grammar_cache$(EXE): tests/test_grammar_cache.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
# Standalone: drives a faithful miniature of moe()'s routing+accumulate and links
# the SAME abl.h the engine links -- no model/weights needed (the ablation-logic gate).
tests/test_ablate$(EXE): tests/test_ablate.c abl.h
diff --git a/c/colibri.c b/c/colibri.c
index 385a87d5..7e8c7196 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -1058,6 +1058,9 @@ typedef struct {
Tok *T;
int on, armed, max;
uint64_t prop, acc;
+ char *src; /* #7: owned copy of the grammar/schema text that built `gram`, used to
+ * skip recompiling when a slot's next request sends the identical schema.
+ * NULL unless a per-request grammar compiled successfully. */
} GrDraft;
/* NO initializer: GrDraft is ~107 KB (Grammar's 1024 static rules + the walker), and any
* initializer β€” even `={.max=24}` β€” moves the whole struct from .bss into .data, writing
@@ -1255,12 +1258,23 @@ static void rope_interleave(float *v, int pos, const Cfg *c){
/* Validate against the fixed buffers (in[256], cache cs/sn[128] -> qk_rope<=256).
* Abort cleanly instead of smashing the stack. (GLM-5.2 qk_rope=64.) (#183) */
if(c->qk_rope > 256){ fprintf(stderr,"qk_rope=%d exceeds rope_interleave buffer (256)\n",c->qk_rope); exit(1); }
- typedef struct { int pos,qk,valid; float theta,cs[128],sn[128]; } RopeCache; /* (#80) */
+ /* inv[j]=powf(theta,-2j/qk_rope) depends only on the model constants theta/qk,
+ * never on pos β€” but the pos-keyed refresh below re-ran `half` powf() calls every
+ * position (32 on GLM qk_rope=64), ~11M powf for a 4k prompt x 90 layers. Cache inv[]
+ * separately, keyed on (qk,theta), and recompute only cs/sn per position. Byte-
+ * identical: `ang = pos*inv[j]` is the same float product against the same inv value
+ * the old code formed inline, so cosf/sinf receive the identical argument. (#80) */
+ typedef struct { int pos,qk,valid,inv_valid; float theta,inv_theta,inv[128],cs[128],sn[128]; } RopeCache;
static _Thread_local RopeCache cache;
float in[256]; memcpy(in,v,c->qk_rope*sizeof(float));
+ if(!cache.inv_valid||cache.qk!=c->qk_rope||cache.inv_theta!=c->theta){
+ for(int j=0;j<half;j++) cache.inv[j]=powf(c->theta,-2.0f*j/c->qk_rope);
+ cache.qk=c->qk_rope; cache.inv_theta=c->theta; cache.inv_valid=1;
+ cache.valid=0; /* inv[] changed -> force cs/sn refresh below */
+ }
if(!cache.valid||cache.pos!=pos||cache.qk!=c->qk_rope||cache.theta!=c->theta){
for(int j=0;j<half;j++){
- float inv=powf(c->theta,-2.0f*j/c->qk_rope),ang=pos*inv;
+ float ang=pos*cache.inv[j];
cache.cs[j]=cosf(ang); cache.sn[j]=sinf(ang);
}
cache.pos=pos; cache.qk=c->qk_rope; cache.theta=c->theta; cache.valid=1;
@@ -3576,7 +3590,16 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
free(dbgQ); free(dbgC); }
#endif
}
- if(!pipe_done) for(int s=0;s<S;s++){
+ /* Every iteration writes a DISTINCT KV row (its own `pos`) and rope_interleave's
+ * cache is _Thread_local, so the positions are independent -> parallelizing is
+ * byte-identical regardless of order. Only the CUDA/VULKAN shadow-shrink writes
+ * touch shared state (m->kv_dev_valid/vk_kv_valid), so the pragma is gated to
+ * CPU-only builds where those blocks compile out; on GPU builds this stays serial. */
+ if(!pipe_done){
+#if !defined(COLI_CUDA) && !defined(COLI_VULKAN)
+ #pragma omp parallel for schedule(static) if(S > 1)
+#endif
+ for(int s=0;s<S;s++){
KVState *ks=kvs?kvs[s]:m->kv;
int pos=positions?positions[s]:pos_base+s;
float *qfull=Q+(int64_t)s*H*qh;
@@ -3597,6 +3620,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
memcpy(Rdst, cs+c->kv_lora, c->qk_rope*sizeof(float));
rope_interleave(Rdst, pos, c); /* k_rot roped, condiviso fra teste */
}
+ }
/* ---- DSA lightning indexer ----
* Layer FULL: k_idx dei token nuovi in cache + selezione top-k per query (riusata
* dai layer SHARED successivi). Selezione attiva solo con contesto > index_topk
@@ -6264,6 +6288,7 @@ static int grammar_setup_text(GrDraft *g, Tok *T, char *txt, const char *label){
/* Release a per-request grammar so the slot can host the next request (keeps max). */
static void grammar_teardown(GrDraft *g){
if(g->gram.n) gr_free(&g->gram);
+ free(g->src); /* #7: release the cached schema text (NULL-safe) */
int max=g->max; memset(g,0,sizeof(*g)); g->max=max;
}
static void grammar_setup(GrDraft *g, Tok *T){
@@ -7465,8 +7490,24 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g
for(int i=0;i<nctx;i++) if(req[i].active && req[i].id==sub.id){
printf("ERROR %llu DUPLICATE_ID\n",sub.id); fflush(stdout); free(gtxt); free(raw); free(line); return 0;
}
- grammar_teardown(&grd[sub.slot]); /* new request owns the slot's grammar state */
- if(gtxt) grammar_setup_text(&grd[sub.slot],T,gtxt,"request"); /* fail-soft; owns gtxt */
+ /* #7: if this slot's last request compiled the *same* schema/grammar text, skip the
+ * schema->GBNF->parse->state-init pipeline (measurable for tool-loop agents that resend
+ * one schema every turn) and just reset the walker -- which is exactly how
+ * grammar_setup_text finishes. Byte-identical to the teardown+recompile path either way. */
+ GrDraft *gd=&grd[sub.slot];
+ if(gtxt && gd->on && gd->src && !strcmp(gd->src,gtxt)){
+ grammar_reset(gd); /* fresh walker state, same compiled grammar */
+ free(gtxt);
+ } else {
+ grammar_teardown(gd); /* new request owns the slot's grammar state */
+ if(gtxt){
+ size_t glen=(size_t)sub.gbytes;
+ char *keep=malloc(glen+1); /* setup_text frees gtxt; keep a copy as the cache key */
+ if(keep){ memcpy(keep,gtxt,glen); keep[glen]=0; }
+ if(grammar_setup_text(gd,T,gtxt,"request")==0) gd->src=keep; /* fail-soft; owns gtxt */
+ else free(keep);
+ }
+ }
ServeCtx *sc=&ctx[sub.slot]; kv_bind(m,&sc->kv);
int *tmp=malloc(maxctx*sizeof(int));
if(!tmp){ fprintf(stderr,"OOM mux_submit tmp\n"); free(raw); free(line); exit(1); }
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 32a14e6d..847f98ee 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -4,6 +4,14 @@
/* Amalgamated deepseek_v4.c β€” GLM-style source; compile with -DCOLI_V4_UNIT_* per object */
/* Umbrella API: deepseek_v4.h (included by units) */
+/* n_routed_experts has no config.json upper bound (only >=1 is checked), so a
+ * fixed-size stack buffer can't be the only path -- an unusually large/hostile
+ * config must still work via heap, just slower. COLI_V4_ROUTE_STACK_EXPERTS
+ * covers every routed-MoE config seen in practice (typical: 64-256) so the
+ * per-token/per-layer routing call in the hot decode path allocates nothing;
+ * anything larger falls back to malloc exactly like before this change. */
+#define COLI_V4_ROUTE_STACK_EXPERTS 512
+
#ifdef COLI_V4_UNIT_ST
/* Shared st.h adapter and V4 tensor materialization helpers. */
#include "deepseek_v4_internal.h"
@@ -37,7 +45,8 @@ int coli_st_index_open(ColiSafetensorsIndex **out, const char *directory,
ColiSafetensorsIndex *index = calloc(1, sizeof(*index));
if (!index)
return set_error(error, error_size, "out of memory opening: %s", directory);
- st_init(index, directory);
+ const char *extra_dirs = getenv("COLI_MODEL_DIRS"); /* SPLIT: shards spread across N drives */
+ st_init_multi(index, directory, (extra_dirs && *extra_dirs) ? extra_dirs : NULL);
if (!index->nfd || !index->n) {
coli_st_index_close(index);
return set_error(error, error_size, "no safetensors tensors in: %s", directory);
@@ -1712,13 +1721,16 @@ int coli_v4_route(float *weights, int *indices, const float *hidden,
if (!weights || !indices || !hidden || !gate || experts < 1 ||
dimension < 1 || topk < 1 || topk > experts)
return -1;
- float *scores = malloc((size_t)experts * sizeof(*scores));
- float *selection = malloc((size_t)experts * sizeof(*selection));
- unsigned char *selected = calloc((size_t)experts, 1);
+ float scores_buf[COLI_V4_ROUTE_STACK_EXPERTS];
+ float selection_buf[COLI_V4_ROUTE_STACK_EXPERTS];
+ unsigned char selected_buf[COLI_V4_ROUTE_STACK_EXPERTS];
+ int on_stack = experts <= COLI_V4_ROUTE_STACK_EXPERTS;
+ float *scores = on_stack ? scores_buf : malloc((size_t)experts * sizeof(*scores));
+ float *selection = on_stack ? selection_buf : malloc((size_t)experts * sizeof(*selection));
+ unsigned char *selected = on_stack ? selected_buf : calloc((size_t)experts, 1);
+ if (on_stack) memset(selected, 0, (size_t)experts);
if (!scores || !selection || !selected) {
- free(scores);
- free(selection);
- free(selected);
+ if (!on_stack) { free(scores); free(selection); free(selected); }
return -1;
}
for (int expert = 0; expert < experts; expert++) {
@@ -1731,9 +1743,7 @@ int coli_v4_route(float *weights, int *indices, const float *hidden,
if (forced_indices) {
for (int rank = 0; rank < topk; rank++) {
if (forced_indices[rank] < 0 || forced_indices[rank] >= experts) {
- free(selected);
- free(selection);
- free(scores);
+ if (!on_stack) { free(selected); free(selection); free(scores); }
return -1;
}
indices[rank] = forced_indices[rank];
@@ -1754,16 +1764,12 @@ int coli_v4_route(float *weights, int *indices, const float *hidden,
for (int rank = 0; rank < topk; rank++)
total += scores[indices[rank]];
if (!(total > 0.0f)) {
- free(selected);
- free(selection);
- free(scores);
+ if (!on_stack) { free(selected); free(selection); free(scores); }
return -1;
}
for (int rank = 0; rank < topk; rank++)
weights[rank] = scores[indices[rank]] / total * route_scale;
- free(selected);
- free(selection);
- free(scores);
+ if (!on_stack) { free(selected); free(selection); free(scores); }
return 0;
}
@@ -7522,15 +7528,17 @@ static int v4_read_expert_record(V4ExpertStoreState *state,
__atomic_fetch_add(&v4_direct_reads, UINT64_C(1), __ATOMIC_RELAXED);
__atomic_fetch_add(&v4_direct_payload_bytes, record->weight_bytes,
__ATOMIC_RELAXED);
- } else {
- if (direct_available)
- __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),
- __ATOMIC_RELAXED);
- if (coli_st_read_at_rep(state->index, record->shard, rep,
+ return coli_st_read_at_rep(state->index, record->shard, rep,
+ record->scale_offset,
+ (size_t)record->scale_bytes, slot->slab);
+ }
+ if (direct_available)
+ __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),
+ __ATOMIC_RELAXED);
+ if (coli_st_read_at_rep(state->index, record->shard, rep,
record->weight_offset,
(size_t)record->weight_bytes,
slot->slab + record->scale_bytes)) return -1;
- }
return coli_st_read_at_rep(state->index, record->shard, rep,
record->scale_offset,
(size_t)record->scale_bytes, slot->slab);
@@ -8273,11 +8281,17 @@ int coli_v4_route_bf16(float *weights, int *indices, const float *hidden,
int topk, float route_scale) {
if (!weights || !indices || !hidden || !gate || experts < 1 ||
dimension < 1 || topk < 1 || topk > experts) return -1;
- float *scores = malloc((size_t)experts * sizeof(*scores));
- float *selection = malloc((size_t)experts * sizeof(*selection));
- unsigned char *selected = calloc((size_t)experts, 1);
+ float scores_buf[COLI_V4_ROUTE_STACK_EXPERTS];
+ float selection_buf[COLI_V4_ROUTE_STACK_EXPERTS];
+ unsigned char selected_buf[COLI_V4_ROUTE_STACK_EXPERTS];
+ int on_stack = experts <= COLI_V4_ROUTE_STACK_EXPERTS;
+ float *scores = on_stack ? scores_buf : malloc((size_t)experts * sizeof(*scores));
+ float *selection = on_stack ? selection_buf : malloc((size_t)experts * sizeof(*selection));
+ unsigned char *selected = on_stack ? selected_buf : calloc((size_t)experts, 1);
+ if (on_stack) memset(selected, 0, (size_t)experts);
if (!scores || !selection || !selected) {
- free(selected); free(selection); free(scores); return -1;
+ if (!on_stack) { free(selected); free(selection); free(scores); }
+ return -1;
}
for (int expert = 0; expert < experts; expert++) {
float sum = 0.0f;
@@ -8290,7 +8304,8 @@ int coli_v4_route_bf16(float *weights, int *indices, const float *hidden,
if (forced_indices) {
for (int rank = 0; rank < topk; rank++) {
if (forced_indices[rank] < 0 || forced_indices[rank] >= experts) {
- free(selected); free(selection); free(scores); return -1;
+ if (!on_stack) { free(selected); free(selection); free(scores); }
+ return -1;
}
indices[rank] = forced_indices[rank];
}
@@ -8309,11 +8324,12 @@ int coli_v4_route_bf16(float *weights, int *indices, const float *hidden,
for (int rank = 0; rank < topk; rank++)
total += scores[indices[rank]];
if (!(total > 0.0f)) {
- free(selected); free(selection); free(scores); return -1;
+ if (!on_stack) { free(selected); free(selection); free(scores); }
+ return -1;
}
for (int rank = 0; rank < topk; rank++)
weights[rank] = scores[indices[rank]] / total * route_scale;
- free(selected); free(selection); free(scores);
+ if (!on_stack) { free(selected); free(selection); free(scores); }
return 0;
}
#endif /* COLI_V4_UNIT_ROUTE_BF16 */
diff --git a/c/download_fp8.py b/c/download_fp8.py
index 12d41c22..fd2707ef 100644
--- a/c/download_fp8.py
+++ b/c/download_fp8.py
@@ -3,12 +3,17 @@ HuggingFace fallback. Parallel shard download, clean progress display.
Usage: python download_fp8.py
python download_fp8.py --parallel 4
python download_fp8.py --source hf (force HuggingFace)
+ python download_fp8.py --dest /data/glm52_fp8 (or set $GLM_DEST)
"""
import os, time, threading, argparse, subprocess
REPO_MS = "ZhipuAI/GLM-5.2-FP8" # ModelScope
REPO_HF = "zai-org/GLM-5.2-FP8" # HuggingFace
-DEST = r"I:\glm52_fp8"
+# Destination for the ~372 GB download. Override with --dest or $GLM_DEST;
+# defaults to ./glm52_fp8 in the current directory (cross-platform β€” the old
+# hardcoded r"I:\glm52_fp8" only worked on a Windows box that happened to have
+# an I: drive).
+DEST = os.environ.get("GLM_DEST", os.path.join(os.getcwd(), "glm52_fp8"))
# ── ANSI colors ──
class C:
@@ -83,12 +88,17 @@ def shard_complete(path, expected_size):
and (expected_size <= 0 or os.path.getsize(path) == expected_size))
def main():
+ global DEST
ap = argparse.ArgumentParser()
ap.add_argument("--parallel", type=int, default=3)
+ ap.add_argument("--dest", default=None,
+ help="download target directory (default: $GLM_DEST or ./glm52_fp8)")
ap.add_argument("--source", choices=["auto", "ms", "hf"], default="auto",
help="auto (try ModelScope first), ms (ModelScope only), hf (HuggingFace only)")
args = ap.parse_args()
+ if args.dest:
+ DEST = args.dest
os.makedirs(DEST, exist_ok=True)
# Determine source and get shard list
diff --git a/c/resource_plan.py b/c/resource_plan.py
index 8e7a324a..fe2f8da8 100644
--- a/c/resource_plan.py
+++ b/c/resource_plan.py
@@ -8,6 +8,7 @@ import shutil
import statistics
import subprocess
import sys
+import threading
from pathlib import Path
from family_registry import expert_contributions, planner_geometry, resolve_model
@@ -16,6 +17,24 @@ from family_registry import expert_contributions, planner_geometry, resolve_mode
GB = 1_000_000_000
EXPERT_RE = re.compile(r"(?:model\.)?layers\.(\d+)\.(?:mlp|ffn)\.experts\.(\d+)\.")
+# analyze_model() scans every shard header + regex-matches ~116k tensor names on the
+# 372 GB model; it reruns on every `coli plan/doctor/tune/run --auto-tier`. Its output
+# is a pure function of each shard's and config.json's (size, mtime), so cache it to a
+# sidecar and self-invalidate on any change. Best-effort: any read/write failure falls
+# straight back to a full recompute (see analyze_model). Sits alongside .coli_usage/.coli_ssd.
+_ANALYSIS_CACHE_NAME = ".coli_analysis.json"
+_ANALYSIS_CACHE_VERSION = 1
+
+
+def _analysis_signature(shards, config_path):
+ parts = [f"v{_ANALYSIS_CACHE_VERSION}"]
+ st = config_path.stat()
+ parts.append(f"config:{st.st_size}:{st.st_mtime_ns}")
+ for shard in shards:
+ s = shard.stat()
+ parts.append(f"{shard.name}:{s.st_size}:{s.st_mtime_ns}")
+ return "|".join(parts)
+
def _tensor_sizes(path):
file_size = path.stat().st_size
@@ -44,6 +63,37 @@ def analyze_model(model):
if not shards:
raise ValueError(f"no safetensors shards: {model}")
+ # Sidecar cache: return the stored analysis if every shard + config is unchanged.
+ # resolve_model() already read config.json, so it exists by this point.
+ signature = _analysis_signature(shards, model / "config.json")
+ cache_path = model / _ANALYSIS_CACHE_NAME
+ try:
+ cached = json.loads(cache_path.read_text(encoding="utf-8"))
+ if isinstance(cached, dict) and cached.get("signature") == signature \
+ and isinstance(cached.get("analysis"), dict):
+ analysis = cached["analysis"]
+ # JSON object keys are always strings: restore the int layer
+ # indices so a cache hit is identical to a fresh scan.
+ by_layer = analysis.get("expert_bytes_by_layer")
+ if isinstance(by_layer, dict):
+ analysis["expert_bytes_by_layer"] = {
+ int(layer): size for layer, size in by_layer.items()}
+ # resolved_family is a live registry object and cannot be JSON'd.
+ # The resolve_model() above recomputed it cheaply (config.json
+ # only); the one scan-derived bit it carries for glm is the
+ # indexer-presence flag, which the cache stores alongside.
+ indexer = analysis.pop("_colibri_indexer_present", None)
+ if indexer is not None and resolved.descriptor.id == "glm":
+ family_cfg = dict(resolved.family_config,
+ _colibri_indexer_present=indexer)
+ resolved = type(resolved)(resolved.descriptor, resolved.model_type,
+ resolved.config, family_cfg,
+ resolved.model_dir)
+ analysis["resolved_family"] = resolved
+ return analysis
+ except (OSError, ValueError):
+ pass # missing/corrupt/unreadable cache -> recompute
+
dense_bytes = 0
expert_groups = {}
tensor_names = set()
@@ -96,7 +146,7 @@ def analyze_model(model):
family_cfg = dict(family_cfg, _colibri_indexer_present=indexer_present)
resolved = type(resolved)(resolved.descriptor, resolved.model_type,
resolved.config, family_cfg, resolved.model_dir)
- return {
+ result = {
"path": str(model),
"shards": len(shards),
"model_bytes": model_bytes,
@@ -111,6 +161,42 @@ def analyze_model(model):
"config": config,
"resolved_family": resolved,
}
+ try: # best-effort write; a read-only model dir must never break planning.
+ # Atomic write (tmp file + os.replace): a concurrent `coli plan` on the
+ # same model dir must never observe a half-written cache -- write_text()
+ # alone can leave a truncated file for a racing reader's json.loads to
+ # choke on, which just falls back to a full recompute (harmless but
+ # defeats the cache for that call). Same directory so the replace stays
+ # on one filesystem (os.replace requires that to be atomic).
+ # resolved_family is a registry object, not JSON-serializable: persist
+ # only the scan-derived indexer flag it carries; the hit path rebuilds
+ # the object from a fresh (cheap) resolve_model().
+ payload = dict(result)
+ resolved_family = payload.pop("resolved_family")
+ family_cfg = getattr(resolved_family, "family_config", None)
+ if isinstance(family_cfg, dict) and "_colibri_indexer_present" in family_cfg:
+ payload["_colibri_indexer_present"] = family_cfg["_colibri_indexer_present"]
+ # The tmp name must be per-THREAD, not per-process: concurrent writers
+ # in one process would otherwise write the same tmp file, and one
+ # thread's os.replace renames it out from under the other. On Windows
+ # the replace can also fail (PermissionError) while a concurrent
+ # reader holds the destination open -- caught below, and the tmp must
+ # not be left behind either way.
+ tmp_path = cache_path.with_suffix(
+ f".{os.getpid()}.{threading.get_ident()}.tmp")
+ try:
+ tmp_path.write_text(
+ json.dumps({"signature": signature, "analysis": payload}),
+ encoding="utf-8")
+ os.replace(tmp_path, cache_path)
+ except (OSError, TypeError, ValueError):
+ try:
+ tmp_path.unlink(missing_ok=True)
+ except OSError:
+ pass
+ except (OSError, TypeError, ValueError):
+ pass
+ return result
def memory_available():
diff --git a/c/tests/test_analysis_cache.py b/c/tests/test_analysis_cache.py
new file mode 100644
index 00000000..87fbe5b5
--- /dev/null
+++ b/c/tests/test_analysis_cache.py
@@ -0,0 +1,167 @@
+"""analyze_model() sidecar cache (#3).
+
+analyze_model() scans every shard header and regex-matches ~116k tensor names on
+the 372 GB model, and reran on every `coli plan/doctor/tune/run --auto-tier`. It
+now caches its result to <model>/.coli_analysis.json, keyed on each shard's and
+config.json's (size, mtime_ns), and self-invalidates. This test pins the three
+properties the cache must have, WITHOUT a real model: a hit returns the identical
+result and does not rescan, a shard change forces a recompute, and a corrupt
+cache falls back to a full recompute instead of raising.
+"""
+import importlib.util
+import json
+import os
+import struct
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+_HERE = Path(__file__).resolve().parent
+_RP_PATH = _HERE.parent / "resource_plan.py"
+_spec = importlib.util.spec_from_file_location("resource_plan", _RP_PATH)
+resource_plan = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(resource_plan)
+
+
+def _write_shard(path, tensors):
+ """Minimal valid safetensors file: u64 header length, JSON header, zero data."""
+ header, off = {}, 0
+ for name, nbytes in tensors.items():
+ header[name] = {"dtype": "F16", "shape": [nbytes // 2],
+ "data_offsets": [off, off + nbytes]}
+ off += nbytes
+ blob = json.dumps(header).encode()
+ with open(path, "wb") as f:
+ f.write(struct.pack("<Q", len(blob)))
+ f.write(blob)
+ f.write(b"\0" * off)
+
+
+class AnalysisCacheTest(unittest.TestCase):
+ def setUp(self):
+ # resolve(): analyze_model() resolves the model path internally, so
+ # comparisons against paths derived from self.dir must use the same
+ # canonical form β€” macOS maps /var -> /private/var, Windows expands
+ # 8.3 short names (ADMINI~1), and an unresolved mkdtemp path fails
+ # the string equality below on both.
+ self.dir = Path(tempfile.mkdtemp(prefix="coli_analysis_")).resolve()
+ (self.dir / "config.json").write_text(
+ '{"model_type": "glm_moe_dsa", "num_hidden_layers": 4}')
+ _write_shard(self.dir / "model-00001.safetensors", {
+ "model.layers.0.mlp.experts.0.gate_proj.weight": 2048,
+ "model.layers.0.mlp.experts.1.gate_proj.weight": 2048,
+ "model.embed_tokens.weight": 1024,
+ })
+
+ def test_hit_is_identical_and_skips_rescan(self):
+ first = resource_plan.analyze_model(self.dir)
+ self.assertTrue((self.dir / resource_plan._ANALYSIS_CACHE_NAME).is_file())
+ # Make a recompute explode, proving the second call is served from cache.
+ orig = resource_plan._tensor_sizes
+ resource_plan._tensor_sizes = lambda p: (_ for _ in ()).throw(
+ AssertionError("recomputed on a cache hit"))
+ try:
+ second = resource_plan.analyze_model(self.dir)
+ finally:
+ resource_plan._tensor_sizes = orig
+ self.assertEqual(first, second)
+
+ def test_shard_change_invalidates(self):
+ first = resource_plan.analyze_model(self.dir)
+ time.sleep(0.01)
+ os.utime(self.dir / "model-00001.safetensors", None) # bump mtime
+ again = resource_plan.analyze_model(self.dir)
+ self.assertEqual(again, first) # same content -> same result, but recomputed
+
+ def test_corrupt_cache_falls_back(self):
+ first = resource_plan.analyze_model(self.dir)
+ (self.dir / resource_plan._ANALYSIS_CACHE_NAME).write_text("{ not valid json")
+ recovered = resource_plan.analyze_model(self.dir)
+ self.assertEqual(recovered, first)
+
+ def test_write_is_atomic_no_tmp_left_behind(self):
+ # A concurrent `coli plan` on the same model dir must never see a
+ # half-written cache file; the write goes through a pid-suffixed tmp
+ # file + os.replace. After a normal write, no .tmp file should remain
+ # and the cache file itself must be valid, complete JSON.
+ resource_plan.analyze_model(self.dir)
+ tmp_leftovers = list(self.dir.glob("*.tmp"))
+ self.assertEqual(tmp_leftovers, [])
+ cache_path = self.dir / resource_plan._ANALYSIS_CACHE_NAME
+ self.assertTrue(cache_path.is_file())
+ parsed = json.loads(cache_path.read_text(encoding="utf-8"))
+ self.assertIn("signature", parsed)
+ self.assertIn("analysis", parsed)
+
+ def test_write_path_uses_tmp_file_and_replace_not_direct_write(self):
+ # Structural check on the ACTUAL write code in analyze_model (not a
+ # copy of it): the final rename must be os.replace(tmp, target), and
+ # nothing may write cache_path directly. A naive cache_path.write_text
+ # regression would still pass the black-box tests above (the window
+ # is too small to reliably race in a unit test), so this pins the
+ # implementation shape instead of relying on timing.
+ from pathlib import Path as _Path
+
+ cache_path = self.dir / resource_plan._ANALYSIS_CACHE_NAME
+ write_calls = []
+ replace_calls = []
+ orig_write_text = _Path.write_text
+ orig_replace = os.replace
+
+ def spy_write_text(self_path, *a, **kw):
+ write_calls.append(self_path)
+ return orig_write_text(self_path, *a, **kw)
+
+ def spy_replace(src, dst, *a, **kw):
+ replace_calls.append((_Path(src), _Path(dst)))
+ return orig_replace(src, dst, *a, **kw)
+
+ _Path.write_text = spy_write_text
+ os.replace = spy_replace
+ try:
+ resource_plan.analyze_model(self.dir)
+ finally:
+ _Path.write_text = orig_write_text
+ os.replace = orig_replace
+
+ self.assertEqual(len(replace_calls), 1)
+ replaced_src, replaced_dst = replace_calls[0]
+ self.assertEqual(replaced_dst, cache_path)
+ self.assertNotEqual(replaced_src, cache_path)
+ # Every write_text call in this run must target the tmp path that was
+ # then replaced onto cache_path, never cache_path directly.
+ self.assertIn(replaced_src, write_calls)
+ self.assertNotIn(cache_path, write_calls)
+
+ def test_concurrent_writers_produce_a_valid_final_cache(self):
+ # Multiple threads racing analyze_model() on the same model dir (each
+ # forcing a fresh recompute+write via a bumped mtime) must never leave
+ # a corrupt or half-written cache file, and no stray tmp files.
+ import threading
+
+ cache_path = self.dir / resource_plan._ANALYSIS_CACHE_NAME
+ shard_path = self.dir / "model-00001.safetensors"
+ errors = []
+
+ def worker(n):
+ try:
+ for _ in range(15):
+ os.utime(shard_path, None) # force a real recompute+write each time
+ resource_plan.analyze_model(self.dir)
+ except Exception as exc: # pragma: no cover
+ errors.append(exc)
+
+ threads = [threading.Thread(target=worker, args=(i,)) for i in range(6)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+
+ self.assertEqual(errors, [])
+ self.assertEqual(list(self.dir.glob("*.tmp")), [])
+ json.loads(cache_path.read_text(encoding="utf-8")) # must not raise
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_grammar_cache.c b/c/tests/test_grammar_cache.c
new file mode 100644
index 00000000..b6c86d1b
--- /dev/null
+++ b/c/tests/test_grammar_cache.c
@@ -0,0 +1,88 @@
+/* schema->GBNF compile-cache correctness (#7).
+ *
+ * mux_submit now skips re-compiling a slot's grammar when the next request sends
+ * the IDENTICAL schema/grammar text: it keeps an owned copy in GrDraft.src and,
+ * on a match, does grammar_reset() instead of grammar_teardown()+setup. That is
+ * only correct if two claims hold, which this test pins on the REAL functions
+ * (via the include-colibri.c pattern):
+ *
+ * (1) EQUIVALENCE: grammar_reset() on an already-compiled grammar yields the
+ * exact same walker behaviour as a fresh grammar_setup_text() of the same
+ * text -- same forced spans, same accept/reject at every byte.
+ * (2) OWNERSHIP: the GrDraft.src lifecycle (set on compile, freed on teardown)
+ * never leaks or double-frees across the miss/hit/fail/no-grammar paths.
+ */
+#define main coli_glm_main_unused
+#include "../colibri.c"
+#undef main
+
+#define CHECK(cond) do { if(!(cond)){ \
+ fprintf(stderr,"%s:%d: check failed: %s\n",__FILE__,__LINE__,#cond); return 1; } } while(0)
+
+/* feed a byte string through a walker; return bytes accepted before a reject */
+static int feed(GrState *S, const char *s){
+ int n=0; while(s[n]){ if(gr_accept(S,(unsigned char)s[n])!=1) break; n++; } return n;
+}
+
+/* Compare fresh-setup vs reset behaviour of the SAME compiled grammar. */
+static int check_equivalence(const char *gbnf, const char *sample){
+ GrDraft a; memset(&a,0,sizeof a); a.max=24;
+ char *t1=malloc(strlen(gbnf)+1); strcpy(t1,gbnf);
+ CHECK(grammar_setup_text(&a,NULL,t1,"A")==0); /* frees t1 */
+ char fa[512]; int fa_n=gr_forced(&a.st,fa,sizeof fa);
+ int acc_a=feed(&a.st,sample);
+
+ /* reset the SAME draft and re-run: must match itself exactly */
+ grammar_reset(&a);
+ char fr[512]; int fr_n=gr_forced(&a.st,fr,sizeof fr);
+ int acc_r=feed(&a.st,sample);
+ CHECK(fr_n==fa_n && !memcmp(fr,fa,(size_t)fa_n) && acc_r==acc_a);
+
+ /* a completely fresh draft from the same text: must also match */
+ GrDraft b; memset(&b,0,sizeof b); b.max=24;
+ char *t2=malloc(strlen(gbnf)+1); strcpy(t2,gbnf);
+ CHECK(grammar_setup_text(&b,NULL,t2,"B")==0);
+ char fb[512]; int fb_n=gr_forced(&b.st,fb,sizeof fb);
+ int acc_b=feed(&b.st,sample);
+ CHECK(fb_n==fa_n && !memcmp(fb,fa,(size_t)fa_n) && acc_b==acc_a);
+
+ grammar_teardown(&a); grammar_teardown(&b);
+ return 0;
+}
+
+int main(void){
+ /* (1) equivalence across representative grammars */
+ CHECK(check_equivalence("root ::= \"{\\\"id\\\":\"", "{\"id\":")==0);
+ CHECK(check_equivalence("root ::= \"a\" (\"b\" | \"c\") \"d\"", "abd")==0);
+ CHECK(check_equivalence("root ::= \"[\" [0-9]+ \"]\"", "[123]")==0);
+
+ /* (2) ownership: replicate the exact mux_submit miss/hit/fail/none flow,
+ * setting GrDraft.src exactly as mux_submit does, and exercising teardown. */
+ GrDraft gd; memset(&gd,0,sizeof gd); gd.max=24;
+ const char *A="root ::= \"x\"";
+ for(int turn=0; turn<3; turn++){
+ /* HIT path if src matches, else MISS (compile) path */
+ if(gd.on && gd.src && !strcmp(gd.src,A)){
+ grammar_reset(&gd); /* cache hit: reset only */
+ } else {
+ grammar_teardown(&gd);
+ char *keep=malloc(strlen(A)+1); strcpy(keep,A);
+ char *txt=malloc(strlen(A)+1); strcpy(txt,A);
+ if(grammar_setup_text(&gd,NULL,txt,"req")==0) gd.src=keep; else free(keep);
+ }
+ CHECK(gd.on && gd.src && !strcmp(gd.src,A));
+ }
+ /* switch to a different grammar: teardown must free the old src */
+ grammar_teardown(&gd);
+ CHECK(gd.src==NULL);
+ /* a grammar that fails to compile: left recursion -> gr_state_init not alive,
+ * grammar_setup_text returns -1. src must stay NULL, no crash. */
+ const char *BAD="root ::= root \"a\"";
+ char *bad=malloc(strlen(BAD)+1); strcpy(bad,BAD);
+ int rc=grammar_setup_text(&gd,NULL,bad,"bad");
+ CHECK(rc!=0 && gd.src==NULL);
+ grammar_teardown(&gd); /* NULL-safe free(src) */
+
+ printf("test_grammar_cache: reset==fresh-setup equivalence + src ownership OK\n");
+ return 0;
+}
diff --git a/c/tests/test_rope_invfreq.c b/c/tests/test_rope_invfreq.c
new file mode 100644
index 00000000..e08ffd42
--- /dev/null
+++ b/c/tests/test_rope_invfreq.c
@@ -0,0 +1,58 @@
+/* RoPE inv_freq precompute (byte-identical guard).
+ *
+ * rope_interleave() used to recompute inv=powf(theta,-2j/qk_rope) for every
+ * position inside its thread-local cache refresh; it now precomputes inv[j]
+ * once (keyed on theta/qk) and forms ang=pos*inv[j] per position. The rotation
+ * is byte-sensitive: cs/sn feed q/k dot products, so any bit drift changes the
+ * sampled token. This test drives the REAL rope_interleave (via the test_topp.c
+ * include-colibri.c pattern) and compares its rotated output, BIT-FOR-BIT,
+ * against an independent reference that reimplements the OLD powf-per-position
+ * formula. Sweeps the qk_rope sizes and rope_theta values GLM/Kimi actually use.
+ */
+#define main coli_glm_main_unused
+#include "../colibri.c"
+#undef main
+
+#include <math.h>
+#include <stdint.h>
+
+#define CHECK(cond) do { if(!(cond)){ \
+ fprintf(stderr,"%s:%d: check failed: %s\n",__FILE__,__LINE__,#cond); return 1; } } while(0)
+
+static int biteq(float a, float b){ uint32_t x,y; memcpy(&x,&a,4); memcpy(&y,&b,4); return x==y; }
+
+/* Reference: the pre-optimisation rotation, powf recomputed every call. */
+static void rope_ref(float *v, int pos, int qk, float theta){
+ int half=qk/2; float in[256]; memcpy(in,v,qk*sizeof(float));
+ for(int j=0;j<half;j++){
+ float inv=powf(theta,-2.0f*j/qk), ang=pos*inv;
+ float cs=cosf(ang), sn=sinf(ang);
+ float a=in[2*j], b=in[2*j+1];
+ v[j] = a*cs - b*sn;
+ v[half+j] = b*cs + a*sn;
+ }
+}
+
+int main(void){
+ const int qks[] = {64,128,96,256};
+ const float ths[] = {10000.f, 1000000.f, 500000.f};
+ long checked=0;
+ for(int qi=0; qi<4; qi++) for(int ti=0; ti<3; ti++){
+ int qk=qks[qi]; float th=ths[ti];
+ Cfg c; memset(&c,0,sizeof c); c.qk_rope=qk; c.theta=th;
+ for(int pos=0; pos<200000; pos += (pos<4096 ? 1 : 37)){
+ float base[256], vr[256], vn[256];
+ /* deterministic, varied input (no rand: reproducible across platforms) */
+ for(int i=0;i<qk;i++) base[i] = sinf((float)(i*7 + pos)) * 1.5f - 0.3f;
+ memcpy(vr,base,qk*sizeof(float)); memcpy(vn,base,qk*sizeof(float));
+ rope_ref(vr,pos,qk,th);
+ rope_interleave(vn,pos,&c); /* the REAL shipping function */
+ for(int i=0;i<qk;i++){
+ checked++;
+ CHECK(biteq(vr[i],vn[i]));
+ }
+ }
+ }
+ printf("test_rope_invfreq: %ld rotated floats bit-identical to reference\n",checked);
+ return 0;
+}