Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
perf: byte-identical CPU/startup optimizations
zh-Processorca17810802ee
diff --git a/c/Makefile b/c/Makefileindex 798d81fc..d832b2f9 100644--- a/c/Makefile+++ b/c/Makefile@@ -75,6 +75,15 @@ CFLAGS += -mcpu=$(ARCH)endifendifLDFLAGS = -lm $(OMPL) $(EXTRA_LDFLAGS)+# LTO is opt-in (LTO=1), not default: cross-TU inlining can change+# floating-point contraction (FMA formation), and colibri's token-exactness+# oracles are validated against the default build's optimization regime+# (#1044/#1024 showed how FMA + accumulator order changes rounding).+# Enable only after a token-exactness A/B on your own toolchain.+ifeq ($(LTO),1)+CFLAGS += -flto+LDFLAGS += -flto+endifEXE =else ifneq ($(IS_WIN),)# --- Windows 11 x86-64 (MinGW-w64 / MSYS2) ---@@ -93,6 +102,10 @@ CFLAGS = -D_FILE_OFFSET_BITS=64 -O3 -march=$(ARCH) -fopenmp -Wall -Wextra -Wno-# (warns), so link psapi explicitly or the build fails undefined-reference on# modern GCC (e.g. 16.x under UCRT). Harmless where the pragma also resolves it.LDFLAGS = -lm -fopenmp -static -lpsapi $(EXTRA_LDFLAGS)+ifeq ($(LTO),1)+CFLAGS += -flto+LDFLAGS += -flto+endifEXE = .exeelseifneq (,$(PPC64))@@ -104,6 +117,10 @@ CC = gccARCH ?= nativeCFLAGS = -O3 -mcpu=$(ARCH) -fopenmp -pthread -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function $(EXTRA_CFLAGS)LDFLAGS = -lm -fopenmp -pthread $(EXTRA_LDFLAGS)+ifeq ($(LTO),1)+CFLAGS += -flto+LDFLAGS += -flto+endifEXE =else ifneq (,$(AARCH64))# --- Linux / *BSD aarch64 (#631) ---@@ -138,6 +155,10 @@ ARCHFLAG := -mcpu=$(ARCH)endifCFLAGS = -O3 $(ARCHFLAG) -fopenmp -pthread -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function $(EXTRA_CFLAGS)LDFLAGS = -lm -fopenmp -pthread $(EXTRA_LDFLAGS)+ifeq ($(LTO),1)+CFLAGS += -flto+LDFLAGS += -flto+endifEXE =else# --- Linux / *BSD x86-64 (percorso originale, invariato) ---@@ -149,6 +170,10 @@ CC = gccARCH ?= nativeCFLAGS = -O3 -march=$(ARCH) -fopenmp -pthread -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function $(EXTRA_CFLAGS)LDFLAGS = -lm -fopenmp -pthread $(EXTRA_LDFLAGS)+ifeq ($(LTO),1)+CFLAGS += -flto+LDFLAGS += -flto+endifEXE =endifendif@@ -932,6 +957,24 @@ tests/test_tier$(EXE): tests/test_tier.c tier.htests/test_grammar$(EXE): tests/test_grammar.c grammar.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# RoPE inv_freq precompute must stay byte-identical to the old powf-per-position+# path; drives the real rope_interleave (includes colibri.c) vs a reference.+# -ffp-contract=off: the in-test reference is a DIFFERENT loop body than the+# shipping one, and on FMA targets (x86-64-v3, where -ffp-contract=fast is the+# default) the compiler contracts the two bodies differently β a*cs-b*sn then+# rounds differently between reference and real function even for pre-change+# code, a harness artifact rather than an engine regression. Engine-level+# equivalence under the shipping flags is covered by the differential dump+# (old vs new rope_interleave are byte-identical); this test guards the+# source-level formula equivalence and must not depend on contraction luck.+tests/test_rope_invfreq$(EXE): tests/test_rope_invfreq.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h+ $(CC) $(CFLAGS) -ffp-contract=off $< -o $@ $(LDFLAGS)++# schema->GBNF compile cache (#7): grammar_reset must equal a fresh setup, and+# the GrDraft.src ownership must not leak/double-free. Includes colibri.c.+tests/test_grammar_cache$(EXE): tests/test_grammar_cache.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# Standalone: drives a faithful miniature of moe()'s routing+accumulate and links# the SAME abl.h the engine links -- no model/weights needed (the ablation-logic gate).tests/test_ablate$(EXE): tests/test_ablate.c abl.hdiff --git a/c/colibri.c b/c/colibri.cindex 385a87d5..7e8c7196 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -1058,6 +1058,9 @@ typedef struct {Tok *T;int on, armed, max;uint64_t prop, acc;+ char *src; /* #7: owned copy of the grammar/schema text that built `gram`, used to+ * skip recompiling when a slot's next request sends the identical schema.+ * NULL unless a per-request grammar compiled successfully. */} GrDraft;/* NO initializer: GrDraft is ~107 KB (Grammar's 1024 static rules + the walker), and any* initializer β even `={.max=24}` β moves the whole struct from .bss into .data, writing@@ -1255,12 +1258,23 @@ static void rope_interleave(float *v, int pos, const Cfg *c){/* Validate against the fixed buffers (in[256], cache cs/sn[128] -> qk_rope<=256).* Abort cleanly instead of smashing the stack. (GLM-5.2 qk_rope=64.) (#183) */if(c->qk_rope > 256){ fprintf(stderr,"qk_rope=%d exceeds rope_interleave buffer (256)\n",c->qk_rope); exit(1); }- typedef struct { int pos,qk,valid; float theta,cs[128],sn[128]; } RopeCache; /* (#80) */+ /* inv[j]=powf(theta,-2j/qk_rope) depends only on the model constants theta/qk,+ * never on pos β but the pos-keyed refresh below re-ran `half` powf() calls every+ * position (32 on GLM qk_rope=64), ~11M powf for a 4k prompt x 90 layers. Cache inv[]+ * separately, keyed on (qk,theta), and recompute only cs/sn per position. Byte-+ * identical: `ang = pos*inv[j]` is the same float product against the same inv value+ * the old code formed inline, so cosf/sinf receive the identical argument. (#80) */+ typedef struct { int pos,qk,valid,inv_valid; float theta,inv_theta,inv[128],cs[128],sn[128]; } RopeCache;static _Thread_local RopeCache cache;float in[256]; memcpy(in,v,c->qk_rope*sizeof(float));+ if(!cache.inv_valid||cache.qk!=c->qk_rope||cache.inv_theta!=c->theta){+ for(int j=0;j<half;j++) cache.inv[j]=powf(c->theta,-2.0f*j/c->qk_rope);+ cache.qk=c->qk_rope; cache.inv_theta=c->theta; cache.inv_valid=1;+ cache.valid=0; /* inv[] changed -> force cs/sn refresh below */+ }if(!cache.valid||cache.pos!=pos||cache.qk!=c->qk_rope||cache.theta!=c->theta){for(int j=0;j<half;j++){- float inv=powf(c->theta,-2.0f*j/c->qk_rope),ang=pos*inv;+ float ang=pos*cache.inv[j];cache.cs[j]=cosf(ang); cache.sn[j]=sinf(ang);}cache.pos=pos; cache.qk=c->qk_rope; cache.theta=c->theta; cache.valid=1;@@ -3576,7 +3590,16 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pfree(dbgQ); free(dbgC); }#endif}- if(!pipe_done) for(int s=0;s<S;s++){+ /* Every iteration writes a DISTINCT KV row (its own `pos`) and rope_interleave's+ * cache is _Thread_local, so the positions are independent -> parallelizing is+ * byte-identical regardless of order. Only the CUDA/VULKAN shadow-shrink writes+ * touch shared state (m->kv_dev_valid/vk_kv_valid), so the pragma is gated to+ * CPU-only builds where those blocks compile out; on GPU builds this stays serial. */+ if(!pipe_done){+#if !defined(COLI_CUDA) && !defined(COLI_VULKAN)+ #pragma omp parallel for schedule(static) if(S > 1)+#endif+ for(int s=0;s<S;s++){KVState *ks=kvs?kvs[s]:m->kv;int pos=positions?positions[s]:pos_base+s;float *qfull=Q+(int64_t)s*H*qh;@@ -3597,6 +3620,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pmemcpy(Rdst, cs+c->kv_lora, c->qk_rope*sizeof(float));rope_interleave(Rdst, pos, c); /* k_rot roped, condiviso fra teste */}+ }/* ---- DSA lightning indexer ----* Layer FULL: k_idx dei token nuovi in cache + selezione top-k per query (riusata* dai layer SHARED successivi). Selezione attiva solo con contesto > index_topk@@ -6264,6 +6288,7 @@ static int grammar_setup_text(GrDraft *g, Tok *T, char *txt, const char *label){/* Release a per-request grammar so the slot can host the next request (keeps max). */static void grammar_teardown(GrDraft *g){if(g->gram.n) gr_free(&g->gram);+ free(g->src); /* #7: release the cached schema text (NULL-safe) */int max=g->max; memset(g,0,sizeof(*g)); g->max=max;}static void grammar_setup(GrDraft *g, Tok *T){@@ -7465,8 +7490,24 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *gfor(int i=0;i<nctx;i++) if(req[i].active && req[i].id==sub.id){printf("ERROR %llu DUPLICATE_ID\n",sub.id); fflush(stdout); free(gtxt); free(raw); free(line); return 0;}- grammar_teardown(&grd[sub.slot]); /* new request owns the slot's grammar state */- if(gtxt) grammar_setup_text(&grd[sub.slot],T,gtxt,"request"); /* fail-soft; owns gtxt */+ /* #7: if this slot's last request compiled the *same* schema/grammar text, skip the+ * schema->GBNF->parse->state-init pipeline (measurable for tool-loop agents that resend+ * one schema every turn) and just reset the walker -- which is exactly how+ * grammar_setup_text finishes. Byte-identical to the teardown+recompile path either way. */+ GrDraft *gd=&grd[sub.slot];+ if(gtxt && gd->on && gd->src && !strcmp(gd->src,gtxt)){+ grammar_reset(gd); /* fresh walker state, same compiled grammar */+ free(gtxt);+ } else {+ grammar_teardown(gd); /* new request owns the slot's grammar state */+ if(gtxt){+ size_t glen=(size_t)sub.gbytes;+ char *keep=malloc(glen+1); /* setup_text frees gtxt; keep a copy as the cache key */+ if(keep){ memcpy(keep,gtxt,glen); keep[glen]=0; }+ if(grammar_setup_text(gd,T,gtxt,"request")==0) gd->src=keep; /* fail-soft; owns gtxt */+ else free(keep);+ }+ }ServeCtx *sc=&ctx[sub.slot]; kv_bind(m,&sc->kv);int *tmp=malloc(maxctx*sizeof(int));if(!tmp){ fprintf(stderr,"OOM mux_submit tmp\n"); free(raw); free(line); exit(1); }diff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 32a14e6d..847f98ee 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -4,6 +4,14 @@/* Amalgamated deepseek_v4.c β GLM-style source; compile with -DCOLI_V4_UNIT_* per object *//* Umbrella API: deepseek_v4.h (included by units) */+/* n_routed_experts has no config.json upper bound (only >=1 is checked), so a+ * fixed-size stack buffer can't be the only path -- an unusually large/hostile+ * config must still work via heap, just slower. COLI_V4_ROUTE_STACK_EXPERTS+ * covers every routed-MoE config seen in practice (typical: 64-256) so the+ * per-token/per-layer routing call in the hot decode path allocates nothing;+ * anything larger falls back to malloc exactly like before this change. */+#define COLI_V4_ROUTE_STACK_EXPERTS 512+#ifdef COLI_V4_UNIT_ST/* Shared st.h adapter and V4 tensor materialization helpers. */#include "deepseek_v4_internal.h"@@ -37,7 +45,8 @@ int coli_st_index_open(ColiSafetensorsIndex **out, const char *directory,ColiSafetensorsIndex *index = calloc(1, sizeof(*index));if (!index)return set_error(error, error_size, "out of memory opening: %s", directory);- st_init(index, directory);+ const char *extra_dirs = getenv("COLI_MODEL_DIRS"); /* SPLIT: shards spread across N drives */+ st_init_multi(index, directory, (extra_dirs && *extra_dirs) ? extra_dirs : NULL);if (!index->nfd || !index->n) {coli_st_index_close(index);return set_error(error, error_size, "no safetensors tensors in: %s", directory);@@ -1712,13 +1721,16 @@ int coli_v4_route(float *weights, int *indices, const float *hidden,if (!weights || !indices || !hidden || !gate || experts < 1 ||dimension < 1 || topk < 1 || topk > experts)return -1;- float *scores = malloc((size_t)experts * sizeof(*scores));- float *selection = malloc((size_t)experts * sizeof(*selection));- unsigned char *selected = calloc((size_t)experts, 1);+ float scores_buf[COLI_V4_ROUTE_STACK_EXPERTS];+ float selection_buf[COLI_V4_ROUTE_STACK_EXPERTS];+ unsigned char selected_buf[COLI_V4_ROUTE_STACK_EXPERTS];+ int on_stack = experts <= COLI_V4_ROUTE_STACK_EXPERTS;+ float *scores = on_stack ? scores_buf : malloc((size_t)experts * sizeof(*scores));+ float *selection = on_stack ? selection_buf : malloc((size_t)experts * sizeof(*selection));+ unsigned char *selected = on_stack ? selected_buf : calloc((size_t)experts, 1);+ if (on_stack) memset(selected, 0, (size_t)experts);if (!scores || !selection || !selected) {- free(scores);- free(selection);- free(selected);+ if (!on_stack) { free(scores); free(selection); free(selected); }return -1;}for (int expert = 0; expert < experts; expert++) {@@ -1731,9 +1743,7 @@ int coli_v4_route(float *weights, int *indices, const float *hidden,if (forced_indices) {for (int rank = 0; rank < topk; rank++) {if (forced_indices[rank] < 0 || forced_indices[rank] >= experts) {- free(selected);- free(selection);- free(scores);+ if (!on_stack) { free(selected); free(selection); free(scores); }return -1;}indices[rank] = forced_indices[rank];@@ -1754,16 +1764,12 @@ int coli_v4_route(float *weights, int *indices, const float *hidden,for (int rank = 0; rank < topk; rank++)total += scores[indices[rank]];if (!(total > 0.0f)) {- free(selected);- free(selection);- free(scores);+ if (!on_stack) { free(selected); free(selection); free(scores); }return -1;}for (int rank = 0; rank < topk; rank++)weights[rank] = scores[indices[rank]] / total * route_scale;- free(selected);- free(selection);- free(scores);+ if (!on_stack) { free(selected); free(selection); free(scores); }return 0;}@@ -7522,15 +7528,17 @@ static int v4_read_expert_record(V4ExpertStoreState *state,__atomic_fetch_add(&v4_direct_reads, UINT64_C(1), __ATOMIC_RELAXED);__atomic_fetch_add(&v4_direct_payload_bytes, record->weight_bytes,__ATOMIC_RELAXED);- } else {- if (direct_available)- __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),- __ATOMIC_RELAXED);- if (coli_st_read_at_rep(state->index, record->shard, rep,+ return coli_st_read_at_rep(state->index, record->shard, rep,+ record->scale_offset,+ (size_t)record->scale_bytes, slot->slab);+ }+ if (direct_available)+ __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),+ __ATOMIC_RELAXED);+ if (coli_st_read_at_rep(state->index, record->shard, rep,record->weight_offset,(size_t)record->weight_bytes,slot->slab + record->scale_bytes)) return -1;- }return coli_st_read_at_rep(state->index, record->shard, rep,record->scale_offset,(size_t)record->scale_bytes, slot->slab);@@ -8273,11 +8281,17 @@ int coli_v4_route_bf16(float *weights, int *indices, const float *hidden,int topk, float route_scale) {if (!weights || !indices || !hidden || !gate || experts < 1 ||dimension < 1 || topk < 1 || topk > experts) return -1;- float *scores = malloc((size_t)experts * sizeof(*scores));- float *selection = malloc((size_t)experts * sizeof(*selection));- unsigned char *selected = calloc((size_t)experts, 1);+ float scores_buf[COLI_V4_ROUTE_STACK_EXPERTS];+ float selection_buf[COLI_V4_ROUTE_STACK_EXPERTS];+ unsigned char selected_buf[COLI_V4_ROUTE_STACK_EXPERTS];+ int on_stack = experts <= COLI_V4_ROUTE_STACK_EXPERTS;+ float *scores = on_stack ? scores_buf : malloc((size_t)experts * sizeof(*scores));+ float *selection = on_stack ? selection_buf : malloc((size_t)experts * sizeof(*selection));+ unsigned char *selected = on_stack ? selected_buf : calloc((size_t)experts, 1);+ if (on_stack) memset(selected, 0, (size_t)experts);if (!scores || !selection || !selected) {- free(selected); free(selection); free(scores); return -1;+ if (!on_stack) { free(selected); free(selection); free(scores); }+ return -1;}for (int expert = 0; expert < experts; expert++) {float sum = 0.0f;@@ -8290,7 +8304,8 @@ int coli_v4_route_bf16(float *weights, int *indices, const float *hidden,if (forced_indices) {for (int rank = 0; rank < topk; rank++) {if (forced_indices[rank] < 0 || forced_indices[rank] >= experts) {- free(selected); free(selection); free(scores); return -1;+ if (!on_stack) { free(selected); free(selection); free(scores); }+ return -1;}indices[rank] = forced_indices[rank];}@@ -8309,11 +8324,12 @@ int coli_v4_route_bf16(float *weights, int *indices, const float *hidden,for (int rank = 0; rank < topk; rank++)total += scores[indices[rank]];if (!(total > 0.0f)) {- free(selected); free(selection); free(scores); return -1;+ if (!on_stack) { free(selected); free(selection); free(scores); }+ return -1;}for (int rank = 0; rank < topk; rank++)weights[rank] = scores[indices[rank]] / total * route_scale;- free(selected); free(selection); free(scores);+ if (!on_stack) { free(selected); free(selection); free(scores); }return 0;}#endif /* COLI_V4_UNIT_ROUTE_BF16 */diff --git a/c/download_fp8.py b/c/download_fp8.pyindex 12d41c22..fd2707ef 100644--- a/c/download_fp8.py+++ b/c/download_fp8.py@@ -3,12 +3,17 @@ HuggingFace fallback. Parallel shard download, clean progress display.Usage: python download_fp8.pypython download_fp8.py --parallel 4python download_fp8.py --source hf (force HuggingFace)+ python download_fp8.py --dest /data/glm52_fp8 (or set $GLM_DEST)"""import os, time, threading, argparse, subprocessREPO_MS = "ZhipuAI/GLM-5.2-FP8" # ModelScopeREPO_HF = "zai-org/GLM-5.2-FP8" # HuggingFace-DEST = r"I:\glm52_fp8"+# Destination for the ~372 GB download. Override with --dest or $GLM_DEST;+# defaults to ./glm52_fp8 in the current directory (cross-platform β the old+# hardcoded r"I:\glm52_fp8" only worked on a Windows box that happened to have+# an I: drive).+DEST = os.environ.get("GLM_DEST", os.path.join(os.getcwd(), "glm52_fp8"))# ββ ANSI colors ββclass C:@@ -83,12 +88,17 @@ def shard_complete(path, expected_size):and (expected_size <= 0 or os.path.getsize(path) == expected_size))def main():+ global DESTap = argparse.ArgumentParser()ap.add_argument("--parallel", type=int, default=3)+ ap.add_argument("--dest", default=None,+ help="download target directory (default: $GLM_DEST or ./glm52_fp8)")ap.add_argument("--source", choices=["auto", "ms", "hf"], default="auto",help="auto (try ModelScope first), ms (ModelScope only), hf (HuggingFace only)")args = ap.parse_args()+ if args.dest:+ DEST = args.destos.makedirs(DEST, exist_ok=True)# Determine source and get shard listdiff --git a/c/resource_plan.py b/c/resource_plan.pyindex 8e7a324a..fe2f8da8 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -8,6 +8,7 @@ import shutilimport statisticsimport subprocessimport sys+import threadingfrom pathlib import Pathfrom family_registry import expert_contributions, planner_geometry, resolve_model@@ -16,6 +17,24 @@ from family_registry import expert_contributions, planner_geometry, resolve_modeGB = 1_000_000_000EXPERT_RE = re.compile(r"(?:model\.)?layers\.(\d+)\.(?:mlp|ffn)\.experts\.(\d+)\.")+# analyze_model() scans every shard header + regex-matches ~116k tensor names on the+# 372 GB model; it reruns on every `coli plan/doctor/tune/run --auto-tier`. Its output+# is a pure function of each shard's and config.json's (size, mtime), so cache it to a+# sidecar and self-invalidate on any change. Best-effort: any read/write failure falls+# straight back to a full recompute (see analyze_model). Sits alongside .coli_usage/.coli_ssd.+_ANALYSIS_CACHE_NAME = ".coli_analysis.json"+_ANALYSIS_CACHE_VERSION = 1+++def _analysis_signature(shards, config_path):+ parts = [f"v{_ANALYSIS_CACHE_VERSION}"]+ st = config_path.stat()+ parts.append(f"config:{st.st_size}:{st.st_mtime_ns}")+ for shard in shards:+ s = shard.stat()+ parts.append(f"{shard.name}:{s.st_size}:{s.st_mtime_ns}")+ return "|".join(parts)+def _tensor_sizes(path):file_size = path.stat().st_size@@ -44,6 +63,37 @@ def analyze_model(model):if not shards:raise ValueError(f"no safetensors shards: {model}")+ # Sidecar cache: return the stored analysis if every shard + config is unchanged.+ # resolve_model() already read config.json, so it exists by this point.+ signature = _analysis_signature(shards, model / "config.json")+ cache_path = model / _ANALYSIS_CACHE_NAME+ try:+ cached = json.loads(cache_path.read_text(encoding="utf-8"))+ if isinstance(cached, dict) and cached.get("signature") == signature \+ and isinstance(cached.get("analysis"), dict):+ analysis = cached["analysis"]+ # JSON object keys are always strings: restore the int layer+ # indices so a cache hit is identical to a fresh scan.+ by_layer = analysis.get("expert_bytes_by_layer")+ if isinstance(by_layer, dict):+ analysis["expert_bytes_by_layer"] = {+ int(layer): size for layer, size in by_layer.items()}+ # resolved_family is a live registry object and cannot be JSON'd.+ # The resolve_model() above recomputed it cheaply (config.json+ # only); the one scan-derived bit it carries for glm is the+ # indexer-presence flag, which the cache stores alongside.+ indexer = analysis.pop("_colibri_indexer_present", None)+ if indexer is not None and resolved.descriptor.id == "glm":+ family_cfg = dict(resolved.family_config,+ _colibri_indexer_present=indexer)+ resolved = type(resolved)(resolved.descriptor, resolved.model_type,+ resolved.config, family_cfg,+ resolved.model_dir)+ analysis["resolved_family"] = resolved+ return analysis+ except (OSError, ValueError):+ pass # missing/corrupt/unreadable cache -> recompute+dense_bytes = 0expert_groups = {}tensor_names = set()@@ -96,7 +146,7 @@ def analyze_model(model):family_cfg = dict(family_cfg, _colibri_indexer_present=indexer_present)resolved = type(resolved)(resolved.descriptor, resolved.model_type,resolved.config, family_cfg, resolved.model_dir)- return {+ result = {"path": str(model),"shards": len(shards),"model_bytes": model_bytes,@@ -111,6 +161,42 @@ def analyze_model(model):"config": config,"resolved_family": resolved,}+ try: # best-effort write; a read-only model dir must never break planning.+ # Atomic write (tmp file + os.replace): a concurrent `coli plan` on the+ # same model dir must never observe a half-written cache -- write_text()+ # alone can leave a truncated file for a racing reader's json.loads to+ # choke on, which just falls back to a full recompute (harmless but+ # defeats the cache for that call). Same directory so the replace stays+ # on one filesystem (os.replace requires that to be atomic).+ # resolved_family is a registry object, not JSON-serializable: persist+ # only the scan-derived indexer flag it carries; the hit path rebuilds+ # the object from a fresh (cheap) resolve_model().+ payload = dict(result)+ resolved_family = payload.pop("resolved_family")+ family_cfg = getattr(resolved_family, "family_config", None)+ if isinstance(family_cfg, dict) and "_colibri_indexer_present" in family_cfg:+ payload["_colibri_indexer_present"] = family_cfg["_colibri_indexer_present"]+ # The tmp name must be per-THREAD, not per-process: concurrent writers+ # in one process would otherwise write the same tmp file, and one+ # thread's os.replace renames it out from under the other. On Windows+ # the replace can also fail (PermissionError) while a concurrent+ # reader holds the destination open -- caught below, and the tmp must+ # not be left behind either way.+ tmp_path = cache_path.with_suffix(+ f".{os.getpid()}.{threading.get_ident()}.tmp")+ try:+ tmp_path.write_text(+ json.dumps({"signature": signature, "analysis": payload}),+ encoding="utf-8")+ os.replace(tmp_path, cache_path)+ except (OSError, TypeError, ValueError):+ try:+ tmp_path.unlink(missing_ok=True)+ except OSError:+ pass+ except (OSError, TypeError, ValueError):+ pass+ return resultdef memory_available():diff --git a/c/tests/test_analysis_cache.py b/c/tests/test_analysis_cache.pynew file mode 100644index 00000000..87fbe5b5--- /dev/null+++ b/c/tests/test_analysis_cache.py@@ -0,0 +1,167 @@+"""analyze_model() sidecar cache (#3).++analyze_model() scans every shard header and regex-matches ~116k tensor names on+the 372 GB model, and reran on every `coli plan/doctor/tune/run --auto-tier`. It+now caches its result to <model>/.coli_analysis.json, keyed on each shard's and+config.json's (size, mtime_ns), and self-invalidates. This test pins the three+properties the cache must have, WITHOUT a real model: a hit returns the identical+result and does not rescan, a shard change forces a recompute, and a corrupt+cache falls back to a full recompute instead of raising.+"""+import importlib.util+import json+import os+import struct+import tempfile+import time+import unittest+from pathlib import Path++_HERE = Path(__file__).resolve().parent+_RP_PATH = _HERE.parent / "resource_plan.py"+_spec = importlib.util.spec_from_file_location("resource_plan", _RP_PATH)+resource_plan = importlib.util.module_from_spec(_spec)+_spec.loader.exec_module(resource_plan)+++def _write_shard(path, tensors):+ """Minimal valid safetensors file: u64 header length, JSON header, zero data."""+ header, off = {}, 0+ for name, nbytes in tensors.items():+ header[name] = {"dtype": "F16", "shape": [nbytes // 2],+ "data_offsets": [off, off + nbytes]}+ off += nbytes+ blob = json.dumps(header).encode()+ with open(path, "wb") as f:+ f.write(struct.pack("<Q", len(blob)))+ f.write(blob)+ f.write(b"\0" * off)+++class AnalysisCacheTest(unittest.TestCase):+ def setUp(self):+ # resolve(): analyze_model() resolves the model path internally, so+ # comparisons against paths derived from self.dir must use the same+ # canonical form β macOS maps /var -> /private/var, Windows expands+ # 8.3 short names (ADMINI~1), and an unresolved mkdtemp path fails+ # the string equality below on both.+ self.dir = Path(tempfile.mkdtemp(prefix="coli_analysis_")).resolve()+ (self.dir / "config.json").write_text(+ '{"model_type": "glm_moe_dsa", "num_hidden_layers": 4}')+ _write_shard(self.dir / "model-00001.safetensors", {+ "model.layers.0.mlp.experts.0.gate_proj.weight": 2048,+ "model.layers.0.mlp.experts.1.gate_proj.weight": 2048,+ "model.embed_tokens.weight": 1024,+ })++ def test_hit_is_identical_and_skips_rescan(self):+ first = resource_plan.analyze_model(self.dir)+ self.assertTrue((self.dir / resource_plan._ANALYSIS_CACHE_NAME).is_file())+ # Make a recompute explode, proving the second call is served from cache.+ orig = resource_plan._tensor_sizes+ resource_plan._tensor_sizes = lambda p: (_ for _ in ()).throw(+ AssertionError("recomputed on a cache hit"))+ try:+ second = resource_plan.analyze_model(self.dir)+ finally:+ resource_plan._tensor_sizes = orig+ self.assertEqual(first, second)++ def test_shard_change_invalidates(self):+ first = resource_plan.analyze_model(self.dir)+ time.sleep(0.01)+ os.utime(self.dir / "model-00001.safetensors", None) # bump mtime+ again = resource_plan.analyze_model(self.dir)+ self.assertEqual(again, first) # same content -> same result, but recomputed++ def test_corrupt_cache_falls_back(self):+ first = resource_plan.analyze_model(self.dir)+ (self.dir / resource_plan._ANALYSIS_CACHE_NAME).write_text("{ not valid json")+ recovered = resource_plan.analyze_model(self.dir)+ self.assertEqual(recovered, first)++ def test_write_is_atomic_no_tmp_left_behind(self):+ # A concurrent `coli plan` on the same model dir must never see a+ # half-written cache file; the write goes through a pid-suffixed tmp+ # file + os.replace. After a normal write, no .tmp file should remain+ # and the cache file itself must be valid, complete JSON.+ resource_plan.analyze_model(self.dir)+ tmp_leftovers = list(self.dir.glob("*.tmp"))+ self.assertEqual(tmp_leftovers, [])+ cache_path = self.dir / resource_plan._ANALYSIS_CACHE_NAME+ self.assertTrue(cache_path.is_file())+ parsed = json.loads(cache_path.read_text(encoding="utf-8"))+ self.assertIn("signature", parsed)+ self.assertIn("analysis", parsed)++ def test_write_path_uses_tmp_file_and_replace_not_direct_write(self):+ # Structural check on the ACTUAL write code in analyze_model (not a+ # copy of it): the final rename must be os.replace(tmp, target), and+ # nothing may write cache_path directly. A naive cache_path.write_text+ # regression would still pass the black-box tests above (the window+ # is too small to reliably race in a unit test), so this pins the+ # implementation shape instead of relying on timing.+ from pathlib import Path as _Path++ cache_path = self.dir / resource_plan._ANALYSIS_CACHE_NAME+ write_calls = []+ replace_calls = []+ orig_write_text = _Path.write_text+ orig_replace = os.replace++ def spy_write_text(self_path, *a, **kw):+ write_calls.append(self_path)+ return orig_write_text(self_path, *a, **kw)++ def spy_replace(src, dst, *a, **kw):+ replace_calls.append((_Path(src), _Path(dst)))+ return orig_replace(src, dst, *a, **kw)++ _Path.write_text = spy_write_text+ os.replace = spy_replace+ try:+ resource_plan.analyze_model(self.dir)+ finally:+ _Path.write_text = orig_write_text+ os.replace = orig_replace++ self.assertEqual(len(replace_calls), 1)+ replaced_src, replaced_dst = replace_calls[0]+ self.assertEqual(replaced_dst, cache_path)+ self.assertNotEqual(replaced_src, cache_path)+ # Every write_text call in this run must target the tmp path that was+ # then replaced onto cache_path, never cache_path directly.+ self.assertIn(replaced_src, write_calls)+ self.assertNotIn(cache_path, write_calls)++ def test_concurrent_writers_produce_a_valid_final_cache(self):+ # Multiple threads racing analyze_model() on the same model dir (each+ # forcing a fresh recompute+write via a bumped mtime) must never leave+ # a corrupt or half-written cache file, and no stray tmp files.+ import threading++ cache_path = self.dir / resource_plan._ANALYSIS_CACHE_NAME+ shard_path = self.dir / "model-00001.safetensors"+ errors = []++ def worker(n):+ try:+ for _ in range(15):+ os.utime(shard_path, None) # force a real recompute+write each time+ resource_plan.analyze_model(self.dir)+ except Exception as exc: # pragma: no cover+ errors.append(exc)++ threads = [threading.Thread(target=worker, args=(i,)) for i in range(6)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()++ self.assertEqual(errors, [])+ self.assertEqual(list(self.dir.glob("*.tmp")), [])+ json.loads(cache_path.read_text(encoding="utf-8")) # must not raise+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/test_grammar_cache.c b/c/tests/test_grammar_cache.cnew file mode 100644index 00000000..b6c86d1b--- /dev/null+++ b/c/tests/test_grammar_cache.c@@ -0,0 +1,88 @@+/* schema->GBNF compile-cache correctness (#7).+ *+ * mux_submit now skips re-compiling a slot's grammar when the next request sends+ * the IDENTICAL schema/grammar text: it keeps an owned copy in GrDraft.src and,+ * on a match, does grammar_reset() instead of grammar_teardown()+setup. That is+ * only correct if two claims hold, which this test pins on the REAL functions+ * (via the include-colibri.c pattern):+ *+ * (1) EQUIVALENCE: grammar_reset() on an already-compiled grammar yields the+ * exact same walker behaviour as a fresh grammar_setup_text() of the same+ * text -- same forced spans, same accept/reject at every byte.+ * (2) OWNERSHIP: the GrDraft.src lifecycle (set on compile, freed on teardown)+ * never leaks or double-frees across the miss/hit/fail/no-grammar paths.+ */+#define main coli_glm_main_unused+#include "../colibri.c"+#undef main++#define CHECK(cond) do { if(!(cond)){ \+ fprintf(stderr,"%s:%d: check failed: %s\n",__FILE__,__LINE__,#cond); return 1; } } while(0)++/* feed a byte string through a walker; return bytes accepted before a reject */+static int feed(GrState *S, const char *s){+ int n=0; while(s[n]){ if(gr_accept(S,(unsigned char)s[n])!=1) break; n++; } return n;+}++/* Compare fresh-setup vs reset behaviour of the SAME compiled grammar. */+static int check_equivalence(const char *gbnf, const char *sample){+ GrDraft a; memset(&a,0,sizeof a); a.max=24;+ char *t1=malloc(strlen(gbnf)+1); strcpy(t1,gbnf);+ CHECK(grammar_setup_text(&a,NULL,t1,"A")==0); /* frees t1 */+ char fa[512]; int fa_n=gr_forced(&a.st,fa,sizeof fa);+ int acc_a=feed(&a.st,sample);++ /* reset the SAME draft and re-run: must match itself exactly */+ grammar_reset(&a);+ char fr[512]; int fr_n=gr_forced(&a.st,fr,sizeof fr);+ int acc_r=feed(&a.st,sample);+ CHECK(fr_n==fa_n && !memcmp(fr,fa,(size_t)fa_n) && acc_r==acc_a);++ /* a completely fresh draft from the same text: must also match */+ GrDraft b; memset(&b,0,sizeof b); b.max=24;+ char *t2=malloc(strlen(gbnf)+1); strcpy(t2,gbnf);+ CHECK(grammar_setup_text(&b,NULL,t2,"B")==0);+ char fb[512]; int fb_n=gr_forced(&b.st,fb,sizeof fb);+ int acc_b=feed(&b.st,sample);+ CHECK(fb_n==fa_n && !memcmp(fb,fa,(size_t)fa_n) && acc_b==acc_a);++ grammar_teardown(&a); grammar_teardown(&b);+ return 0;+}++int main(void){+ /* (1) equivalence across representative grammars */+ CHECK(check_equivalence("root ::= \"{\\\"id\\\":\"", "{\"id\":")==0);+ CHECK(check_equivalence("root ::= \"a\" (\"b\" | \"c\") \"d\"", "abd")==0);+ CHECK(check_equivalence("root ::= \"[\" [0-9]+ \"]\"", "[123]")==0);++ /* (2) ownership: replicate the exact mux_submit miss/hit/fail/none flow,+ * setting GrDraft.src exactly as mux_submit does, and exercising teardown. */+ GrDraft gd; memset(&gd,0,sizeof gd); gd.max=24;+ const char *A="root ::= \"x\"";+ for(int turn=0; turn<3; turn++){+ /* HIT path if src matches, else MISS (compile) path */+ if(gd.on && gd.src && !strcmp(gd.src,A)){+ grammar_reset(&gd); /* cache hit: reset only */+ } else {+ grammar_teardown(&gd);+ char *keep=malloc(strlen(A)+1); strcpy(keep,A);+ char *txt=malloc(strlen(A)+1); strcpy(txt,A);+ if(grammar_setup_text(&gd,NULL,txt,"req")==0) gd.src=keep; else free(keep);+ }+ CHECK(gd.on && gd.src && !strcmp(gd.src,A));+ }+ /* switch to a different grammar: teardown must free the old src */+ grammar_teardown(&gd);+ CHECK(gd.src==NULL);+ /* a grammar that fails to compile: left recursion -> gr_state_init not alive,+ * grammar_setup_text returns -1. src must stay NULL, no crash. */+ const char *BAD="root ::= root \"a\"";+ char *bad=malloc(strlen(BAD)+1); strcpy(bad,BAD);+ int rc=grammar_setup_text(&gd,NULL,bad,"bad");+ CHECK(rc!=0 && gd.src==NULL);+ grammar_teardown(&gd); /* NULL-safe free(src) */++ printf("test_grammar_cache: reset==fresh-setup equivalence + src ownership OK\n");+ return 0;+}diff --git a/c/tests/test_rope_invfreq.c b/c/tests/test_rope_invfreq.cnew file mode 100644index 00000000..e08ffd42--- /dev/null+++ b/c/tests/test_rope_invfreq.c@@ -0,0 +1,58 @@+/* RoPE inv_freq precompute (byte-identical guard).+ *+ * rope_interleave() used to recompute inv=powf(theta,-2j/qk_rope) for every+ * position inside its thread-local cache refresh; it now precomputes inv[j]+ * once (keyed on theta/qk) and forms ang=pos*inv[j] per position. The rotation+ * is byte-sensitive: cs/sn feed q/k dot products, so any bit drift changes the+ * sampled token. This test drives the REAL rope_interleave (via the test_topp.c+ * include-colibri.c pattern) and compares its rotated output, BIT-FOR-BIT,+ * against an independent reference that reimplements the OLD powf-per-position+ * formula. Sweeps the qk_rope sizes and rope_theta values GLM/Kimi actually use.+ */+#define main coli_glm_main_unused+#include "../colibri.c"+#undef main++#include <math.h>+#include <stdint.h>++#define CHECK(cond) do { if(!(cond)){ \+ fprintf(stderr,"%s:%d: check failed: %s\n",__FILE__,__LINE__,#cond); return 1; } } while(0)++static int biteq(float a, float b){ uint32_t x,y; memcpy(&x,&a,4); memcpy(&y,&b,4); return x==y; }++/* Reference: the pre-optimisation rotation, powf recomputed every call. */+static void rope_ref(float *v, int pos, int qk, float theta){+ int half=qk/2; float in[256]; memcpy(in,v,qk*sizeof(float));+ for(int j=0;j<half;j++){+ float inv=powf(theta,-2.0f*j/qk), ang=pos*inv;+ float cs=cosf(ang), sn=sinf(ang);+ float a=in[2*j], b=in[2*j+1];+ v[j] = a*cs - b*sn;+ v[half+j] = b*cs + a*sn;+ }+}++int main(void){+ const int qks[] = {64,128,96,256};+ const float ths[] = {10000.f, 1000000.f, 500000.f};+ long checked=0;+ for(int qi=0; qi<4; qi++) for(int ti=0; ti<3; ti++){+ int qk=qks[qi]; float th=ths[ti];+ Cfg c; memset(&c,0,sizeof c); c.qk_rope=qk; c.theta=th;+ for(int pos=0; pos<200000; pos += (pos<4096 ? 1 : 37)){+ float base[256], vr[256], vn[256];+ /* deterministic, varied input (no rand: reproducible across platforms) */+ for(int i=0;i<qk;i++) base[i] = sinf((float)(i*7 + pos)) * 1.5f - 0.3f;+ memcpy(vr,base,qk*sizeof(float)); memcpy(vn,base,qk*sizeof(float));+ rope_ref(vr,pos,qk,th);+ rope_interleave(vn,pos,&c); /* the REAL shipping function */+ for(int i=0;i<qk;i++){+ checked++;+ CHECK(biteq(vr[i],vn[i]));+ }+ }+ }+ printf("test_rope_invfreq: %ld rotated floats bit-identical to reference\n",checked);+ return 0;+}