Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
serve: honor --ngen when the client omits max_tokens; convert: print the resolved plan (#382, #383)
JustVuggf8e4dc95b5ad
diff --git a/c/openai_server.py b/c/openai_server.pyindex d55fbc84..f56e20a2 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -406,7 +406,10 @@ def generation_options(body, limit):maximum = body.get("max_tokens")maximum_param = "max_tokens"if maximum is None:- maximum = min(256, limit)+ # Client omitted max_tokens: honor the operator's configured budget (--max-tokens /+ # --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382).+ # Generation still ends at EOS, so this is a cap, not a target.+ maximum = limittemperature = body.get("temperature")top_p = body.get("top_p")temperature = 0.7 if temperature is None else temperaturediff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.pyindex fb2fb738..9cd69376 100644--- a/c/tools/convert_fp8_to_int4.py+++ b/c/tools/convert_fp8_to_int4.py@@ -299,6 +299,16 @@ def main():if bits_map:print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items())))+ # Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il+ # default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e'+ # gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job+ # da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything.+ mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model"+ grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \+ (f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row")+ print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | "+ f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}")+if a.selftest_nvfp4:import torch# 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi.