Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

serve: honor --ngen when the client omits max_tokens; convert: print the resolved plan (#382, #383)

JustVuggf8e4dc95b5ad
diff --git a/c/openai_server.py b/c/openai_server.py
index d55fbc84..f56e20a2 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -406,7 +406,10 @@ def generation_options(body, limit):
maximum = body.get("max_tokens")
maximum_param = "max_tokens"
if maximum is None:
- maximum = min(256, limit)
+ # Client omitted max_tokens: honor the operator's configured budget (--max-tokens /
+ # --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382).
+ # Generation still ends at EOS, so this is a cap, not a target.
+ maximum = limit
temperature = body.get("temperature")
top_p = body.get("top_p")
temperature = 0.7 if temperature is None else temperature
diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py
index fb2fb738..9cd69376 100644
--- a/c/tools/convert_fp8_to_int4.py
+++ b/c/tools/convert_fp8_to_int4.py
@@ -299,6 +299,16 @@ def main():
if bits_map:
print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items())))
+ # Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il
+ # default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e'
+ # gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job
+ # da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything.
+ mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model"
+ grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \
+ (f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row")
+ print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | "
+ f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}")
+
if a.selftest_nvfp4:
import torch
# 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi.