Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

refactor(core): make model families registry-owned

Mario Gutierrezfe3879cd94af
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 1e570427..b7036e93 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -135,6 +135,7 @@ jobs:
cp c/version.py dist/
cp c/openai_server.py dist/
cp c/v4_dsml.py dist/ # openai_server imports v4_dsml (vendored DeepSeek V4 DSML primitives); without it the packaged server won't import
+ cp c/family_registry.py dist/ # authoritative model-family dispatch/capability registry
cp c/resource_plan.py dist/
cp c/doctor.py dist/
cp c/autotune.py dist/
@@ -184,11 +185,9 @@ jobs:
# fine. The check was wrong, not the artifact.
python3 -c "import ast; ast.parse(open('tools/k3_tokenizer.py', encoding='utf-8').read())" \
|| { echo "FAIL: packaged k3_tokenizer.py does not parse"; exit 1; }
- # COLI_EXPECT rather than a literal tuple: the list is platform-dependent
- # now, and one hardcoded list that silently omitted an engine is what #858
- # was. Keep the expectation in a single place, above.
COLI_EXPECT="$SIBLINGS" python3 - <<'PYCHK'
import importlib.machinery, importlib.util, json, os, sys, tempfile
+ from family_registry import all_families
exe = ".exe" if os.name == "nt" else ""
expect = os.environ["COLI_EXPECT"].split()
@@ -213,16 +212,25 @@ jobs:
loader.exec_module(cli)
bad = []
- for model_type, label, _ in cli._BANNER_MODELS:
- with tempfile.TemporaryDirectory() as d:
+ for family in all_families():
+ if family.id == "deepseek_v4" and "deepseek_v4" not in expect:
+ continue
+ model_type = family.model_types[0]
+ label = family.display_name
+ with tempfile.TemporaryDirectory() as d:
with open(os.path.join(d, "config.json"), "w", encoding="utf-8") as f:
json.dump({"model_type": model_type}, f)
open(os.path.join(d, "tokenizer.json"), "w").close()
engine = os.path.basename(cli.engine_for(d))
arch = cli.model_arch(d)
- if model_type != "glm" and arch == "glm":
- bad.append("%s (%s) -> the GLM engine; the banner names a family "
- "the launcher cannot dispatch" % (label, model_type))
+ expected_file = family.engine_artifact + exe
+ if not os.path.exists(expected_file):
+ bad.append("%s: expected artifact %s is absent" %
+ (label, expected_file))
+ elif arch != family.id or engine != expected_file:
+ bad.append("%s (%s) -> %s/%s, expected %s/%s" %
+ (label, model_type, arch, engine,
+ family.id, expected_file))
if bad:
sys.exit("FAIL: packaged launcher misroutes:\n " + "\n ".join(bad))
print("OK: every engine is packaged AND coli dispatches to it:", " ".join(expect))
diff --git a/c/Makefile b/c/Makefile
index 26b97fa5..f16b66c8 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1170,17 +1170,21 @@ check:
$(MAKE) portable
$(MAKE) test
-install: colibri$(EXE) olmoe$(EXE) $(if $(filter 1,$(COLI_V4_SUPPORTED)),deepseek-v4,)
+install: colibri$(EXE) inkling$(EXE) kimi_k3$(EXE) olmoe$(EXE) \
+ $(if $(filter 1,$(COLI_V4_SUPPORTED)),deepseek-v4,)
$(INSTALL) -d $(DESTDIR)$(BINDIR)
$(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)
$(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/tools
$(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli
$(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE)
+ $(INSTALL) -m 755 inkling$(EXE) $(DESTDIR)$(LIBEXECDIR)/inkling$(EXE)
+ $(INSTALL) -m 755 kimi_k3$(EXE) $(DESTDIR)$(LIBEXECDIR)/kimi_k3$(EXE)
$(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE)
@if [ -f deepseek_v4$(EXE) ]; then \
$(INSTALL) -m 755 deepseek_v4$(EXE) $(DESTDIR)$(LIBEXECDIR)/deepseek_v4$(EXE); \
fi
- $(INSTALL) -m 644 resource_plan.py doctor.py autotune.py openai_server.py version.py $(DESTDIR)$(LIBEXECDIR)/
+ $(INSTALL) -m 644 family_registry.py resource_plan.py doctor.py autotune.py \
+ openai_server.py v4_dsml.py version.py $(DESTDIR)$(LIBEXECDIR)/
$(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/
@# The dashboard is an optional build artifact (cd web && npm run build), so install
@# it only when it exists. It goes NEXT TO openai_server.py, which probes ./web/dist.
diff --git a/c/coli b/c/coli
index 2ce41bd1..9060cec2 100755
--- a/c/coli
+++ b/c/coli
@@ -68,6 +68,9 @@ except ModuleNotFoundError:
except ModuleNotFoundError:
_version = "unknown"
+from family_registry import (FamilyConfigError, UnknownFamilyError, all_families,
+ family_by_id, resolve_model)
+
# Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the
# support modules (resource_plan.py, doctor.py, autotune.py, openai_server.py) and
# tools/ all live next to this script — unchanged from before.
@@ -156,13 +159,8 @@ def sprite_lines():
# Parameter counts are the roster's, i.e. the README's, so the two cannot drift
# apart quietly. A model_type absent from this table is NOT forced into a name
# here -- it prints its own model_type and its measured geometry instead.
-_BANNER_MODELS = (
- ("deepseek_v4", "DeepSeek V4 Flash", "284B"), # before "deepseek": V4 is its own engine
- ("inkling", "Inkling", "975B"),
- ("kimi", "Kimi K3", "2.8T"),
- ("olmoe", "OLMoE", "7B"),
- ("glm", "GLM-5.2", "744B"),
-)
+_BANNER_MODELS = tuple((family.model_types[0], family.display_name, family.display_scale)
+ for family in all_families())
def model_banner_line(model):
"""Third banner line: what is actually loaded, or the generic tagline.
@@ -180,15 +178,13 @@ def model_banner_line(model):
except (OSError, ValueError, TypeError):
return generic
kind = (cfg.get("model_type") or "").lower()
-
- name, scale = "", ""
- for key, display, params in _BANNER_MODELS:
- if key in kind:
- name, scale = display, params
- break
- if not name:
+ try:
+ family = resolve_model(model).descriptor
+ name, scale = family.display_name, family.display_scale
+ except (FamilyConfigError, UnknownFamilyError):
# Unknown checkpoint: say so with its own words rather than guess.
name = kind or "unknown model"
+ scale = ""
bits = [name]
if scale:
@@ -310,27 +306,20 @@ def redraw_prompt_box(message, width):
return clean
def model_arch(model):
- try:
- with open(os.path.join(model, "config.json"), encoding="utf-8") as f:
- model_type = (json.load(f).get("model_type") or "").lower()
- except (OSError, ValueError, TypeError):
- return "glm"
- if "inkling" in model_type:
- return "inkling"
- if "kimi" in model_type:
- return "kimi"
- if "olmoe" in model_type:
- return "olmoe"
- if "deepseek_v4" in model_type or ("deepseek" in model_type and "v4" in model_type):
- return "deepseek_v4"
- return "glm"
+ return resolve_model(model).descriptor.id
def engine_for(model):
- arch = model_arch(model)
- if arch == "glm" or os.environ.get("COLI_ENGINE"):
+ family = resolve_model(model).descriptor
+ if os.environ.get("COLI_ENGINE"):
+ if (family.id != "glm" and
+ os.environ.get("COLI_DOCKER_GLM_ONLY") == "1"):
+ raise UnknownFamilyError(
+ f"this image contains only the GLM engine; {family.display_name} "
+ "needs a full release archive or source build")
+ return GLM
+ if family.id == "glm":
return GLM
- name = {"inkling": "inkling", "kimi": "kimi_k3", "olmoe": "olmoe",
- "deepseek_v4": "deepseek_v4"}[arch]
+ name = family.engine_artifact
local = os.path.join(HERE, name + _EXE)
return local if os.path.exists(local) else os.path.join(_LIBEXEC, name + _EXE)
@@ -343,10 +332,7 @@ def need_model(model, engine=None):
sys.exit(f"{C.yel}tokenizer.json is missing from {model}{C.r}")
engine = engine or engine_for(model)
if not os.path.exists(engine):
- arch = model_arch(model)
- target = ("kimi_k3" if arch == "kimi" else "inkling" if arch == "inkling" else
- "olmoe" if arch == "olmoe" else
- "deepseek-v4" if arch == "deepseek_v4" else "colibri")
+ target = resolve_model(model).descriptor.build_target
sys.exit(f"{C.yel}{target} engine is not built.{C.r} Run: make -C c {target}")
# One-shot runs keep the historic 1024. Interactive sessions get 16384, because
@@ -354,8 +340,12 @@ def need_model(model, engine=None):
# --ngen as a hard ceiling it clamps to (#260) -- so a low ceiling makes the
# user's own control inert instead of merely cautious. Still a ceiling, not a
# target: generation ends at EOS either way.
-def ngen_for(a, interactive=False):
+def ngen_for(a, interactive=False, family=None):
if getattr(a, "ngen", None): return a.ngen
+ if family is not None:
+ if isinstance(family, str): family = family_by_id(family)
+ limits = family.limits
+ return limits.interactive_max_output if interactive else limits.default_max_output
return 16384 if interactive else 1024
def env_for_engine(a, arch):
@@ -393,11 +383,11 @@ def env_for_engine(a, arch):
if sys.platform != "win32":
env.setdefault("OMP_PROC_BIND", "close")
env.setdefault("OMP_PLACES", "cores")
- env["NGEN"] = str(ngen_for(a)) # ngen_for keeps the historic 1024 here
+ env["NGEN"] = str(ngen_for(a, family=arch))
if a.temp is not None: env["COLI_TEMP"] = str(a.temp)
if arch == "olmoe":
env["CHAT"] = "1"
- env["MAX_NEW"] = str(ngen_for(a))
+ env["MAX_NEW"] = str(ngen_for(a, family=arch))
# olmoe reads COLI_TEMP like every other engine (line above). Setting the
# legacy TEMP alias here also clobbered %TEMP% with "0.7" for every child
# of the chat process on Windows — and olmoe never read COLI_TEMP, so the
@@ -408,8 +398,9 @@ def env_for_engine(a, arch):
# user's session ran itself out of memory with the flag apparently set.
if arch in ("deepseek_v4", "kimi"):
if a.ram: env["RAM_GB"] = str(a.ram)
+ if a.ctx:
+ env[family_by_id(arch).limits.context_env] = str(a.ctx)
if arch == "deepseek_v4":
- if a.ctx: env["CTX"] = str(a.ctx)
# Speculation stays opt-in. Real multi-turn chat measured only 1/15
# accepted prompt-lookup candidates; recurrent-state replay dominated
# the visible decode. V4_DRAFT=5 remains available for repeated code.
@@ -448,7 +439,11 @@ def cuda_binary():
return False
def resource_request(a, env):
- ctx=a.ctx or int(env.get("CTX",4096))
+ family=resolve_model(a.model).descriptor if getattr(a,"model",None) else family_by_id("glm")
+ ctx=a.ctx or int(env.get(family.limits.context_env,family.limits.default_context))
+ if ctx<1 or ctx>family.limits.max_context:
+ raise ValueError(f"--ctx must be between 1 and {family.limits.max_context} "
+ f"for {family.display_name}")
def num(v):
try: return float(v)
except ValueError: return 0.0 # "auto": the planner sizes it; the engine reads the string itself
@@ -461,6 +456,12 @@ def resource_request(a, env):
[int(value) for value in gpu.split(",")])
return ram,ctx,devices,vram
+
+def requested_kv_slots(a, env):
+ value=getattr(a,"kv_slots",None)
+ if value is None: value=int(env.get("COLI_KV_SLOTS","1"))
+ return value
+
def env_for(a):
explicit_env = set(os.environ)
e = dict(os.environ, SNAP=a.model)
@@ -537,7 +538,8 @@ def env_for(a):
if a.vram and a.gpu!="none": e["CUDA_EXPERT_GB"]=str(a.vram)
try:
ram,ctx,devices,vram=resource_request(a,e)
- plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy)
+ plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy,
+ kv_slots=requested_kv_slots(a,e))
except (OSError,ValueError,json.JSONDecodeError) as error:
sys.exit(f"{C.yel}invalid resource plan:{C.r} {error}")
has_cuda=cuda_binary()
@@ -588,7 +590,8 @@ def env_for(a):
# as --auto-tier, just without requiring the user to pass it.
ram,ctx,devices,vram_req = resource_request(a, e)
try:
- plan=build_plan(a.model,ram,ctx,devices,vram_req,policy=a.policy)
+ plan=build_plan(a.model,ram,ctx,devices,vram_req,policy=a.policy,
+ kv_slots=requested_kv_slots(a,e))
e.update(environment_for_plan(plan,e,cuda_enabled=True))
vt=plan["tiers"]["vram"]
names=",".join(g["name"].strip() for g in gpus)
@@ -799,9 +802,8 @@ def cmd_build(a):
if not os.path.exists(os.path.join(HERE, "Makefile")):
sys.exit(f"{C.yel}coli build{C.r} only works from a source checkout (this is an installed copy).\n"
f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c colibri.")
- arch=model_arch(a.model) if a.model else "glm"
- target={"glm":"colibri","inkling":"inkling","kimi":"kimi_k3","olmoe":"olmoe",
- "deepseek_v4":"deepseek-v4"}[arch]
+ family=resolve_model(a.model).descriptor if a.model else family_by_id("glm")
+ target=family.build_target
sys.exit(subprocess.call(["make","-C",HERE,target]))
def cmd_info(a):
@@ -809,6 +811,12 @@ def cmd_info(a):
if not a.model:
sys.exit(f"{C.yel}no model directory given.{C.r}\n {NO_MODEL_HINT}")
cfgp=os.path.join(a.model,"config.json")
+ try:
+ family=resolve_model(a.model).descriptor
+ except (FamilyConfigError,UnknownFamilyError) as error:
+ if os.path.exists(cfgp):
+ sys.exit(f"{C.yel}unsupported model:{C.r} {error}")
+ family=None
def row(k,v): print(f" {C.gray}{k:<10}{C.r} {v}")
if os.path.exists(cfgp):
c=json.load(open(cfgp))
@@ -831,7 +839,8 @@ def cmd_info(a):
row("disk", f"{fs.free/1e9:.0f} GB free")
except OSError:
row("disk", "? GB (unavailable)")
- row("engine", "ready ✓" if os.path.exists(GLM) else "not built (coli build)")
+ engine=engine_for(a.model) if family else GLM
+ row("engine", "ready ✓" if os.path.exists(engine) else "not built (coli build)")
knobs=[]
if a.ram: knobs.append(f"ram {a.ram}GB")
if a.topp: knobs.append(f"topp {a.topp}")
@@ -847,7 +856,8 @@ def cmd_plan(a):
ram,ctx,devices,vram=resource_request(a,os.environ)
if ctx<1: raise ValueError("--ctx must be positive")
if a.vram<0: raise ValueError("--vram cannot be negative")
- plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy)
+ plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy,
+ kv_slots=requested_kv_slots(a,os.environ))
except (OSError, ValueError, json.JSONDecodeError) as error:
sys.exit(f"{C.yel}cannot create resource plan:{C.r} {error}")
if a.json:
@@ -859,10 +869,16 @@ def cmd_plan(a):
def cmd_doctor(a):
from doctor import exit_code, format_doctor, run_doctor
+ if not a.model:
+ report={"schema_version":1,"status":"error","model":None,
+ "mode":"deep" if a.deep else "standard",
+ "checks":[{"id":"config.arguments","status":"fail",
+ "summary":f"no model directory given: {NO_MODEL_HINT}"}],
+ "plan":None}
+ print(json.dumps(report,indent=2) if a.json else format_doctor(report)); return 2
try:
# doctor reports rather than exits, so a missing model is a failed check and not
# a sys.exit like everywhere else — it is exactly the thing doctor exists to say.
- if not a.model: raise ValueError(f"no model directory given: {NO_MODEL_HINT}")
ram,ctx,devices,vram=resource_request(a,os.environ)
if ctx<1: raise ValueError("--ctx must be positive")
if ram<0: raise ValueError("--ram cannot be negative")
@@ -874,12 +890,20 @@ def cmd_doctor(a):
"plan":None}
print(json.dumps(report,indent=2) if a.json else format_doctor(report))
return 2
+ engine_error=None
+ try:
+ engine=engine_for(a.model)
+ except (FamilyConfigError,UnknownFamilyError) as error:
+ engine=GLM
+ engine_error=error
report=run_doctor(
# engine_path was hardcoded to GLM, so `coli doctor` inspected colibri
# no matter which model it was pointed at: a built kimi_k3/inkling was
# reported "engine is not built", and a stale colibri would have been
# reported ready for a model that never runs on it (#783).
- a.model, ram, ctx, devices, vram, engine_path=engine_for(a.model), deep=a.deep,
+ a.model, ram, ctx, devices, vram, engine_path=engine, deep=a.deep,
+ kv_slots=requested_kv_slots(a,os.environ),
+ engine_error=engine_error,
mirror_dir=os.environ.get("COLI_MODEL_MIRROR"),
)
print(json.dumps(report,indent=2) if a.json else format_doctor(report))
@@ -898,7 +922,8 @@ def cmd_tune(a):
from resource_plan import build_plan
try:
ram,ctx,devices,vram=resource_request(a,os.environ)
- plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy)
+ plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy,
+ kv_slots=requested_kv_slots(a,os.environ))
except (OSError,ValueError,json.JSONDecodeError) as error:
sys.exit(f"{C.yel}cannot create tuning plan:{C.r} {error}")
a.auto_tier=True
@@ -908,7 +933,10 @@ def cmd_tune(a):
# launched the GLM engine, which died on "this engine requires n_group=1"
# (#898). Same shape as #879, one command over -- the launcher naming a
# family it then mis-dispatched.
- arch=model_arch(a.model)
+ family=resolve_model(a.model).descriptor
+ arch=family.id
+ if family.gateway_adapter not in ("glm","inkling","kimi","olmoe","deepseek_v4"):
+ sys.exit(f"{family.display_name}: gateway adapter {family.gateway_adapter!r} is not wired")
engine=engine_for(a.model)
need_model(a.model,engine)
base_env=env_for_engine(a,arch) if arch!="glm" else env_for(a)
@@ -945,11 +973,16 @@ def cmd_tune(a):
print(f" profile: {path}\n")
def cmd_run(a):
- arch=model_arch(a.model)
+ need_model(a.model)
+ family=resolve_model(a.model).descriptor
+ arch=family.id
engine=engine_for(a.model)
need_model(a.model,engine)
prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"')
banner("run", model=a.model)
+ if family.cli_adapter not in ("glm","deepseek_v4","olmoe"):
+ sys.exit(f"{C.yel}coli run is not wired for {family.display_name};{C.r} "
+ f"use coli chat or coli serve")
if arch=="deepseek_v4":
prompt_path=None
try:
@@ -963,7 +996,7 @@ def cmd_run(a):
cmd += ["--prompt-file",prompt_path]
else:
cmd.append(prompt)
- cmd += ["--max-tokens",str(ngen_for(a,interactive=True))]
+ cmd += ["--max-tokens",str(ngen_for(a,interactive=True,family=family))]
if a.ram: cmd += ["--memory-gb",str(a.ram)]
if os.environ.get("COLI_THINK","0")=="1": cmd.append("--thinking")
result=subprocess.call(cmd,env=env_for_engine(a,arch))
@@ -1154,13 +1187,15 @@ def cmd_chat(a):
chat_attached(a, base, mid); return
if getattr(a,"attach",None):
sys.exit(f"--attach: no coli serve answering at {base} (start one with: coli serve --model <dir>)")
- arch=model_arch(a.model)
+ need_model(a.model)
+ family=resolve_model(a.model).descriptor
+ arch=family.id
+ if family.gateway_adapter not in ("glm","inkling","kimi","olmoe","deepseek_v4"):
+ sys.exit(f"{family.display_name}: gateway adapter {family.gateway_adapter!r} is not wired")
if arch!="glm":
engine=engine_for(a.model)
need_model(a.model,engine)
- model_id=("kimi-k3-colibri" if arch=="kimi" else
- "inkling-colibri" if arch=="inkling" else
- "olmoe-colibri" if arch=="olmoe" else "deepseek-v4-colibri")
+ model_id=family.default_model_id
banner(f"chat · {model_id} · local server", model=a.model)
import openai_server
# --cap rides along so an explicit `coli chat --cap N` reaches the
@@ -1170,7 +1205,8 @@ def cmd_chat(a):
# legacy 8, an explicit value -- 0 included -- passes verbatim).
cmd=[sys.executable,openai_server.__file__,"--model",a.model,
"--engine",engine,"--arch",arch,"--model-id",model_id,
- "--host","127.0.0.1","--port","8000","--max-tokens",str(ngen_for(a,interactive=True))]
+ "--host","127.0.0.1","--port","8000","--max-tokens",
+ str(ngen_for(a,interactive=True,family=family))]
if a.cap is not None: cmd+=["--cap",str(a.cap)]
if a.api_key: cmd+=["--api-key",a.api_key]
p=subprocess.Popen(cmd,env=env_for_engine(a,arch))
@@ -1344,11 +1380,15 @@ def _serve_engine_env(a, arch):
return env
def cmd_serve(a):
- arch=model_arch(a.model)
+ need_model(a.model)
+ family=resolve_model(a.model).descriptor
+ arch=family.id
+ if family.gateway_adapter not in ("glm","inkling","kimi","olmoe","deepseek_v4"):
+ sys.exit(f"{family.display_name}: gateway adapter {family.gateway_adapter!r} is not wired")
engine=engine_for(a.model)
need_model(a.model,engine)
- if arch!="glm" and a.kv_slots!=1:
- sys.exit(f"{arch} currently supports exactly one KV slot")
+ if a.kv_slots>family.limits.max_kv_slots:
+ sys.exit(f"{arch} currently supports at most {family.limits.max_kv_slots} KV slot(s)")
# pidfile: cosi' `coli stop` spegne tutto con un comando, senza pkill a mano.
# EN: pidfile so `coli stop` can shut everything down without manual pkill.
try:
@@ -1356,16 +1396,14 @@ def cmd_serve(a):
except OSError: pass
import openai_server
openai_server.ARCH=arch
- model_id=a.model_id or ("kimi-k3-colibri" if arch=="kimi" else
- "inkling-colibri" if arch=="inkling" else
- "deepseek-v4-colibri" if arch=="deepseek_v4" else
- "olmoe-colibri" if arch=="olmoe" else
- "glm-5.2-colibri")
+ model_id=a.model_id or family.default_model_id
try:
if a.temp is not None: os.environ["COLI_TEMP"] = str(a.temp)
openai_server.serve(a.model,a.host,a.port,model_id,a.api_key,
- a.cap,ngen_for(a,interactive=True),engine,_serve_engine_env(a,arch),a.cors_origin,
- a.max_queue,a.queue_timeout,a.kv_slots,allowed_hosts=a.allowed_host)
+ a.cap,ngen_for(a,interactive=True,family=family),engine,
+ _serve_engine_env(a,arch),a.cors_origin,
+ a.max_queue,a.queue_timeout,a.kv_slots,allowed_hosts=a.allowed_host,
+ family=family)
finally:
try: os.unlink(serve_pidfile(a.port))
except OSError: pass
@@ -1394,7 +1432,9 @@ def cmd_stop(a):
if pid!=os.getpid() and _serve_cmdline_matches_port(cmd,a.port):
targets.setdefault(pid,f"coli serve (cmdline, port {a.port})")
with open(f"/proc/{pd}/comm") as f: comm=f.read().strip()
- if comm in ("colibri","glm","exe","olmoe","inkling","kimi_k3","deepseek_v4"):
+ process_names={"exe"}
+ for family in all_families(): process_names.update(family.process_names)
+ if comm in process_names:
with open(f"/proc/{pd}/environ","rb") as f: env=f.read()
if _serve_environ_matches_port(env,a.port):
targets.setdefault(pid,f"engine `{comm}` (port {a.port})")
@@ -1646,7 +1686,11 @@ def main():
"doctor":cmd_doctor,"tune":cmd_tune,
"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench,
"convert":cmd_convert,"web":cmd_web}.get(a.cmd)
- if handler: sys.exit(handler(a) or 0)
+ if handler:
+ try:
+ sys.exit(handler(a) or 0)
+ except (FamilyConfigError,UnknownFamilyError) as error:
+ sys.exit(f"{C.yel}unsupported model:{C.r} {error}")
banner(); print(__doc__)
if __name__=="__main__":
diff --git a/c/doctor.py b/c/doctor.py
index 6ec6cc87..f149d184 100644
--- a/c/doctor.py
+++ b/c/doctor.py
@@ -8,6 +8,8 @@ import re
import subprocess
from pathlib import Path
+from family_registry import (FamilyConfigError, PlannerUnsupportedError, UnknownFamilyError,
+ public_metadata, resolve_model)
from resource_plan import (GB, SSD_PROBE_PENDING, build_plan, discover_gpus, format_plan,
memory_available)
@@ -423,11 +425,13 @@ def missing_shared_libraries(engine_path):
def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
engine_path, available_memory=None, available_disk=None, gpus=None,
- linkage=None, deep=False, mirror_dir=None):
+ linkage=None, deep=False, mirror_dir=None, kv_slots=1,
+ engine_error=None):
"""Collect a complete report. No model payload, engine, or CUDA context is loaded."""
model = Path(model).expanduser().resolve()
checks = []
plan = None
+ resolved = None
if model.is_dir() and os.access(model, os.R_OK):
checks.append(_check("model.path", "pass", "model directory is readable", path=str(model)))
@@ -443,6 +447,18 @@ def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
valid_config = False
checks.append(_check("model.config", "pass" if valid_config else "fail",
"config.json is valid" if valid_config else "config.json is missing or invalid"))
+ if valid_config:
+ try:
+ resolved = resolve_model(model)
+ checks.append(_check("model.family", "pass",
+ f"{resolved.descriptor.display_name} family is registered",
+ family_id=resolved.descriptor.id,
+ model_type=resolved.model_type,
+ descriptor=public_metadata(resolved.descriptor)))
+ except (FamilyConfigError, UnknownFamilyError) as error:
+ checks.append(_check("model.family", "fail", str(error)))
+ else:
+ checks.append(_check("model.family", "skip", "family detection requires a valid config"))
tokenizer = model / "tokenizer.json"
checks.append(_check("model.tokenizer", "pass" if tokenizer.is_file() else "fail",
"tokenizer.json found" if tokenizer.is_file() else "tokenizer.json is missing"))
@@ -463,7 +479,9 @@ def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
engine_ok = engine.is_file()
else:
engine_ok = engine.is_file() and os.access(engine, os.X_OK)
- if engine_ok:
+ if engine_error:
+ checks.append(_check("engine.binary", "fail", str(engine_error), path=str(engine)))
+ elif engine_ok:
unresolved = missing_shared_libraries(engine)
if unresolved:
checks.append(_check("engine.binary", "fail",
@@ -504,9 +522,11 @@ def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
checks.append(_check("accelerator.gpu", "skip", "no supported GPU detected; CPU path is available"))
try:
+ if resolved is None:
+ raise ValueError("placement requires a registered model family")
plan = build_plan(model, ram_gb, context, gpu_indices, vram_gb,
available_memory=available_memory, available_disk=available_disk,
- gpus=detected_gpus)
+ gpus=detected_gpus, kv_slots=kv_slots)
model_info = plan["model"]
checks.append(_check("model.shards", "pass", "safetensors headers are valid",
shards=model_info["shards"], model_bytes=model_info["model_bytes"]))
@@ -547,6 +567,16 @@ def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
else:
checks.append(_check("storage.ssd_probe", "skip",
"no cached probe yet; measured on the first Metal+darwin engine start"))
+ except PlannerUnsupportedError as error:
+ checks.append(_check("model.shards", "pass", "safetensors headers are readable",
+ shards=len(list(model.glob("*.safetensors")))))
+ checks.append(_check("storage.disk", "skip",
+ "storage projection requires a family planner"))
+ checks.append(_check("memory.ram", "skip",
+ "RAM projection requires a family planner"))
+ checks.append(_check("placement.plan", "skip", str(error)))
+ checks.append(_check("storage.ssd_probe", "skip",
+ "probe surfacing requires a family planner"))
except (OSError, ValueError, KeyError, TypeError) as error:
checks.append(_check("model.shards", "fail", str(error)))
checks.append(_check("storage.disk", "skip", "storage check requires a valid model"))
diff --git a/c/family_registry.py b/c/family_registry.py
new file mode 100644
index 00000000..c72b037c
--- /dev/null
+++ b/c/family_registry.py
@@ -0,0 +1,352 @@
+#!/usr/bin/env python3
+"""Authoritative model-family registry for Colibri's Python control plane."""
+
+from dataclasses import dataclass
+import json
+import re
+from pathlib import Path
+
+
+class RegistryError(ValueError):
+ pass
+
+
+class FamilyConfigError(ValueError):
+ pass
+
+
+class UnknownFamilyError(ValueError):
+ pass
+
+
+class PlannerUnsupportedError(ValueError):
+ pass
+
+
+@dataclass(frozen=True, slots=True)
+class FamilyCapabilities:
+ tools: bool
+ grammar_payload: bool
+ audio_payload: bool
+ thinking: bool
+
+
+@dataclass(frozen=True, slots=True)
+class FamilyLimits:
+ default_context: int
+ max_context: int
+ default_max_output: int
+ interactive_max_output: int
+ max_kv_slots: int
+ implicit_cap: int
+ context_env: str
+
+
+@dataclass(frozen=True, slots=True)
+class PlannerGeometry:
+ context_state_bytes: int
+ fixed_state_bytes: int
+ workspace_bytes: int
+ configured_experts: int
+
+
+@dataclass(frozen=True, slots=True)
+class FamilyDescriptor:
+ id: str
+ model_types: tuple
+ display_name: str
+ display_scale: str
+ engine_artifact: str
+ engine_aliases: tuple
+ engine_group: str
+ internal_arch: str
+ build_target: str
+ process_names: tuple
+ default_model_id: str
+ cli_adapter: str
+ gateway_adapter: str
+ planner_id: str
+ planner_geometry: object
+ planner_unsupported_reason: str
+ expert_inventory: object
+ config_section: str
+ limits: FamilyLimits
+ capabilities: FamilyCapabilities
+
+
+@dataclass(frozen=True, slots=True)
+class ResolvedFamily:
+ descriptor: FamilyDescriptor
+ model_type: str
+ config: dict
+ family_config: dict
+ model_dir: str
+
+
+def _required_int(config, key, family, minimum=1):
+ value = config.get(key)
+ if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
+ raise ValueError(f"{family}: missing or invalid planning key {key!r}")
+ return value
+
+
+def _optional_int(config, key, default=0, minimum=0):
+ value = config.get(key, default)
+ if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
+ raise ValueError(f"invalid planning key {key!r}")
+ return value
+
+
+def _glm_geometry(config, context, _model_dir):
+ layers = _required_int(config, "num_hidden_layers", "glm") + 1
+ experts = _required_int(config, "n_routed_experts", "glm")
+ kv_lora = _required_int(config, "kv_lora_rank", "glm")
+ rope = _required_int(config, "qk_rope_head_dim", "glm")
+ heads = _required_int(config, "num_attention_heads", "glm")
+ nope = _required_int(config, "qk_nope_head_dim", "glm")
+ value = _required_int(config, "v_head_dim", "glm")
+ state = layers * context * (kv_lora + rope) * 4
+ index_dim = _optional_int(config, "index_head_dim", 0)
+ if index_dim and config.get("_colibri_indexer_present", False):
+ kinds = config.get("indexer_types")
+ if isinstance(kinds, list):
+ active = sum(kind == "full" for kind in kinds[:layers - 1])
+ else:
+ frequency = max(1, _optional_int(config, "index_topk_freq", 1, 1))
+ offset = _optional_int(config, "index_skip_topk_offset", 2)
+ active = 0
+ for layer in range(layers - 1):
+ index_value = max(layer - offset + 1, 0)
+ active += index_value % frequency == 0
+ state += active * context * index_dim * 4
+ workspace = context * heads * (nope + value) * 4
+ return PlannerGeometry(state, 0, workspace, experts)
+
+
+_GLM_EXPERT = re.compile(
+ r"(?:^|\.)model\.layers\.(\d+)\.mlp\.experts\.(\d+)\."
+)
+_KIMI_EXPERT = re.compile(
+ r"^(?:language_model\.)?model\.layers\.(\d+)\.block_sparse_moe\."
+ r"experts\.(\d+)\."
+)
+_V4_EXPERT = re.compile(r"^layers\.(\d+)\.ffn\.experts\.(\d+)\.")
+_INKLING_EXPERT = re.compile(
+ r"^model\.layers\.(\d+)\.mlp\.experts\."
+ r"(?:gate_up_proj|down_proj)(?:\.|$)"
+)
+
+
+def _individual_expert_inventory(pattern):
+ def inventory(name, size, _config):
+ match = pattern.search(name)
+ if match is None:
+ return ()
+ return ((int(match.group(1)), int(match.group(2)), size),)
+ return inventory
+
+
+def _inkling_expert_inventory(name, size, config):
+ match = _INKLING_EXPERT.fullmatch(name)
+ if match is None:
+ return ()
+ experts = _required_int(config, "n_routed_experts", "inkling")
+ if size % experts:
+ raise ValueError(f"inkling: fused expert tensor {name!r} is not divisible "
+ f"by {experts} experts")
+ per_expert = size // experts
+ layer = int(match.group(1))
+ return tuple((layer, expert, per_expert) for expert in range(experts))
+
+
+COMMON_CAP = FamilyCapabilities(False, False, False, True)
+
+FAMILIES = (
+ FamilyDescriptor(
+ "glm", ("glm_moe_dsa", "glm5_moe", "glm"), "GLM-5.2", "744B",
+ "colibri", ("glm",), "colibri-core", "glm", "colibri",
+ ("colibri", "glm"), "glm-5.2-colibri", "glm", "glm", "glm_mla",
+ _glm_geometry, "", _individual_expert_inventory(_GLM_EXPERT), "root",
+ FamilyLimits(4096, 1048576, 1024, 16384, 16, 0, "CTX"),
+ FamilyCapabilities(True, True, False, True)),
+ FamilyDescriptor(
+ "inkling", ("inkling_mm_model", "inkling"), "Inkling", "975B",
+ "inkling", (), "inkling", "inkling", "inkling", ("inkling",),
+ "inkling-colibri", "inkling", "inkling", "inkling_hybrid", None,
+ "Inkling planning awaits a measured hybrid GQA/sconv runtime adapter",
+ _inkling_expert_inventory, "text_config",
+ FamilyLimits(8192, 1048576, 1024, 1024, 1, 8, "CTX_MAX"),
+ FamilyCapabilities(False, False, True, True)),
+ FamilyDescriptor(
+ "kimi", ("kimi_k3",), "Kimi K3", "2.8T", "kimi_k3", (), "kimi_k3",
+ "kimi_k3", "kimi_k3", ("kimi_k3",), "kimi-k3-colibri", "kimi", "kimi",
+ "kimi_hybrid", None,
+ "Kimi planning awaits measured KDA recurrent-state and native MXFP4 reserves",
+ _individual_expert_inventory(_KIMI_EXPERT), "text_config",
+ FamilyLimits(8192, 1048576, 1024, 1024, 1, 8, "K3_MAXT"), COMMON_CAP),
+ FamilyDescriptor(
+ "olmoe", ("olmoe",), "OLMoE", "7B", "olmoe", (), "olmoe", "olmoe",
+ "olmoe", ("olmoe",), "olmoe-colibri", "olmoe", "olmoe", "olmoe_gqa",
+ None, "OLMoE planning awaits a measured runtime/workspace reserve",
+ _individual_expert_inventory(_GLM_EXPERT), "root",
+ FamilyLimits(4096, 4096, 1024, 1024, 1, 8, "CTX"),
+ FamilyCapabilities(False, False, False, False)),
+ FamilyDescriptor(
+ "deepseek_v4", ("deepseek_v4",), "DeepSeek V4 Flash", "284B",
+ "deepseek_v4", (), "deepseek_v4", "deepseek_v4", "deepseek-v4",
+ ("deepseek_v4",), "deepseek-v4-colibri", "deepseek_v4", "deepseek_v4",
+ "deepseek_v4", None,
+ "DeepSeek V4 uses its C resident-tier planner; Python parity is not yet proven",
+ _individual_expert_inventory(_V4_EXPERT), "root",
+ FamilyLimits(4096, 1048576, 1024, 16384, 1, 8, "CTX"),
+ FamilyCapabilities(True, False, False, True)),
+)
+
+
+def _build_registry(families):
+ by_id = {}
+ by_type = {}
+ identities = set()
+ for family in families:
+ if not re.fullmatch(r"[a-z0-9_]+", family.id) or family.id in by_id:
+ raise RegistryError(f"invalid or duplicate family id: {family.id!r}")
+ if (not family.model_types or
+ (not callable(family.planner_geometry) and
+ not family.planner_unsupported_reason) or
+ not callable(family.expert_inventory)):
+ raise RegistryError(f"incomplete family descriptor: {family.id}")
+ identity = (family.engine_artifact, family.internal_arch)
+ if identity in identities:
+ raise RegistryError(f"duplicate engine identity: {identity}")
+ identities.add(identity)
+ by_id[family.id] = family
+ for model_type in family.model_types:
+ normalized = _normalize_model_type(model_type)
+ if normalized in by_type:
+ raise RegistryError(f"duplicate model_type alias: {normalized}")
+ by_type[normalized] = family
+ if (family.limits.default_context < 1 or family.limits.max_context < family.limits.default_context or
+ family.limits.default_max_output < 1 or family.limits.interactive_max_output < 1 or
+ family.limits.max_kv_slots < 1 or family.limits.implicit_cap < 0):
+ raise RegistryError(f"invalid limits for family: {family.id}")
+ return by_id, by_type
+
+
+def _normalize_model_type(model_type):
+ if not isinstance(model_type, str) or not model_type.strip():
+ raise FamilyConfigError("config.json has no non-empty string model_type")
+ return model_type.strip().lower()
+
+
+_BY_ID, _BY_TYPE = _build_registry(FAMILIES)
+
+
+def all_families():
+ return FAMILIES
+
+
+def family_ids():
+ return tuple(family.id for family in FAMILIES)
+
+
+def family_by_id(family_id):
+ try:
+ return _BY_ID[family_id]
+ except KeyError as error:
+ raise UnknownFamilyError(f"unknown model family: {family_id}") from error
+
+
+def family_for_config(config):
+ if not isinstance(config, dict):
+ raise FamilyConfigError("config.json is not a JSON object")
+ model_type = _normalize_model_type(config.get("model_type"))
+ try:
+ return _BY_TYPE[model_type]
+ except KeyError as error:
+ raise UnknownFamilyError(f"unsupported model_type: {model_type}") from error
+
+
+def resolve_model(model_dir):
+ model = Path(model_dir).expanduser().resolve()
+ path = model / "config.json"
+ try:
+ config = json.loads(path.read_text(encoding="utf-8"))
+ except OSError as error:
+ raise FamilyConfigError(f"cannot read config.json: {model}") from error
+ except json.JSONDecodeError as error:
+ raise FamilyConfigError(f"invalid config.json: {error}") from error
+ family = family_for_config(config)
+ family_config = config
+ if family.config_section == "text_config":
+ family_config = config.get("text_config", config)
+ if not isinstance(family_config, dict):
+ raise FamilyConfigError(f"{family.id}: text_config is not an object")
+ return ResolvedFamily(family, _normalize_model_type(config.get("model_type")),
+ config, family_config, str(model))
+
+
+def planner_geometry(resolved, context):
+ if isinstance(context, bool) or not isinstance(context, int) or context < 1:
+ raise ValueError("context must be a positive integer")
+ if context > resolved.descriptor.limits.max_context:
+ raise ValueError(f"{resolved.descriptor.id}: context {context} exceeds "
+ f"the registered maximum {resolved.descriptor.limits.max_context}")
+ if resolved.descriptor.planner_geometry is None:
+ raise PlannerUnsupportedError(
+ f"{resolved.descriptor.display_name}: "
+ f"{resolved.descriptor.planner_unsupported_reason}")
+ geometry = resolved.descriptor.planner_geometry(
+ resolved.family_config, context, resolved.model_dir)
+ if not isinstance(geometry, PlannerGeometry) or any(
+ isinstance(value, bool) or not isinstance(value, int) or value < 0
+ for value in (geometry.context_state_bytes, geometry.fixed_state_bytes,
+ geometry.workspace_bytes, geometry.configured_experts)):
+ raise RegistryError(f"invalid planner geometry for {resolved.descriptor.id}")
+ if geometry.configured_experts < 1:
+ raise ValueError(f"{resolved.descriptor.id}: configured expert count is zero")
+ return geometry
+
+
+def expert_contributions(resolved, name, size):
+ if isinstance(size, bool) or not isinstance(size, int) or size < 0:
+ raise ValueError("tensor size must be a non-negative integer")
+ contributions = resolved.descriptor.expert_inventory(
+ name, size, resolved.family_config)
+ for layer, expert, byte_count in contributions:
+ if layer < 0 or expert < 0 or byte_count < 0:
+ raise RegistryError(f"invalid expert inventory for {resolved.descriptor.id}")
+ return contributions
+
+
+def public_metadata(family):
+ return {
+ "id": family.id,
+ "model_types": list(family.model_types),
+ "display_name": family.display_name,
+ "display_scale": family.display_scale,
+ "engine_artifact": family.engine_artifact,
+ "engine_aliases": list(family.engine_aliases),
+ "engine_group": family.engine_group,
+ "internal_arch": family.internal_arch,
+ "build_target": family.build_target,
+ "process_names": list(family.process_names),
+ "default_model_id": family.default_model_id,
+ "cli_adapter": family.cli_adapter,
+ "gateway_adapter": family.gateway_adapter,
+ "planner_id": family.planner_id,
+ "limits": {
+ "default_context": family.limits.default_context,
+ "max_context": family.limits.max_context,
+ "default_max_output": family.limits.default_max_output,
+ "interactive_max_output": family.limits.interactive_max_output,
+ "max_kv_slots": family.limits.max_kv_slots,
+ "implicit_cap": family.limits.implicit_cap,
+ "context_env": family.limits.context_env,
+ },
+ "capabilities": {
+ "tools": family.capabilities.tools,
+ "grammar_payload": family.capabilities.grammar_payload,
+ "audio_payload": family.capabilities.audio_payload,
+ "thinking": family.capabilities.thinking,
+ },
+ }
diff --git a/c/openai_server.py b/c/openai_server.py
index d2fbce69..82b5fe06 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -21,6 +21,8 @@ import time
import uuid
import v4_dsml # vendored DeepSeek V4 DSML reference primitives
+from family_registry import (FamilyConfigError, UnknownFamilyError, family_by_id,
+ family_ids, resolve_model)
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import unquote, urlsplit
@@ -29,16 +31,16 @@ from urllib.parse import unquote, urlsplit
HERE = Path(__file__).resolve().parent
-def default_engine():
- """The engine next to this file. Since #391 it is built as `colibri`; `glm` stays as a
- fallback so an old tree (or an old hand-built binary) still starts. Reported by
- @RDouglasSharp in #488: the default still said `glm`, so `python3 openai_server.py`
- on a clean checkout looked for a file the build no longer produces."""
- for name in ("colibri", "colibri.exe", "glm", "glm.exe"):
+def default_engine(family=None):
+ """Resolve the registered family's engine next to this file."""
+ family = family or family_by_id("glm")
+ names = (family.engine_artifact, *family.engine_aliases)
+ candidates = tuple(name + suffix for name in names for suffix in ("", ".exe"))
+ for name in candidates:
candidate = HERE / name
if candidate.exists():
return candidate
- return HERE / "colibri"
+ return HERE / family.engine_artifact
END = b"\x01\x01END\x01\x01\n"
READY = b"\x01\x01READY\x01\x01\n"
MAX_BODY = 4 << 20
@@ -1611,23 +1613,8 @@ def read_engine_turn(stream, sentinel, on_bytes):
def model_arch(model):
- """The model's engine family from its config.json model_type -- the same
- rule as coli's model_arch(): "inkling"/"kimi"/"olmoe" substring, everything
- else (including an unreadable config) is glm."""
- try:
- with open(Path(model) / "config.json", encoding="utf-8") as fh:
- model_type = (json.load(fh).get("model_type") or "").lower()
- except (OSError, ValueError, TypeError):
- return "glm"
- if "inkling" in model_type:
- return "inkling"
- if "kimi" in model_type:
- return "kimi"
- if "deepseek_v4" in model_type or ("deepseek" in model_type and "v4" in model_type):
- return "deepseek_v4"
- if "olmoe" in model_type:
- return "olmoe"
- return "glm"
+ """Compatibility wrapper over the mandatory family registry."""
+ return resolve_model(model).descriptor.id
def cap_for_arch(arch, cap):
@@ -1652,7 +1639,7 @@ def cap_for_arch(arch, cap):
-> this shim must be removed and re-derived."""
if cap is not None:
return cap
- return 0 if arch == "glm" else 8
+ return family_by_id(arch).limits.implicit_cap
def tune_child_env(env, arch):
@@ -1692,8 +1679,16 @@ class Engine:
# cap_for_arch above. Same convention as the --cap flags in coli and
# main() below, so programmatic callers that never pass cap get the same
# auto behavior as the CLI; an explicit int (0 included) is verbatim.
- def __init__(self, executable, model, cap=None, max_tokens=1024, env=None, kv_slots=1):
- arch = model_arch(model)
+ def __init__(self, executable, model, cap=None, max_tokens=1024, env=None, kv_slots=1,
+ family=None):
+ if family is None:
+ # Protocol unit tests and embedders may use a synthetic model name.
+ # Real launcher/server entry points resolve strictly before this
+ # constructor; never reinterpret an existing invalid config.
+ config = Path(model) / "config.json"
+ family = (resolve_model(model).descriptor if config.exists()
+ else family_by_id(ARCH))
+ arch = family.id
child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",
NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))
tune_child_env(child_env, arch)
@@ -2477,7 +2472,8 @@ class APIHandler(BaseHTTPRequestHandler):
sys.stderr.flush()
maximum, temperature, top_p, grammar, _requested_stop_sequences = generation_options(
body, self.server.max_tokens)
- if grammar is not None and ARCH in ("inkling", "kimi", "olmoe"):
+ family = family_by_id(ARCH)
+ if grammar is not None and not family.capabilities.grammar_payload:
# sibling engines speak the 6-field SUBMIT header only; sending the
# grammar payload extension would desync its stdin framing.
raise APIError(400, f"`response_format` grammars are not supported by the {ARCH} "
@@ -3056,11 +3052,9 @@ class APIHandler(BaseHTTPRequestHandler):
self.generation(body, prompt, request_id, False)
-def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_key=None,
+def serve(model, host="127.0.0.1", port=8000, model_id=None, api_key=None,
cap=None, max_tokens=1024, engine=None, env=None, cors_origins=None,
- max_queue=8, queue_timeout=300, kv_slots=1, allowed_hosts=()):
- if engine is None:
- engine = default_engine()
+ max_queue=8, queue_timeout=300, kv_slots=1, allowed_hosts=(), family=None):
if not 1 <= max_tokens:
raise ValueError("max_tokens must be positive")
if not 1 <= port <= 65535:
@@ -3071,8 +3065,9 @@ def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_ke
raise ValueError("queue_timeout must be positive")
if not 1 <= kv_slots <= 16:
raise ValueError("kv_slots must be between 1 and 16")
- if ARCH in ("inkling", "kimi", "deepseek_v4", "olmoe") and kv_slots != 1:
- raise ValueError(f"{ARCH} engine currently supports exactly one KV slot")
+ pending_family = family
+ pending_model_id = model_id
+ pending_engine = engine
if host not in ("127.0.0.1", "localhost", "::1") and not api_key:
# (#SEC-6) Fail closed: an unauthenticated engine on a non-loopback bind exposes
# a compute-heavy API to the network. Refuse unless explicitly overridden.
@@ -3094,7 +3089,16 @@ def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_ke
runtime = None
previous_sigterm = signal.getsignal(signal.SIGTERM)
try:
- runtime = Engine(engine,model,cap,max_tokens,env,kv_slots)
+ family = pending_family or resolve_model(model).descriptor
+ global ARCH
+ ARCH = family.id
+ engine = pending_engine or default_engine(family)
+ model_id = pending_model_id or family.default_model_id
+ server.model_id = model_id
+ if kv_slots > family.limits.max_kv_slots:
+ raise ValueError(f"{family.id} engine supports at most "
+ f"{family.limits.max_kv_slots} KV slot(s)")
+ runtime = Engine(engine,model,cap,max_tokens,env,kv_slots,family)
server.engine = runtime
print(f"OpenAI-compatible API listening on http://{host}:{port}/v1", file=sys.stderr)
signal.signal(signal.SIGTERM, lambda *_: threading.Thread(target=server.shutdown, daemon=True).start())
@@ -3110,8 +3114,8 @@ def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_ke
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", default=os.environ.get("COLI_MODEL"), required=not os.environ.get("COLI_MODEL"))
- parser.add_argument("--engine", default=str(default_engine()))
- parser.add_argument("--arch", choices=("auto", "glm", "inkling", "kimi", "deepseek_v4", "olmoe"), default="auto",
+ parser.add_argument("--engine")
+ parser.add_argument("--arch", choices=("auto", *family_ids()), default="auto",
help="chat-template family; auto reads model_type from the model's config.json")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=8000)
@@ -3134,20 +3138,23 @@ def main():
"(reverse proxy / MagicDNS in front of the loopback bind); repeat as needed, "
"or set COLI_ALLOWED_HOSTS as a comma-separated list")
args = parser.parse_args()
+ try:
+ resolved = resolve_model(args.model)
+ except (FamilyConfigError, UnknownFamilyError) as error:
+ parser.error(str(error))
+ family = resolved.descriptor
+ if args.arch != "auto" and args.arch != family.id:
+ parser.error(f"--arch {args.arch} conflicts with model family {family.id}")
global ARCH
- ARCH = args.arch
- if ARCH == "auto":
- ARCH = model_arch(args.model)
+ ARCH = family.id
+ if args.engine is None:
+ args.engine = str(default_engine(family))
if args.model_id is None:
- args.model_id = ("inkling-colibri" if ARCH == "inkling" else
- "kimi-k3-colibri" if ARCH == "kimi" else
- "deepseek-v4-colibri" if ARCH == "deepseek_v4" else
- "olmoe-colibri" if ARCH == "olmoe" else
- "glm-5.2-colibri")
+ args.model_id = family.default_model_id
serve(args.model, args.host, args.port, args.model_id, args.api_key,
args.cap,args.max_tokens,args.engine,cors_origins=args.cors_origin,
max_queue=args.max_queue,queue_timeout=args.queue_timeout,kv_slots=args.kv_slots,
- allowed_hosts=args.allowed_host)
+ allowed_hosts=args.allowed_host,family=family)
if __name__ == "__main__":
diff --git a/c/resource_plan.py b/c/resource_plan.py
index 7a599bfd..ad70504e 100644
--- a/c/resource_plan.py
+++ b/c/resource_plan.py
@@ -10,6 +10,8 @@ import subprocess
import sys
from pathlib import Path
+from family_registry import expert_contributions, planner_geometry, resolve_model
+
GB = 1_000_000_000
EXPERT_RE = re.compile(r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.")
@@ -35,17 +37,16 @@ def _tensor_sizes(path):
def analyze_model(model):
- model = Path(model).resolve()
- config_path = model / "config.json"
- if not config_path.is_file():
- raise ValueError(f"missing config.json: {model}")
- config = json.loads(config_path.read_text(encoding="utf-8"))
+ resolved = resolve_model(model)
+ model = Path(resolved.model_dir)
+ config = resolved.config
shards = sorted(model.glob("*.safetensors"))
if not shards:
raise ValueError(f"no safetensors shards: {model}")
dense_bytes = 0
expert_groups = {}
+ tensor_names = set()
for shard in shards:
try:
sizes = list(_tensor_sizes(shard))
@@ -60,10 +61,12 @@ def analyze_model(model):
raise OSError(error.errno,
f"{error.strerror or error}: {shard}") from error
for name, size in sizes:
- match = EXPERT_RE.search(name)
- if match:
- key = tuple(map(int, match.groups()))
- expert_groups[key] = expert_groups.get(key, 0) + size
+ tensor_names.add(name)
+ contributions = expert_contributions(resolved, name, size)
+ if contributions:
+ for layer, expert, byte_count in contributions:
+ key = (layer, expert)
+ expert_groups[key] = expert_groups.get(key, 0) + byte_count
else:
dense_bytes += size
@@ -75,6 +78,24 @@ def analyze_model(model):
typical_expert_bytes = int(statistics.median(per_layer.values())) if per_layer else 0
max_expert_bytes = max(per_layer.values(), default=0)
model_bytes = sum(shard.stat().st_size for shard in shards)
+ if resolved.descriptor.id == "glm":
+ family_cfg = resolved.family_config
+ layers = int(family_cfg.get("num_hidden_layers") or 0)
+ kinds = family_cfg.get("indexer_types")
+ if isinstance(kinds, list):
+ required = [layer for layer, kind in enumerate(kinds[:layers])
+ if kind == "full"]
+ else:
+ frequency = max(1, int(family_cfg.get("index_topk_freq") or 1))
+ offset = int(family_cfg.get("index_skip_topk_offset") or 2)
+ required = [layer for layer in range(layers)
+ if max(layer - offset + 1, 0) % frequency == 0]
+ indexer_present = bool(required and all(
+ f"model.layers.{layer}.self_attn.indexer.wq_b.weight" in tensor_names
+ for layer in required))
+ family_cfg = dict(family_cfg, _colibri_indexer_present=indexer_present)
+ resolved = type(resolved)(resolved.descriptor, resolved.model_type,
+ resolved.config, family_cfg, resolved.model_dir)
return {
"path": str(model),
"shards": len(shards),
@@ -88,6 +109,7 @@ def analyze_model(model):
"expert_bytes_by_layer": per_layer,
"per_cap_bytes": per_cap_bytes,
"config": config,
+ "resolved_family": resolved,
}
@@ -689,13 +711,18 @@ POLICIES = {
def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
available_memory=None, available_disk=None, gpus=None,
- policy="quality", physical_cpus=None, cpu_sockets=None):
+ policy="quality", physical_cpus=None, cpu_sockets=None,
+ kv_slots=1):
if policy not in POLICIES:
raise ValueError(f"unknown policy: {policy}")
info = analyze_model(model)
physical_cpus = physical_cpu_count() if physical_cpus is None else physical_cpus
cpu_sockets = cpu_socket_count() if cpu_sockets is None else cpu_sockets
- cfg = info["config"]
+ resolved = info["resolved_family"]
+ geometry = planner_geometry(resolved, context)
+ if (isinstance(kv_slots, bool) or not isinstance(kv_slots, int) or
+ not 1 <= kv_slots <= resolved.descriptor.limits.max_kv_slots):
+ raise ValueError(f"{resolved.descriptor.id}: invalid KV slot count {kv_slots}")
available_memory = memory_available() if available_memory is None else available_memory
if available_disk is None:
try:
@@ -717,14 +744,11 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
unified = any(gpu.get("unified_memory", False) for gpu in planning_gpus)
typical = info["typical_expert_bytes"]
max_expert = info["max_expert_bytes"] or typical
- layers = int(cfg.get("num_hidden_layers") or 0) + 1
- kv_bytes = layers * context * (int(cfg.get("kv_lora_rank") or 0) +
- int(cfg.get("qk_rope_head_dim") or 0)) * 4
- kv_buffer = context * int(cfg.get("num_attention_heads") or 0) * (
- int(cfg.get("qk_nope_head_dim") or 0) + int(cfg.get("v_head_dim") or 0)) * 4
+ kv_bytes = (geometry.context_state_bytes + geometry.fixed_state_bytes) * kv_slots
+ kv_buffer = geometry.workspace_bytes
runtime_bytes = int(1.2 * GB + 2.5 * GB + 64 * max_expert + kv_bytes + kv_buffer)
per_cap = info["per_cap_bytes"]
- configured_experts = int(cfg.get("n_routed_experts") or 0)
+ configured_experts = geometry.configured_experts
reserve = 2 * GB
gpu_plan = []
@@ -827,7 +851,11 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
"version": 2,
"policy": {"name": policy, **POLICIES[policy],
"quality_preserving": policy != "experimental-fast"},
- "model": {key: value for key, value in info.items() if key != "config"},
+ "model": {**{key: value for key, value in info.items()
+ if key not in ("config", "resolved_family")},
+ "family_id": resolved.descriptor.id,
+ "model_type": resolved.model_type,
+ "configured_experts": configured_experts},
"cpu": {"physical_cores": _resolve_physical_cores(physical_cpus),
"sockets": max(1, int(cpu_sockets)),
"thread_policy": "physical-cores"},
@@ -837,7 +865,11 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
"available_bytes": available_disk, "cold_expert_bytes": cold_bytes},
"ram": {"role": "resident+warm-experts", "available_bytes": available_memory,
"budget_bytes": ram_budget, "dense_bytes": info["dense_bytes"],
- "runtime_bytes": runtime_bytes, "expert_cache_bytes": cache_bytes,
+ "runtime_bytes": runtime_bytes,
+ "sequence_state_bytes": geometry.context_state_bytes,
+ "fixed_state_bytes": geometry.fixed_state_bytes,
+ "workspace_bytes": geometry.workspace_bytes,
+ "expert_cache_bytes": cache_bytes,
"warm_expert_bytes": warm_bytes, "cache_slots_per_layer": cap},
"vram": {"role": "hot-experts", "devices": gpu_plan,
"budget_bytes": vram_budget, "hot_expert_bytes": hot_bytes,
diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.py
index 655713e4..0d38f4f0 100644
--- a/c/tests/test_cli_output.py
+++ b/c/tests/test_cli_output.py
@@ -214,7 +214,7 @@ class BannerModelLineTest(unittest.TestCase):
def test_each_engine_names_itself(self):
for model_type, expected in (
- ("glm5_moe", "GLM-5.2"),
+ ("glm_moe_dsa", "GLM-5.2"),
("inkling", "Inkling"),
("kimi_k3", "Kimi K3"),
("deepseek_v4", "DeepSeek V4 Flash"),
@@ -253,7 +253,7 @@ class BannerModelLineTest(unittest.TestCase):
def test_size_is_reported_without_rounding_to_zero(self):
small = self.line({"model_type": "olmoe"}, shard_bytes=4_200_000_000)
self.assertIn("4.2 GB on disk", small)
- large = self.line({"model_type": "glm5_moe"}, shard_bytes=372_000_000_000)
+ large = self.line({"model_type": "glm_moe_dsa"}, shard_bytes=372_000_000_000)
self.assertIn("372 GB on disk", large)
tiny = self.line({"model_type": "olmoe"}, shard_bytes=3_000_000)
self.assertIn("MB on disk", tiny)
diff --git a/c/tests/test_doctor.py b/c/tests/test_doctor.py
index 3f08df12..cfa23833 100644
--- a/c/tests/test_doctor.py
+++ b/c/tests/test_doctor.py
@@ -32,6 +32,7 @@ class DoctorTest(unittest.TestCase):
self.model = self.root / "model"
self.model.mkdir()
(self.model / "config.json").write_text(json.dumps({
+ "model_type": "glm_moe_dsa",
"num_hidden_layers": 2,
"n_routed_experts": 2,
"kv_lora_rank": 4,
diff --git a/c/tests/test_doctor_engine_arch.py b/c/tests/test_doctor_engine_arch.py
index 70ed3986..b9fd0f99 100644
--- a/c/tests/test_doctor_engine_arch.py
+++ b/c/tests/test_doctor_engine_arch.py
@@ -18,10 +18,11 @@ HERE = Path(__file__).resolve().parent.parent
CLI = HERE / "coli"
ARCHES = [
- ("glm-5.2", "colibri"),
+ ("glm_moe_dsa", "colibri"),
("inkling", "inkling"),
("kimi_k3", "kimi_k3"),
("olmoe", "olmoe"),
+ ("deepseek_v4", "deepseek_v4"),
]
diff --git a/c/tests/test_env_defaults.py b/c/tests/test_env_defaults.py
index c1c93205..2a8cfb37 100644
--- a/c/tests/test_env_defaults.py
+++ b/c/tests/test_env_defaults.py
@@ -9,7 +9,9 @@ Carica `coli` come modulo (ha la guardia __main__) e verifica il contratto:
import importlib.machinery
import importlib.util
import os
+import json
import sys
+import tempfile
import types
import unittest
from pathlib import Path
@@ -23,9 +25,14 @@ _spec = importlib.util.spec_from_loader("coli_cli", _loader)
coli = importlib.util.module_from_spec(_spec)
_loader.exec_module(coli)
+_MODEL_DIR = tempfile.TemporaryDirectory()
+_MODEL = Path(_MODEL_DIR.name)
+(_MODEL / "config.json").write_text(
+ json.dumps({"model_type": "glm_moe_dsa"}), encoding="utf-8")
+
def args(**over):
- base = dict(model="X", policy="quality", ram=0, ngen=0, topp=0, topk=0,
+ base = dict(model=str(_MODEL), policy="quality", ram=0, ngen=0, topp=0, topk=0,
temp=None, repin=0, ctx=0, auto_tier=False, gpu=None, vram=0)
base.update(over)
return types.SimpleNamespace(**base)
diff --git a/c/tests/test_family_registry.py b/c/tests/test_family_registry.py
new file mode 100644
index 00000000..41bfec8d
--- /dev/null
+++ b/c/tests/test_family_registry.py
@@ -0,0 +1,230 @@
+import json
+import re
+import tempfile
+import unittest
+from pathlib import Path
+
+from family_registry import (
+ FAMILIES,
+ FamilyCapabilities,
+ FamilyConfigError,
+ FamilyDescriptor,
+ FamilyLimits,
+ PlannerGeometry,
+ UnknownFamilyError,
+ PlannerUnsupportedError,
+ _build_registry,
+ family_for_config,
+ planner_geometry,
+ public_metadata,
+ resolve_model,
+)
+
+
+def qwen_geometry(config, context, _model_dir):
+ layers = config["num_hidden_layers"]
+ full = sum(kind == "full_attention" for kind in config["layer_types"])
+ kv = full * context * config["num_key_value_heads"] * config["head_dim"] * 2 * 4
+ conv_dim = (config["linear_num_key_heads"] * config["linear_key_head_dim"] * 2 +
+ config["linear_num_value_heads"] * config["linear_value_head_dim"])
+ fixed = (layers - full) * (
+ config["linear_num_value_heads"] * config["linear_key_head_dim"] *
+ config["linear_value_head_dim"] +
+ conv_dim * (config["linear_conv_kernel_dim"] - 1)) * 4
+ return PlannerGeometry(kv, fixed, 0, config["num_experts"])
+
+
+def minimax_geometry(config, context, _model_dir):
+ state = ((config["num_hidden_layers"] + 1) * context *
+ config["num_key_value_heads"] * config["head_dim"] * 2 * 4)
+ sparse = config["sparse_attention_config"]
+ state += sum(bool(value) for value in sparse["sparse_attention_freq"]) * \
+ context * sparse["sparse_index_dim"] * 4
+ return PlannerGeometry(state, 0, 0, config["num_local_experts"])
+
+
+TEST_INVENTORY = lambda _name, _size, _config: ()
+QWEN36_FIXTURE = FamilyDescriptor(
+ "qwen36", ("qwen3_5_moe_text",), "Qwen3.6", "", "qwen36", (), "qwen36",
+ "qwen36", "qwen36", ("qwen36",), "qwen3.6-colibri", "qwen36", "qwen36",
+ "qwen36_hybrid", qwen_geometry, "", TEST_INVENTORY, "root",
+ FamilyLimits(8192, 262144, 1024, 8192, 1, 8, "Q36_MAXT"),
+ FamilyCapabilities(False, False, False, True))
+MINIMAX_M3_FIXTURE = FamilyDescriptor(
+ "minimax_m3", ("minimax_m3",), "MiniMax M3", "", "colibri", (),
+ "colibri-core", "minimax_m3", "colibri", ("colibri",), "minimax-m3-colibri",
+ "minimax_m3", "minimax_m3", "minimax_m3_gqa", minimax_geometry, "",
+ TEST_INVENTORY, "root", FamilyLimits(8192, 262144, 1024, 8192, 1, 8, "CTX"),
+ FamilyCapabilities(True, False, False, True))
+
+
+class FamilyRegistryTest(unittest.TestCase):
+ def test_production_descriptors_are_complete_unique_and_serializable(self):
+ self.assertGreaterEqual(len(FAMILIES), 5)
+ by_id, by_type = _build_registry(FAMILIES)
+ self.assertEqual(len(by_id), len(FAMILIES))
+ self.assertGreaterEqual(len(by_type), len(FAMILIES))
+ for family in FAMILIES:
+ with self.subTest(family=family.id):
+ json.dumps(public_metadata(family))
+ self.assertIn(family.id, by_id)
+
+ def test_unknown_or_invalid_config_never_falls_back_to_glm(self):
+ with self.assertRaises(UnknownFamilyError):
+ family_for_config({"model_type": "qwen3_moe"})
+ for config in ({}, {"model_type": ""}, {"model_type": []}, None):
+ with self.subTest(config=config), self.assertRaises(FamilyConfigError):
+ family_for_config(config)
+
+ def test_model_resolution_reads_text_config_without_changing_identity(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ config = {"model_type": "kimi_k3", "text_config": {"num_hidden_layers": 2}}
+ (root / "config.json").write_text(json.dumps(config), encoding="utf-8")
+ resolved = resolve_model(root)
+ self.assertEqual(resolved.descriptor.id, "kimi")
+ self.assertEqual(resolved.family_config, config["text_config"])
+
+ def test_qwen_fixture_models_gqa_and_fixed_deltanet_state(self):
+ config = {
+ "model_type": "qwen3_5_moe_text",
+ "num_hidden_layers": 8,
+ "num_attention_heads": 4,
+ "num_key_value_heads": 2,
+ "head_dim": 16,
+ "num_experts": 8,
+ "num_experts_per_tok": 2,
+ "layer_types": ["linear_attention"] * 3 + ["full_attention"] +
+ ["linear_attention"] * 3 + ["full_attention"],
+ "linear_num_value_heads": 8, "linear_num_key_heads": 4,
+ "linear_key_head_dim": 8, "linear_value_head_dim": 8,
+ "linear_conv_kernel_dim": 4,
+ }
+ by_id, by_type = _build_registry(FAMILIES + (QWEN36_FIXTURE,))
+ family = by_type[config["model_type"]]
+ self.assertEqual(family, by_id["qwen36"])
+ resolved = type("R", (), {"descriptor": family, "family_config": config,
+ "model_dir": "."})()
+ geometry = planner_geometry(resolved, 32)
+ self.assertEqual(geometry.configured_experts, 8)
+ self.assertEqual(geometry.context_state_bytes, 16_384)
+ self.assertEqual(geometry.fixed_state_bytes, 6 * (8 * 8 * 8 + 128 * 3) * 4)
+ for model_type in ("qwen2", "qwen3_moe", "my_qwen_model"):
+ self.assertNotIn(model_type, by_type)
+
+ def test_minimax_fixture_can_share_colibri_without_becoming_glm(self):
+ config = {
+ "model_type": "minimax_m3",
+ "num_hidden_layers": 2,
+ "num_attention_heads": 4,
+ "num_key_value_heads": 2,
+ "head_dim": 8,
+ "num_local_experts": 4,
+ "num_experts_per_tok": 2,
+ "sparse_attention_config": {
+ "use_sparse_attention": True,
+ "sparse_index_dim": 8,
+ "sparse_attention_freq": [0, 1],
+ },
+ }
+ by_id, by_type = _build_registry(FAMILIES + (MINIMAX_M3_FIXTURE,))
+ family = by_type[config["model_type"]]
+ self.assertEqual(family.engine_artifact, by_id["glm"].engine_artifact)
+ self.assertEqual(family.engine_group, by_id["glm"].engine_group)
+ self.assertNotEqual(family.internal_arch, by_id["glm"].internal_arch)
+ resolved = type("R", (), {"descriptor": family, "family_config": config,
+ "model_dir": "."})()
+ geometry = planner_geometry(resolved, 32)
+ self.assertEqual(geometry.configured_experts, 4)
+ self.assertEqual(geometry.context_state_bytes, 13_312)
+
+ def test_unproven_production_planners_refuse_instead_of_inventing_zero(self):
+ for model_type in ("inkling", "kimi_k3", "olmoe", "deepseek_v4"):
+ family = family_for_config({"model_type": model_type})
+ resolved = type("R", (), {"descriptor": family, "family_config": {},
+ "model_dir": "."})()
+ with self.subTest(model_type=model_type), \
+ self.assertRaises(PlannerUnsupportedError):
+ planner_geometry(resolved, 32)
+
+ def test_cli_and_gateway_adapter_sets_equal_the_registry(self):
+ import openai_server
+ from importlib.machinery import SourceFileLoader
+ import importlib.util
+
+ cli_path = Path(__file__).resolve().parent.parent / "coli"
+ loader = SourceFileLoader("family_registry_cli_test", str(cli_path))
+ spec = importlib.util.spec_from_loader(loader.name, loader)
+ cli = importlib.util.module_from_spec(spec)
+ loader.exec_module(cli)
+
+ cli_adapters = {"glm", "inkling", "kimi", "olmoe", "deepseek_v4"}
+ gateway_adapters = {"glm", "inkling", "kimi", "olmoe", "deepseek_v4"}
+ self.assertEqual({family.cli_adapter for family in FAMILIES}, cli_adapters)
+ self.assertEqual({family.gateway_adapter for family in FAMILIES},
+ gateway_adapters)
+ self.assertEqual(set(openai_server.family_ids()),
+ {family.id for family in FAMILIES})
+ self.assertEqual({family.id for family in cli.all_families()},
+ {family.id for family in FAMILIES})
+
+ def test_doctor_reports_unknown_family_instead_of_falling_back(self):
+ from doctor import run_doctor
+
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ (root / "config.json").write_text(
+ json.dumps({"model_type": "qwen3_moe"}), encoding="utf-8")
+ (root / "tokenizer.json").write_text("{}", encoding="utf-8")
+ report = run_doctor(root, engine_path=root / "colibri",
+ available_memory=16_000_000_000,
+ available_disk=1, gpus=[],
+ linkage={"linked": False, "missing": False})
+ checks = {item["id"]: item for item in report["checks"]}
+ self.assertEqual(checks["model.family"]["status"], "fail")
+ self.assertIn("unsupported model_type", checks["model.family"]["summary"])
+ self.assertIsNone(report["plan"])
+
+ def test_doctor_reports_engine_capability_failure(self):
+ from doctor import run_doctor
+
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ (root / "config.json").write_text(
+ json.dumps({"model_type": "kimi_k3"}), encoding="utf-8")
+ (root / "tokenizer.json").write_text("{}", encoding="utf-8")
+ report = run_doctor(
+ root, engine_path=root / "colibri",
+ engine_error=UnknownFamilyError("this image contains only GLM"),
+ available_memory=16_000_000_000, available_disk=1, gpus=[],
+ linkage={"linked": False, "missing": False})
+ checks = {item["id"]: item for item in report["checks"]}
+ self.assertEqual(checks["engine.binary"]["status"], "fail")
+ self.assertEqual(report["status"], "error")
+
+ def test_build_install_ci_and_release_cover_registered_engines(self):
+ repo = Path(__file__).resolve().parents[2]
+ makefile = (repo / "c" / "Makefile").read_text(encoding="utf-8")
+ ci = (repo / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8")
+ release = (repo / ".github" / "workflows" / "release.yml").read_text(
+ encoding="utf-8")
+ docker = (repo / "docker" / "Dockerfile.slim").read_text(encoding="utf-8")
+ for family in FAMILIES:
+ with self.subTest(family=family.id):
+ self.assertRegex(
+ makefile,
+ rf"(?m)^{re.escape(family.build_target)}(?:\$\(EXE\))?:")
+ if family.id != "deepseek_v4":
+ self.assertIn(family.build_target,
+ re.search(r'ENGINES="([^"]+)"', ci).group(1).split())
+ self.assertIn(f"cp c/{family.engine_artifact}", release)
+ self.assertIn(f"$(LIBEXECDIR)/{family.engine_artifact}", makefile)
+ else:
+ self.assertIn("deepseek-v4", ci)
+ self.assertIn("cp c/deepseek_v4", release)
+ for text in (makefile, release, docker):
+ self.assertIn("family_registry.py", text)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_launcher_dispatch.py b/c/tests/test_launcher_dispatch.py
index 25bc7770..63443a45 100644
--- a/c/tests/test_launcher_dispatch.py
+++ b/c/tests/test_launcher_dispatch.py
@@ -31,6 +31,8 @@ import unittest
from pathlib import Path
from unittest import mock
+from family_registry import all_families
+
HERE = Path(__file__).resolve().parent.parent
CLI = HERE / "coli"
@@ -59,53 +61,40 @@ class LauncherDispatchTest(unittest.TestCase):
return str(root)
def test_every_named_model_type_is_classified(self):
- """A model_type in the banner roster must not fall through to the
- `return "glm"` default -- except glm itself, whose default that is."""
- for model_type, label, _ in self.cli._BANNER_MODELS:
- with self.subTest(model_type=model_type):
- arch = self.cli.model_arch(self._model_dir(model_type))
- if model_type == "glm":
- self.assertEqual(arch, "glm")
- continue
- self.assertNotEqual(
- arch, "glm",
- f"the banner prints {label!r} for model_type {model_type!r} "
- f"and model_arch() returns 'glm' -- the launcher names a "
- f"family it cannot dispatch (#879)")
+ for family in all_families():
+ for model_type in family.model_types:
+ with self.subTest(model_type=model_type):
+ self.assertEqual(self.cli.model_arch(self._model_dir(model_type)),
+ family.id)
def test_every_named_model_type_resolves_to_a_distinct_engine(self):
"""Two families must not resolve to the same binary. That is the shape
the bug took: everything unrecognised quietly became the GLM engine."""
- seen = {}
- for model_type, label, _ in self.cli._BANNER_MODELS:
+ for family in all_families():
+ model_type = family.model_types[0]
with self.subTest(model_type=model_type):
engine = os.path.basename(
self.cli.engine_for(self._model_dir(model_type)))
- self.assertTrue(engine, f"{label}: engine_for returned nothing")
- previous = seen.get(engine)
- self.assertIsNone(
- previous,
- f"{label} ({model_type}) and {previous} both dispatch to "
- f"{engine!r} -- one of them is being misrouted (#879)")
- seen[engine] = label
+ self.assertEqual(engine, family.engine_artifact + self.cli._EXE)
def test_engine_for_has_a_branch_for_every_arch_model_arch_can_return(self):
"""engine_for() indexes a dict with model_arch()'s return value. A value
model_arch can produce and the dict lacks is a KeyError at runtime, on
the user's machine, after the banner has already printed."""
- for model_type, label, _ in self.cli._BANNER_MODELS:
+ for family in all_families():
+ model_type = family.model_types[0]
with self.subTest(model_type=model_type):
try:
self.cli.engine_for(self._model_dir(model_type))
except KeyError as error:
- self.fail(f"{label}: engine_for raised KeyError({error}) -- "
+ self.fail(f"{family.display_name}: engine_for raised KeyError({error}) -- "
f"model_arch() classifies this family and the "
f"engine dict has no entry for it")
def test_the_roster_is_not_empty(self):
"""Guards the whole file: if _BANNER_MODELS is ever renamed or emptied,
the loops above pass vacuously and this suite proves nothing."""
- self.assertGreaterEqual(len(self.cli._BANNER_MODELS), 5)
+ self.assertGreaterEqual(len(all_families()), 5)
class ProjectPythonTest(unittest.TestCase):
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 911379c2..60f2a2ab 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -733,14 +733,14 @@ class CapSentinelShimTest(unittest.TestCase):
[executable, want],
f"arch={model_type} exe={executable} kwargs={kwargs}")
- def test_missing_or_unreadable_config_is_glm(self):
- # historic default: anything that cannot be classified is glm
+ def test_synthetic_engine_without_config_uses_explicit_arch(self):
self.assertEqual(self._spawn_argv("engine", "/nonexistent/model"),
["engine", "0"])
model = Path(self.tmp.name) / "model-broken"
model.mkdir()
(model / "config.json").write_text("{not json")
- self.assertEqual(self._spawn_argv("engine", str(model)), ["engine", "0"])
+ with self.assertRaisesRegex(ValueError, "invalid config.json"):
+ self._spawn_argv("engine", str(model))
def test_cap_for_arch_is_the_single_translation_point(self):
self.assertEqual(cap_for_arch("glm", None), 0)
@@ -758,7 +758,8 @@ class CapSentinelShimTest(unittest.TestCase):
self.assertEqual(model_arch(self._model("kimi_k3")), "kimi")
self.assertEqual(model_arch(self._model("deepseek_v4")), "deepseek_v4")
self.assertEqual(model_arch(self._model("olmoe")), "olmoe")
- self.assertEqual(model_arch("/nonexistent"), "glm")
+ with self.assertRaisesRegex(ValueError, "cannot read config.json"):
+ model_arch("/nonexistent")
def test_direct_v4_server_gets_bounded_dspark_defaults(self):
env = {"V4_MTP_CONF": "0.7"}
diff --git a/c/tests/test_openai_tools_e2e.py b/c/tests/test_openai_tools_e2e.py
index c6c43e69..fada942a 100644
--- a/c/tests/test_openai_tools_e2e.py
+++ b/c/tests/test_openai_tools_e2e.py
@@ -80,6 +80,8 @@ class ToolCallingE2E(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.TemporaryDirectory()
+ (Path(cls.tmp.name) / "config.json").write_text(
+ json.dumps({"model_type": "glm_moe_dsa"}), encoding="utf-8")
mock = Path(cls.tmp.name) / "mock_engine.py"
mock.write_text(MOCK_ENGINE)
mock.chmod(0o755)
diff --git a/c/tests/test_openai_tools_v4_e2e.py b/c/tests/test_openai_tools_v4_e2e.py
index 6c877162..36432f1d 100644
--- a/c/tests/test_openai_tools_v4_e2e.py
+++ b/c/tests/test_openai_tools_v4_e2e.py
@@ -73,6 +73,8 @@ class ToolCallingV4E2E(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.TemporaryDirectory()
+ (Path(cls.tmp.name) / "config.json").write_text(
+ json.dumps({"model_type": "deepseek_v4"}), encoding="utf-8")
mock = Path(cls.tmp.name) / "mock_engine.py"
mock.write_text(MOCK_ENGINE)
mock.chmod(0o755)
diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py
index a1aa44c5..4c3f2cf0 100644
--- a/c/tests/test_resource_plan.py
+++ b/c/tests/test_resource_plan.py
@@ -40,6 +40,7 @@ class ResourcePlanTest(unittest.TestCase):
self.tmp = tempfile.TemporaryDirectory()
self.model = Path(self.tmp.name)
(self.model / "config.json").write_text(json.dumps({
+ "model_type": "glm_moe_dsa",
"num_hidden_layers": 2,
"n_routed_experts": 2,
"kv_lora_rank": 4,
@@ -203,6 +204,36 @@ class ResourcePlanTest(unittest.TestCase):
self.assertIn("clamped", plan["warnings"][0])
self.assertIn("0:test-gpu", format_plan(plan))
+ def test_glm_kv_slots_scale_the_planned_state_pool(self):
+ one = build_plan(self.model, context=32, kv_slots=1,
+ available_memory=32 * GB, available_disk=1, gpus=[])
+ four = build_plan(self.model, context=32, kv_slots=4,
+ available_memory=32 * GB, available_disk=1, gpus=[])
+ self.assertEqual(four["tiers"]["ram"]["sequence_state_bytes"],
+ one["tiers"]["ram"]["sequence_state_bytes"])
+ delta = (four["tiers"]["ram"]["runtime_bytes"] -
+ one["tiers"]["ram"]["runtime_bytes"])
+ per_slot = (one["tiers"]["ram"]["sequence_state_bytes"] +
+ one["tiers"]["ram"]["fixed_state_bytes"])
+ self.assertEqual(delta, 3 * per_slot)
+
+ def test_glm_dsa_state_is_charged_only_when_every_indexer_weight_exists(self):
+ config = json.loads((self.model / "config.json").read_text())
+ config.update({"index_head_dim": 16,
+ "indexer_types": ["full", "shared"]})
+ (self.model / "config.json").write_text(json.dumps(config))
+ absent = build_plan(self.model, context=32, available_memory=32 * GB,
+ available_disk=1, gpus=[])
+ write_shard(self.model / "indexer.safetensors", [
+ ("model.layers.0.self_attn.indexer.wq_b.weight", 4),
+ ])
+ present = build_plan(self.model, context=32, available_memory=32 * GB,
+ available_disk=1, gpus=[])
+ self.assertEqual(
+ present["tiers"]["ram"]["sequence_state_bytes"] -
+ absent["tiers"]["ram"]["sequence_state_bytes"],
+ 1 * 32 * 16 * 4)
+
def test_unified_memory_uses_one_shared_pool(self):
gpus = [{"index": 0, "name": "NVIDIA GB10", "total_bytes": 130 * GB,
"free_bytes": 128 * GB, "unified_memory": True}]
@@ -643,6 +674,7 @@ memInfo.free: 23.50 GB (97%)
big = tempfile.TemporaryDirectory()
bigmodel = Path(big.name)
(bigmodel / "config.json").write_text(json.dumps({
+ "model_type": "glm_moe_dsa",
"num_hidden_layers": 2, "n_routed_experts": 4,
"kv_lora_rank": 4, "qk_rope_head_dim": 2,
"qk_nope_head_dim": 3, "v_head_dim": 5, "num_attention_heads": 2,
diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.py
index 61b4bcfe..4645a14b 100644
--- a/c/tests/test_v4_cli.py
+++ b/c/tests/test_v4_cli.py
@@ -153,12 +153,11 @@ class V4CliTest(unittest.TestCase):
env = self.cli.env_for_engine(args, "deepseek_v4")
self.assertEqual(env["NGEN"], "1024")
- def test_kimi_does_not_get_v4_only_settings(self):
- """The widening is RAM_GB alone; CTX and the V4 speculation defaults stay
- where they were."""
+ def test_kimi_context_uses_its_registered_environment_channel(self):
args = argparse.Namespace(ngen=8, temp=0.0, ram=242, ctx=4096)
env = self.cli.env_for_engine(args, "kimi")
self.assertNotIn("CTX", env)
+ self.assertEqual(env["K3_MAXT"], "4096")
self.assertNotIn("V4_MTP", env)
def test_windows_v4_run_passes_chinese_prompt_as_utf8_file(self):
diff --git a/docker/Dockerfile.slim b/docker/Dockerfile.slim
index aadb3604..96f4dc02 100644
--- a/docker/Dockerfile.slim
+++ b/docker/Dockerfile.slim
@@ -38,11 +38,12 @@ WORKDIR /app
# The engine binary plus the launcher and the files it resolves at runtime
# (version.py, the gateway, resource planner, doctor, tokenizer headers, tools).
COPY --from=build /src/c/colibri ./colibri
-COPY c/coli c/version.py c/openai_server.py c/v4_dsml.py c/resource_plan.py c/doctor.py c/autotune.py ./
+COPY c/coli c/version.py c/openai_server.py c/v4_dsml.py c/family_registry.py c/resource_plan.py c/doctor.py c/autotune.py ./
COPY c/tools/ ./tools/
ENV COLI_MODEL=/model \
COLI_ENGINE=/app/colibri \
+ COLI_DOCKER_GLM_ONLY=1 \
PYTHONUNBUFFERED=1
# A bind-mounted model lands here; declaring the volume documents the contract.
VOLUME ["/model"]