Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
refactor(core): finish registry-owned family dispatch
Mario Gutierrez2ecb5d472434
diff --git a/c/coli b/c/coliindex e0c63d89..803a9be8 100755--- a/c/coli+++ b/c/coli@@ -69,7 +69,7 @@ except ModuleNotFoundError:_version = "unknown"from family_registry import (FamilyConfigError, UnknownFamilyError, all_families,- family_by_id, resolve_model)+ family_by_id, resolve_model, tuning_replay_prompt)# Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the# support modules (resource_plan.py, doctor.py, autotune.py, openai_server.py) and@@ -973,19 +973,15 @@ def cmd_tune(a):# family it then mis-dispatched.family=resolve_model(a.model).descriptorarch=family.id- if family.gateway_adapter not in ("glm","inkling","kimi","olmoe","deepseek_v4"):- sys.exit(f"{family.display_name}: gateway adapter {family.gateway_adapter!r} is not wired")+ if not family.has_gateway_adapter:+ sys.exit(f"{family.display_name}: gateway adapter is not wired")engine=engine_for(a.model)need_model(a.model,engine)base_env=env_for_engine(a,arch) if arch!="glm" else env_for(a)# The replay prompt has to be the engine's own template: GLM's [gMASK]<sop># markers are ordinary text to every other tokenizer, so the sweep would have# measured a different workload than the one users run.- prompt=({"glm": f"[gMASK]<sop><|user|>{a.prompt}<|assistant|><think></think>",- "inkling": f"<|user|>{a.prompt}<|assistant|>",- "kimi": f"K3CHAT1\nM user {len(a.prompt)}\n{a.prompt}G 0\n\n",- "olmoe": f"<|user|>\n{a.prompt}\n<|assistant|>\n",- }.get(arch, a.prompt))+ prompt=tuning_replay_prompt(family,a.prompt)banner("tune · measured execution profile", model=a.model)print(f" fixed replay: {a.tokens} tokens · {a.repeats} run(s) per candidate")print(" only quality-preserving scheduling knobs are eligible\n")@@ -1018,7 +1014,7 @@ def cmd_run(a):need_model(a.model,engine)prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"')banner("run", model=a.model)- if family.cli_adapter not in ("glm","deepseek_v4","olmoe"):+ if not family.has_cli_adapter:sys.exit(f"{C.yel}coli run is not wired for {family.display_name};{C.r} "f"use coli chat or coli serve")if arch=="deepseek_v4":@@ -1228,8 +1224,8 @@ def cmd_chat(a):need_model(a.model)family=resolve_model(a.model).descriptorarch=family.id- if family.gateway_adapter not in ("glm","inkling","kimi","olmoe","deepseek_v4"):- sys.exit(f"{family.display_name}: gateway adapter {family.gateway_adapter!r} is not wired")+ if not family.has_gateway_adapter:+ sys.exit(f"{family.display_name}: gateway adapter is not wired")if arch!="glm":engine=engine_for(a.model)need_model(a.model,engine)@@ -1421,8 +1417,8 @@ def cmd_serve(a):need_model(a.model)family=resolve_model(a.model).descriptorarch=family.id- if family.gateway_adapter not in ("glm","inkling","kimi","olmoe","deepseek_v4"):- sys.exit(f"{family.display_name}: gateway adapter {family.gateway_adapter!r} is not wired")+ if not family.has_gateway_adapter:+ sys.exit(f"{family.display_name}: gateway adapter is not wired")engine=engine_for(a.model)need_model(a.model,engine)if a.kv_slots>family.limits.max_kv_slots:diff --git a/c/family_registry.py b/c/family_registry.pyindex c72b037c..ee26c4c0 100644--- a/c/family_registry.py+++ b/c/family_registry.py@@ -72,6 +72,9 @@ class FamilyDescriptor:config_section: strlimits: FamilyLimitscapabilities: FamilyCapabilities+ has_gateway_adapter: bool = False+ has_cli_adapter: bool = False+ tune_prompt_template: str = "{prompt}"@dataclass(frozen=True, slots=True)@@ -163,43 +166,127 @@ COMMON_CAP = FamilyCapabilities(False, False, False, True)FAMILIES = (FamilyDescriptor(- "glm", ("glm_moe_dsa", "glm5_moe", "glm"), "GLM-5.2", "744B",- "colibri", ("glm",), "colibri-core", "glm", "colibri",- ("colibri", "glm"), "glm-5.2-colibri", "glm", "glm", "glm_mla",- _glm_geometry, "", _individual_expert_inventory(_GLM_EXPERT), "root",- FamilyLimits(4096, 1048576, 1024, 16384, 16, 0, "CTX"),- FamilyCapabilities(True, True, False, True)),+ id="glm",+ model_types=("glm_moe_dsa", "glm5_moe", "glm"),+ display_name="GLM-5.2",+ display_scale="744B",+ engine_artifact="colibri",+ engine_aliases=("glm",),+ engine_group="colibri-core",+ internal_arch="glm",+ build_target="colibri",+ process_names=("colibri", "glm"),+ default_model_id="glm-5.2-colibri",+ cli_adapter="glm",+ gateway_adapter="glm",+ planner_id="glm_mla",+ planner_geometry=_glm_geometry,+ planner_unsupported_reason="",+ expert_inventory=_individual_expert_inventory(_GLM_EXPERT),+ config_section="root",+ limits=FamilyLimits(4096, 1048576, 1024, 16384, 16, 0, "CTX"),+ capabilities=FamilyCapabilities(True, True, False, True),+ has_gateway_adapter=True,+ has_cli_adapter=True,+ tune_prompt_template="[gMASK]<sop><|user|>{prompt}<|assistant|><think></think>",+ ),FamilyDescriptor(- "inkling", ("inkling_mm_model", "inkling"), "Inkling", "975B",- "inkling", (), "inkling", "inkling", "inkling", ("inkling",),- "inkling-colibri", "inkling", "inkling", "inkling_hybrid", None,- "Inkling planning awaits a measured hybrid GQA/sconv runtime adapter",- _inkling_expert_inventory, "text_config",- FamilyLimits(8192, 1048576, 1024, 1024, 1, 8, "CTX_MAX"),- FamilyCapabilities(False, False, True, True)),+ id="inkling",+ model_types=("inkling_mm_model", "inkling"),+ display_name="Inkling",+ display_scale="975B",+ engine_artifact="inkling",+ engine_aliases=(),+ engine_group="inkling",+ internal_arch="inkling",+ build_target="inkling",+ process_names=("inkling",),+ default_model_id="inkling-colibri",+ cli_adapter="inkling",+ gateway_adapter="inkling",+ planner_id="inkling_hybrid",+ planner_geometry=None,+ planner_unsupported_reason="Inkling planning awaits a measured hybrid GQA/sconv runtime adapter",+ expert_inventory=_inkling_expert_inventory,+ config_section="text_config",+ limits=FamilyLimits(8192, 1048576, 1024, 1024, 1, 8, "CTX_MAX"),+ capabilities=FamilyCapabilities(False, False, True, True),+ has_gateway_adapter=True,+ tune_prompt_template="<|user|>{prompt}<|assistant|>",+ ),FamilyDescriptor(- "kimi", ("kimi_k3",), "Kimi K3", "2.8T", "kimi_k3", (), "kimi_k3",- "kimi_k3", "kimi_k3", ("kimi_k3",), "kimi-k3-colibri", "kimi", "kimi",- "kimi_hybrid", None,- "Kimi planning awaits measured KDA recurrent-state and native MXFP4 reserves",- _individual_expert_inventory(_KIMI_EXPERT), "text_config",- FamilyLimits(8192, 1048576, 1024, 1024, 1, 8, "K3_MAXT"), COMMON_CAP),+ id="kimi",+ model_types=("kimi_k3",),+ display_name="Kimi K3",+ display_scale="2.8T",+ engine_artifact="kimi_k3",+ engine_aliases=(),+ engine_group="kimi_k3",+ internal_arch="kimi_k3",+ build_target="kimi_k3",+ process_names=("kimi_k3",),+ default_model_id="kimi-k3-colibri",+ cli_adapter="kimi",+ gateway_adapter="kimi",+ planner_id="kimi_hybrid",+ planner_geometry=None,+ planner_unsupported_reason="Kimi planning awaits measured KDA recurrent-state and native MXFP4 reserves",+ expert_inventory=_individual_expert_inventory(_KIMI_EXPERT),+ config_section="text_config",+ limits=FamilyLimits(8192, 1048576, 1024, 1024, 1, 8, "K3_MAXT"),+ capabilities=COMMON_CAP,+ has_gateway_adapter=True,+ tune_prompt_template="K3CHAT1\nM user {prompt_len}\n{prompt}G 0\n\n",+ ),FamilyDescriptor(- "olmoe", ("olmoe",), "OLMoE", "7B", "olmoe", (), "olmoe", "olmoe",- "olmoe", ("olmoe",), "olmoe-colibri", "olmoe", "olmoe", "olmoe_gqa",- None, "OLMoE planning awaits a measured runtime/workspace reserve",- _individual_expert_inventory(_GLM_EXPERT), "root",- FamilyLimits(4096, 4096, 1024, 1024, 1, 8, "CTX"),- FamilyCapabilities(False, False, False, False)),+ id="olmoe",+ model_types=("olmoe",),+ display_name="OLMoE",+ display_scale="7B",+ engine_artifact="olmoe",+ engine_aliases=(),+ engine_group="olmoe",+ internal_arch="olmoe",+ build_target="olmoe",+ process_names=("olmoe",),+ default_model_id="olmoe-colibri",+ cli_adapter="olmoe",+ gateway_adapter="olmoe",+ planner_id="olmoe_gqa",+ planner_geometry=None,+ planner_unsupported_reason="OLMoE planning awaits a measured runtime/workspace reserve",+ expert_inventory=_individual_expert_inventory(_GLM_EXPERT),+ config_section="root",+ limits=FamilyLimits(4096, 4096, 1024, 1024, 1, 8, "CTX"),+ capabilities=FamilyCapabilities(False, False, False, False),+ has_gateway_adapter=True,+ has_cli_adapter=True,+ tune_prompt_template="<|user|>\n{prompt}\n<|assistant|>\n",+ ),FamilyDescriptor(- "deepseek_v4", ("deepseek_v4",), "DeepSeek V4 Flash", "284B",- "deepseek_v4", (), "deepseek_v4", "deepseek_v4", "deepseek-v4",- ("deepseek_v4",), "deepseek-v4-colibri", "deepseek_v4", "deepseek_v4",- "deepseek_v4", None,- "DeepSeek V4 uses its C resident-tier planner; Python parity is not yet proven",- _individual_expert_inventory(_V4_EXPERT), "root",- FamilyLimits(4096, 1048576, 1024, 16384, 1, 8, "CTX"),- FamilyCapabilities(True, False, False, True)),+ id="deepseek_v4",+ model_types=("deepseek_v4",),+ display_name="DeepSeek V4 Flash",+ display_scale="284B",+ engine_artifact="deepseek_v4",+ engine_aliases=(),+ engine_group="deepseek_v4",+ internal_arch="deepseek_v4",+ build_target="deepseek-v4",+ process_names=("deepseek_v4",),+ default_model_id="deepseek-v4-colibri",+ cli_adapter="deepseek_v4",+ gateway_adapter="deepseek_v4",+ planner_id="deepseek_v4",+ planner_geometry=None,+ planner_unsupported_reason="DeepSeek V4 uses its C resident-tier planner; Python parity is not yet proven",+ expert_inventory=_individual_expert_inventory(_V4_EXPERT),+ config_section="root",+ limits=FamilyLimits(4096, 1048576, 1024, 16384, 1, 8, "CTX"),+ capabilities=FamilyCapabilities(True, False, False, True),+ has_gateway_adapter=True,+ has_cli_adapter=True,+ ),)@@ -212,9 +299,17 @@ def _build_registry(families):raise RegistryError(f"invalid or duplicate family id: {family.id!r}")if (not family.model_types or(not callable(family.planner_geometry) and- not family.planner_unsupported_reason) or- not callable(family.expert_inventory)):+ not family.planner_unsupported_reason) or+ not callable(family.expert_inventory) or+ not isinstance(family.has_gateway_adapter, bool) or+ not isinstance(family.has_cli_adapter, bool) or+ not isinstance(family.tune_prompt_template, str) or+ "{prompt}" not in family.tune_prompt_template):raise RegistryError(f"incomplete family descriptor: {family.id}")+ try:+ family.tune_prompt_template.format(prompt="test", prompt_len=4)+ except (AttributeError, IndexError, KeyError, TypeError, ValueError) as error:+ raise RegistryError(f"invalid tune prompt template: {family.id}") from erroridentity = (family.engine_artifact, family.internal_arch)if identity in identities:raise RegistryError(f"duplicate engine identity: {identity}")@@ -266,6 +361,12 @@ def family_for_config(config):raise UnknownFamilyError(f"unsupported model_type: {model_type}") from error+def tuning_replay_prompt(family, prompt):+ if not isinstance(prompt, str):+ raise ValueError("tuning prompt must be a string")+ return family.tune_prompt_template.format(prompt=prompt, prompt_len=len(prompt))++def resolve_model(model_dir):model = Path(model_dir).expanduser().resolve()path = model / "config.json"diff --git a/c/tests/test_family_registry.py b/c/tests/test_family_registry.pyindex 41bfec8d..190ae257 100644--- a/c/tests/test_family_registry.py+++ b/c/tests/test_family_registry.py@@ -2,7 +2,9 @@ import jsonimport reimport tempfileimport unittest+from dataclasses import replacefrom pathlib import Path+from unittest import mockfrom family_registry import (FAMILIES,@@ -11,6 +13,7 @@ from family_registry import (FamilyDescriptor,FamilyLimits,PlannerGeometry,+ RegistryError,UnknownFamilyError,PlannerUnsupportedError,_build_registry,@@ -18,6 +21,7 @@ from family_registry import (planner_geometry,public_metadata,resolve_model,+ tuning_replay_prompt,)@@ -45,17 +49,49 @@ def minimax_geometry(config, context, _model_dir):TEST_INVENTORY = lambda _name, _size, _config: ()QWEN36_FIXTURE = FamilyDescriptor(- "qwen36", ("qwen3_5_moe_text",), "Qwen3.6", "", "qwen36", (), "qwen36",- "qwen36", "qwen36", ("qwen36",), "qwen3.6-colibri", "qwen36", "qwen36",- "qwen36_hybrid", qwen_geometry, "", TEST_INVENTORY, "root",- FamilyLimits(8192, 262144, 1024, 8192, 1, 8, "Q36_MAXT"),- FamilyCapabilities(False, False, False, True))+ id="qwen36",+ model_types=("qwen3_5_moe_text",),+ display_name="Qwen3.6",+ display_scale="",+ engine_artifact="qwen36",+ engine_aliases=(),+ engine_group="qwen36",+ internal_arch="qwen36",+ build_target="qwen36",+ process_names=("qwen36",),+ default_model_id="qwen3.6-colibri",+ cli_adapter="qwen36",+ gateway_adapter="qwen36",+ planner_id="qwen36_hybrid",+ planner_geometry=qwen_geometry,+ planner_unsupported_reason="",+ expert_inventory=TEST_INVENTORY,+ config_section="root",+ limits=FamilyLimits(8192, 262144, 1024, 8192, 1, 8, "Q36_MAXT"),+ capabilities=FamilyCapabilities(False, False, False, True),+)MINIMAX_M3_FIXTURE = FamilyDescriptor(- "minimax_m3", ("minimax_m3",), "MiniMax M3", "", "colibri", (),- "colibri-core", "minimax_m3", "colibri", ("colibri",), "minimax-m3-colibri",- "minimax_m3", "minimax_m3", "minimax_m3_gqa", minimax_geometry, "",- TEST_INVENTORY, "root", FamilyLimits(8192, 262144, 1024, 8192, 1, 8, "CTX"),- FamilyCapabilities(True, False, False, True))+ id="minimax_m3",+ model_types=("minimax_m3",),+ display_name="MiniMax M3",+ display_scale="",+ engine_artifact="colibri",+ engine_aliases=(),+ engine_group="colibri-core",+ internal_arch="minimax_m3",+ build_target="colibri",+ process_names=("colibri",),+ default_model_id="minimax-m3-colibri",+ cli_adapter="minimax_m3",+ gateway_adapter="minimax_m3",+ planner_id="minimax_m3_gqa",+ planner_geometry=minimax_geometry,+ planner_unsupported_reason="",+ expert_inventory=TEST_INVENTORY,+ config_section="root",+ limits=FamilyLimits(8192, 262144, 1024, 8192, 1, 8, "CTX"),+ capabilities=FamilyCapabilities(True, False, False, True),+)class FamilyRegistryTest(unittest.TestCase):@@ -147,7 +183,7 @@ class FamilyRegistryTest(unittest.TestCase):self.assertRaises(PlannerUnsupportedError):planner_geometry(resolved, 32)- def test_cli_and_gateway_adapter_sets_equal_the_registry(self):+ def test_cli_and_gateway_dispatch_follow_the_registry(self):import openai_serverfrom importlib.machinery import SourceFileLoaderimport importlib.util@@ -158,16 +194,53 @@ class FamilyRegistryTest(unittest.TestCase):cli = importlib.util.module_from_spec(spec)loader.exec_module(cli)- cli_adapters = {"glm", "inkling", "kimi", "olmoe", "deepseek_v4"}- gateway_adapters = {"glm", "inkling", "kimi", "olmoe", "deepseek_v4"}- self.assertEqual({family.cli_adapter for family in FAMILIES}, cli_adapters)- self.assertEqual({family.gateway_adapter for family in FAMILIES},- gateway_adapters)+ source = cli_path.read_text(encoding="utf-8")+ self.assertNotRegex(source, r"family\.(?:cli|gateway)_adapter\s+not in")+ self.assertNotIn("K3CHAT1", source)+ self.assertEqual(source.count("if not family.has_gateway_adapter:"), 3)+ self.assertEqual(source.count("if not family.has_cli_adapter:"), 1)self.assertEqual(set(openai_server.family_ids()),{family.id for family in FAMILIES})self.assertEqual({family.id for family in cli.all_families()},{family.id for family in FAMILIES})+ resolved = type("R", (), {"descriptor": replace(+ FAMILIES[0], has_cli_adapter=False, has_gateway_adapter=False)})()+ args = type("A", (), {"model": ".", "prompt": ["hello"],+ "no_attach": True})()+ with mock.patch.object(cli, "need_model"), \+ mock.patch.object(cli, "resolve_model", return_value=resolved), \+ mock.patch.object(cli, "engine_for", return_value="engine"), \+ mock.patch.object(cli, "banner"):+ with self.assertRaisesRegex(SystemExit, "coli run is not wired"):+ cli.cmd_run(args)+ with self.assertRaisesRegex(SystemExit, "gateway adapter is not wired"):+ cli.cmd_chat(args)++ def test_tuning_replay_prompts_are_registry_owned(self):+ prompt = "hello {world}"+ expected = {+ "glm": "[gMASK]<sop><|user|>hello {world}<|assistant|><think></think>",+ "inkling": "<|user|>hello {world}<|assistant|>",+ "kimi": "K3CHAT1\nM user 13\nhello {world}G 0\n\n",+ "olmoe": "<|user|>\nhello {world}\n<|assistant|>\n",+ "deepseek_v4": "hello {world}",+ }+ self.assertEqual(+ {family.id: tuning_replay_prompt(family, prompt) for family in FAMILIES},+ expected,+ )++ def test_optional_adapters_and_prompt_template_are_registry_invariants(self):+ self.assertFalse(QWEN36_FIXTURE.has_cli_adapter)+ self.assertFalse(QWEN36_FIXTURE.has_gateway_adapter)+ self.assertEqual(tuning_replay_prompt(QWEN36_FIXTURE, "hello"), "hello")++ for template in ("static", "{unknown}", "{prompt", "{prompt[foo]}"):+ with self.subTest(template=template), self.assertRaises(RegistryError):+ _build_registry((replace(QWEN36_FIXTURE,+ tune_prompt_template=template),))+def test_doctor_reports_unknown_family_instead_of_falling_back(self):from doctor import run_doctor