Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

refactor(core): finish registry-owned family dispatch

Mario Gutierrez2ecb5d472434
diff --git a/c/coli b/c/coli
index e0c63d89..803a9be8 100755
--- a/c/coli
+++ b/c/coli
@@ -69,7 +69,7 @@ except ModuleNotFoundError:
_version = "unknown"
from family_registry import (FamilyConfigError, UnknownFamilyError, all_families,
- family_by_id, resolve_model)
+ family_by_id, resolve_model, tuning_replay_prompt)
# Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the
# support modules (resource_plan.py, doctor.py, autotune.py, openai_server.py) and
@@ -973,19 +973,15 @@ def cmd_tune(a):
# family it then mis-dispatched.
family=resolve_model(a.model).descriptor
arch=family.id
- if family.gateway_adapter not in ("glm","inkling","kimi","olmoe","deepseek_v4"):
- sys.exit(f"{family.display_name}: gateway adapter {family.gateway_adapter!r} is not wired")
+ if not family.has_gateway_adapter:
+ sys.exit(f"{family.display_name}: gateway adapter is not wired")
engine=engine_for(a.model)
need_model(a.model,engine)
base_env=env_for_engine(a,arch) if arch!="glm" else env_for(a)
# The replay prompt has to be the engine's own template: GLM's [gMASK]<sop>
# markers are ordinary text to every other tokenizer, so the sweep would have
# measured a different workload than the one users run.
- prompt=({"glm": f"[gMASK]<sop><|user|>{a.prompt}<|assistant|><think></think>",
- "inkling": f"<|user|>{a.prompt}<|assistant|>",
- "kimi": f"K3CHAT1\nM user {len(a.prompt)}\n{a.prompt}G 0\n\n",
- "olmoe": f"<|user|>\n{a.prompt}\n<|assistant|>\n",
- }.get(arch, a.prompt))
+ prompt=tuning_replay_prompt(family,a.prompt)
banner("tune · measured execution profile", model=a.model)
print(f" fixed replay: {a.tokens} tokens · {a.repeats} run(s) per candidate")
print(" only quality-preserving scheduling knobs are eligible\n")
@@ -1018,7 +1014,7 @@ def cmd_run(a):
need_model(a.model,engine)
prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"')
banner("run", model=a.model)
- if family.cli_adapter not in ("glm","deepseek_v4","olmoe"):
+ if not family.has_cli_adapter:
sys.exit(f"{C.yel}coli run is not wired for {family.display_name};{C.r} "
f"use coli chat or coli serve")
if arch=="deepseek_v4":
@@ -1228,8 +1224,8 @@ def cmd_chat(a):
need_model(a.model)
family=resolve_model(a.model).descriptor
arch=family.id
- if family.gateway_adapter not in ("glm","inkling","kimi","olmoe","deepseek_v4"):
- sys.exit(f"{family.display_name}: gateway adapter {family.gateway_adapter!r} is not wired")
+ if not family.has_gateway_adapter:
+ sys.exit(f"{family.display_name}: gateway adapter is not wired")
if arch!="glm":
engine=engine_for(a.model)
need_model(a.model,engine)
@@ -1421,8 +1417,8 @@ def cmd_serve(a):
need_model(a.model)
family=resolve_model(a.model).descriptor
arch=family.id
- if family.gateway_adapter not in ("glm","inkling","kimi","olmoe","deepseek_v4"):
- sys.exit(f"{family.display_name}: gateway adapter {family.gateway_adapter!r} is not wired")
+ if not family.has_gateway_adapter:
+ sys.exit(f"{family.display_name}: gateway adapter is not wired")
engine=engine_for(a.model)
need_model(a.model,engine)
if a.kv_slots>family.limits.max_kv_slots:
diff --git a/c/family_registry.py b/c/family_registry.py
index c72b037c..ee26c4c0 100644
--- a/c/family_registry.py
+++ b/c/family_registry.py
@@ -72,6 +72,9 @@ class FamilyDescriptor:
config_section: str
limits: FamilyLimits
capabilities: FamilyCapabilities
+ has_gateway_adapter: bool = False
+ has_cli_adapter: bool = False
+ tune_prompt_template: str = "{prompt}"
@dataclass(frozen=True, slots=True)
@@ -163,43 +166,127 @@ COMMON_CAP = FamilyCapabilities(False, False, False, True)
FAMILIES = (
FamilyDescriptor(
- "glm", ("glm_moe_dsa", "glm5_moe", "glm"), "GLM-5.2", "744B",
- "colibri", ("glm",), "colibri-core", "glm", "colibri",
- ("colibri", "glm"), "glm-5.2-colibri", "glm", "glm", "glm_mla",
- _glm_geometry, "", _individual_expert_inventory(_GLM_EXPERT), "root",
- FamilyLimits(4096, 1048576, 1024, 16384, 16, 0, "CTX"),
- FamilyCapabilities(True, True, False, True)),
+ id="glm",
+ model_types=("glm_moe_dsa", "glm5_moe", "glm"),
+ display_name="GLM-5.2",
+ display_scale="744B",
+ engine_artifact="colibri",
+ engine_aliases=("glm",),
+ engine_group="colibri-core",
+ internal_arch="glm",
+ build_target="colibri",
+ process_names=("colibri", "glm"),
+ default_model_id="glm-5.2-colibri",
+ cli_adapter="glm",
+ gateway_adapter="glm",
+ planner_id="glm_mla",
+ planner_geometry=_glm_geometry,
+ planner_unsupported_reason="",
+ expert_inventory=_individual_expert_inventory(_GLM_EXPERT),
+ config_section="root",
+ limits=FamilyLimits(4096, 1048576, 1024, 16384, 16, 0, "CTX"),
+ capabilities=FamilyCapabilities(True, True, False, True),
+ has_gateway_adapter=True,
+ has_cli_adapter=True,
+ tune_prompt_template="[gMASK]<sop><|user|>{prompt}<|assistant|><think></think>",
+ ),
FamilyDescriptor(
- "inkling", ("inkling_mm_model", "inkling"), "Inkling", "975B",
- "inkling", (), "inkling", "inkling", "inkling", ("inkling",),
- "inkling-colibri", "inkling", "inkling", "inkling_hybrid", None,
- "Inkling planning awaits a measured hybrid GQA/sconv runtime adapter",
- _inkling_expert_inventory, "text_config",
- FamilyLimits(8192, 1048576, 1024, 1024, 1, 8, "CTX_MAX"),
- FamilyCapabilities(False, False, True, True)),
+ id="inkling",
+ model_types=("inkling_mm_model", "inkling"),
+ display_name="Inkling",
+ display_scale="975B",
+ engine_artifact="inkling",
+ engine_aliases=(),
+ engine_group="inkling",
+ internal_arch="inkling",
+ build_target="inkling",
+ process_names=("inkling",),
+ default_model_id="inkling-colibri",
+ cli_adapter="inkling",
+ gateway_adapter="inkling",
+ planner_id="inkling_hybrid",
+ planner_geometry=None,
+ planner_unsupported_reason="Inkling planning awaits a measured hybrid GQA/sconv runtime adapter",
+ expert_inventory=_inkling_expert_inventory,
+ config_section="text_config",
+ limits=FamilyLimits(8192, 1048576, 1024, 1024, 1, 8, "CTX_MAX"),
+ capabilities=FamilyCapabilities(False, False, True, True),
+ has_gateway_adapter=True,
+ tune_prompt_template="<|user|>{prompt}<|assistant|>",
+ ),
FamilyDescriptor(
- "kimi", ("kimi_k3",), "Kimi K3", "2.8T", "kimi_k3", (), "kimi_k3",
- "kimi_k3", "kimi_k3", ("kimi_k3",), "kimi-k3-colibri", "kimi", "kimi",
- "kimi_hybrid", None,
- "Kimi planning awaits measured KDA recurrent-state and native MXFP4 reserves",
- _individual_expert_inventory(_KIMI_EXPERT), "text_config",
- FamilyLimits(8192, 1048576, 1024, 1024, 1, 8, "K3_MAXT"), COMMON_CAP),
+ id="kimi",
+ model_types=("kimi_k3",),
+ display_name="Kimi K3",
+ display_scale="2.8T",
+ engine_artifact="kimi_k3",
+ engine_aliases=(),
+ engine_group="kimi_k3",
+ internal_arch="kimi_k3",
+ build_target="kimi_k3",
+ process_names=("kimi_k3",),
+ default_model_id="kimi-k3-colibri",
+ cli_adapter="kimi",
+ gateway_adapter="kimi",
+ planner_id="kimi_hybrid",
+ planner_geometry=None,
+ planner_unsupported_reason="Kimi planning awaits measured KDA recurrent-state and native MXFP4 reserves",
+ expert_inventory=_individual_expert_inventory(_KIMI_EXPERT),
+ config_section="text_config",
+ limits=FamilyLimits(8192, 1048576, 1024, 1024, 1, 8, "K3_MAXT"),
+ capabilities=COMMON_CAP,
+ has_gateway_adapter=True,
+ tune_prompt_template="K3CHAT1\nM user {prompt_len}\n{prompt}G 0\n\n",
+ ),
FamilyDescriptor(
- "olmoe", ("olmoe",), "OLMoE", "7B", "olmoe", (), "olmoe", "olmoe",
- "olmoe", ("olmoe",), "olmoe-colibri", "olmoe", "olmoe", "olmoe_gqa",
- None, "OLMoE planning awaits a measured runtime/workspace reserve",
- _individual_expert_inventory(_GLM_EXPERT), "root",
- FamilyLimits(4096, 4096, 1024, 1024, 1, 8, "CTX"),
- FamilyCapabilities(False, False, False, False)),
+ id="olmoe",
+ model_types=("olmoe",),
+ display_name="OLMoE",
+ display_scale="7B",
+ engine_artifact="olmoe",
+ engine_aliases=(),
+ engine_group="olmoe",
+ internal_arch="olmoe",
+ build_target="olmoe",
+ process_names=("olmoe",),
+ default_model_id="olmoe-colibri",
+ cli_adapter="olmoe",
+ gateway_adapter="olmoe",
+ planner_id="olmoe_gqa",
+ planner_geometry=None,
+ planner_unsupported_reason="OLMoE planning awaits a measured runtime/workspace reserve",
+ expert_inventory=_individual_expert_inventory(_GLM_EXPERT),
+ config_section="root",
+ limits=FamilyLimits(4096, 4096, 1024, 1024, 1, 8, "CTX"),
+ capabilities=FamilyCapabilities(False, False, False, False),
+ has_gateway_adapter=True,
+ has_cli_adapter=True,
+ tune_prompt_template="<|user|>\n{prompt}\n<|assistant|>\n",
+ ),
FamilyDescriptor(
- "deepseek_v4", ("deepseek_v4",), "DeepSeek V4 Flash", "284B",
- "deepseek_v4", (), "deepseek_v4", "deepseek_v4", "deepseek-v4",
- ("deepseek_v4",), "deepseek-v4-colibri", "deepseek_v4", "deepseek_v4",
- "deepseek_v4", None,
- "DeepSeek V4 uses its C resident-tier planner; Python parity is not yet proven",
- _individual_expert_inventory(_V4_EXPERT), "root",
- FamilyLimits(4096, 1048576, 1024, 16384, 1, 8, "CTX"),
- FamilyCapabilities(True, False, False, True)),
+ id="deepseek_v4",
+ model_types=("deepseek_v4",),
+ display_name="DeepSeek V4 Flash",
+ display_scale="284B",
+ engine_artifact="deepseek_v4",
+ engine_aliases=(),
+ engine_group="deepseek_v4",
+ internal_arch="deepseek_v4",
+ build_target="deepseek-v4",
+ process_names=("deepseek_v4",),
+ default_model_id="deepseek-v4-colibri",
+ cli_adapter="deepseek_v4",
+ gateway_adapter="deepseek_v4",
+ planner_id="deepseek_v4",
+ planner_geometry=None,
+ planner_unsupported_reason="DeepSeek V4 uses its C resident-tier planner; Python parity is not yet proven",
+ expert_inventory=_individual_expert_inventory(_V4_EXPERT),
+ config_section="root",
+ limits=FamilyLimits(4096, 1048576, 1024, 16384, 1, 8, "CTX"),
+ capabilities=FamilyCapabilities(True, False, False, True),
+ has_gateway_adapter=True,
+ has_cli_adapter=True,
+ ),
)
@@ -212,9 +299,17 @@ def _build_registry(families):
raise RegistryError(f"invalid or duplicate family id: {family.id!r}")
if (not family.model_types or
(not callable(family.planner_geometry) and
- not family.planner_unsupported_reason) or
- not callable(family.expert_inventory)):
+ not family.planner_unsupported_reason) or
+ not callable(family.expert_inventory) or
+ not isinstance(family.has_gateway_adapter, bool) or
+ not isinstance(family.has_cli_adapter, bool) or
+ not isinstance(family.tune_prompt_template, str) or
+ "{prompt}" not in family.tune_prompt_template):
raise RegistryError(f"incomplete family descriptor: {family.id}")
+ try:
+ family.tune_prompt_template.format(prompt="test", prompt_len=4)
+ except (AttributeError, IndexError, KeyError, TypeError, ValueError) as error:
+ raise RegistryError(f"invalid tune prompt template: {family.id}") from error
identity = (family.engine_artifact, family.internal_arch)
if identity in identities:
raise RegistryError(f"duplicate engine identity: {identity}")
@@ -266,6 +361,12 @@ def family_for_config(config):
raise UnknownFamilyError(f"unsupported model_type: {model_type}") from error
+def tuning_replay_prompt(family, prompt):
+ if not isinstance(prompt, str):
+ raise ValueError("tuning prompt must be a string")
+ return family.tune_prompt_template.format(prompt=prompt, prompt_len=len(prompt))
+
+
def resolve_model(model_dir):
model = Path(model_dir).expanduser().resolve()
path = model / "config.json"
diff --git a/c/tests/test_family_registry.py b/c/tests/test_family_registry.py
index 41bfec8d..190ae257 100644
--- a/c/tests/test_family_registry.py
+++ b/c/tests/test_family_registry.py
@@ -2,7 +2,9 @@ import json
import re
import tempfile
import unittest
+from dataclasses import replace
from pathlib import Path
+from unittest import mock
from family_registry import (
FAMILIES,
@@ -11,6 +13,7 @@ from family_registry import (
FamilyDescriptor,
FamilyLimits,
PlannerGeometry,
+ RegistryError,
UnknownFamilyError,
PlannerUnsupportedError,
_build_registry,
@@ -18,6 +21,7 @@ from family_registry import (
planner_geometry,
public_metadata,
resolve_model,
+ tuning_replay_prompt,
)
@@ -45,17 +49,49 @@ def minimax_geometry(config, context, _model_dir):
TEST_INVENTORY = lambda _name, _size, _config: ()
QWEN36_FIXTURE = FamilyDescriptor(
- "qwen36", ("qwen3_5_moe_text",), "Qwen3.6", "", "qwen36", (), "qwen36",
- "qwen36", "qwen36", ("qwen36",), "qwen3.6-colibri", "qwen36", "qwen36",
- "qwen36_hybrid", qwen_geometry, "", TEST_INVENTORY, "root",
- FamilyLimits(8192, 262144, 1024, 8192, 1, 8, "Q36_MAXT"),
- FamilyCapabilities(False, False, False, True))
+ id="qwen36",
+ model_types=("qwen3_5_moe_text",),
+ display_name="Qwen3.6",
+ display_scale="",
+ engine_artifact="qwen36",
+ engine_aliases=(),
+ engine_group="qwen36",
+ internal_arch="qwen36",
+ build_target="qwen36",
+ process_names=("qwen36",),
+ default_model_id="qwen3.6-colibri",
+ cli_adapter="qwen36",
+ gateway_adapter="qwen36",
+ planner_id="qwen36_hybrid",
+ planner_geometry=qwen_geometry,
+ planner_unsupported_reason="",
+ expert_inventory=TEST_INVENTORY,
+ config_section="root",
+ limits=FamilyLimits(8192, 262144, 1024, 8192, 1, 8, "Q36_MAXT"),
+ capabilities=FamilyCapabilities(False, False, False, True),
+)
MINIMAX_M3_FIXTURE = FamilyDescriptor(
- "minimax_m3", ("minimax_m3",), "MiniMax M3", "", "colibri", (),
- "colibri-core", "minimax_m3", "colibri", ("colibri",), "minimax-m3-colibri",
- "minimax_m3", "minimax_m3", "minimax_m3_gqa", minimax_geometry, "",
- TEST_INVENTORY, "root", FamilyLimits(8192, 262144, 1024, 8192, 1, 8, "CTX"),
- FamilyCapabilities(True, False, False, True))
+ id="minimax_m3",
+ model_types=("minimax_m3",),
+ display_name="MiniMax M3",
+ display_scale="",
+ engine_artifact="colibri",
+ engine_aliases=(),
+ engine_group="colibri-core",
+ internal_arch="minimax_m3",
+ build_target="colibri",
+ process_names=("colibri",),
+ default_model_id="minimax-m3-colibri",
+ cli_adapter="minimax_m3",
+ gateway_adapter="minimax_m3",
+ planner_id="minimax_m3_gqa",
+ planner_geometry=minimax_geometry,
+ planner_unsupported_reason="",
+ expert_inventory=TEST_INVENTORY,
+ config_section="root",
+ limits=FamilyLimits(8192, 262144, 1024, 8192, 1, 8, "CTX"),
+ capabilities=FamilyCapabilities(True, False, False, True),
+)
class FamilyRegistryTest(unittest.TestCase):
@@ -147,7 +183,7 @@ class FamilyRegistryTest(unittest.TestCase):
self.assertRaises(PlannerUnsupportedError):
planner_geometry(resolved, 32)
- def test_cli_and_gateway_adapter_sets_equal_the_registry(self):
+ def test_cli_and_gateway_dispatch_follow_the_registry(self):
import openai_server
from importlib.machinery import SourceFileLoader
import importlib.util
@@ -158,16 +194,53 @@ class FamilyRegistryTest(unittest.TestCase):
cli = importlib.util.module_from_spec(spec)
loader.exec_module(cli)
- cli_adapters = {"glm", "inkling", "kimi", "olmoe", "deepseek_v4"}
- gateway_adapters = {"glm", "inkling", "kimi", "olmoe", "deepseek_v4"}
- self.assertEqual({family.cli_adapter for family in FAMILIES}, cli_adapters)
- self.assertEqual({family.gateway_adapter for family in FAMILIES},
- gateway_adapters)
+ source = cli_path.read_text(encoding="utf-8")
+ self.assertNotRegex(source, r"family\.(?:cli|gateway)_adapter\s+not in")
+ self.assertNotIn("K3CHAT1", source)
+ self.assertEqual(source.count("if not family.has_gateway_adapter:"), 3)
+ self.assertEqual(source.count("if not family.has_cli_adapter:"), 1)
self.assertEqual(set(openai_server.family_ids()),
{family.id for family in FAMILIES})
self.assertEqual({family.id for family in cli.all_families()},
{family.id for family in FAMILIES})
+ resolved = type("R", (), {"descriptor": replace(
+ FAMILIES[0], has_cli_adapter=False, has_gateway_adapter=False)})()
+ args = type("A", (), {"model": ".", "prompt": ["hello"],
+ "no_attach": True})()
+ with mock.patch.object(cli, "need_model"), \
+ mock.patch.object(cli, "resolve_model", return_value=resolved), \
+ mock.patch.object(cli, "engine_for", return_value="engine"), \
+ mock.patch.object(cli, "banner"):
+ with self.assertRaisesRegex(SystemExit, "coli run is not wired"):
+ cli.cmd_run(args)
+ with self.assertRaisesRegex(SystemExit, "gateway adapter is not wired"):
+ cli.cmd_chat(args)
+
+ def test_tuning_replay_prompts_are_registry_owned(self):
+ prompt = "hello {world}"
+ expected = {
+ "glm": "[gMASK]<sop><|user|>hello {world}<|assistant|><think></think>",
+ "inkling": "<|user|>hello {world}<|assistant|>",
+ "kimi": "K3CHAT1\nM user 13\nhello {world}G 0\n\n",
+ "olmoe": "<|user|>\nhello {world}\n<|assistant|>\n",
+ "deepseek_v4": "hello {world}",
+ }
+ self.assertEqual(
+ {family.id: tuning_replay_prompt(family, prompt) for family in FAMILIES},
+ expected,
+ )
+
+ def test_optional_adapters_and_prompt_template_are_registry_invariants(self):
+ self.assertFalse(QWEN36_FIXTURE.has_cli_adapter)
+ self.assertFalse(QWEN36_FIXTURE.has_gateway_adapter)
+ self.assertEqual(tuning_replay_prompt(QWEN36_FIXTURE, "hello"), "hello")
+
+ for template in ("static", "{unknown}", "{prompt", "{prompt[foo]}"):
+ with self.subTest(template=template), self.assertRaises(RegistryError):
+ _build_registry((replace(QWEN36_FIXTURE,
+ tune_prompt_template=template),))
+
def test_doctor_reports_unknown_family_instead_of_falling_back(self):
from doctor import run_doctor