Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(convert): pick the converter from the checkpoint (#1368)
JustVuggd4d11efce39a
diff --git a/c/coli b/c/coliindex 6cbdeb2e..77f86a1c 100755--- a/c/coli+++ b/c/coli@@ -69,7 +69,8 @@ except ModuleNotFoundError:_version = "unknown"from family_registry import (FamilyConfigError, UnknownFamilyError, all_families,- family_by_id, resolve_model, tuning_replay_prompt)+ family_by_id, family_for_config, resolve_model,+ tuning_replay_prompt)_TUNE_ROTATION_PROMPT = ("A service has high median throughput but poor tail latency. Explain how "@@ -174,7 +175,13 @@ def model_banner_line(model):this runs before every command, and a banner may not cost a scan of a400 GB checkpoint."""- generic = "GLM-5.2 · 744B MoE · int4 · streaming CPU"+ # Senza un modello non si nomina un modello. Diceva "GLM-5.2 · 744B", che+ # era la famiglia di punta quando e' stata scritta e che dal #1367 non si+ # chiama nemmeno piu' cosi'; una riga che invecchia perche' cita un caso+ # invece di descrivere il programma. Il conto viene dal registry: aggiungere+ # una famiglia lo aggiorna, dimenticarsene non lo sbaglia.+ generic = (f"{len(_BANNER_MODELS)} model families · "+ f"MoE experts streamed from disk · CPU")if not model:return generictry:@@ -1948,6 +1955,38 @@ def cmd_bench(a):print(f" {C.dim}decode is disk-bound: this takes HOURS on slow hardware. Raise --limit on faster machines.{C.r}\n")sys.exit(subprocess.call(cmd, env=e))+#: I flag di `coli convert` che scelgono la precisione, e il loro nome sulla+#: riga di comando. Serve a sapere se l'utente li ha SCRITTI: un default non+#: passato al convertitore non fa danno, uno scritto e ignorato si'.+_CONVERT_PRECISION_FLAGS = {"ebits": "--ebits", "io_bits": "--io-bits",+ "xbits": "--xbits", "group_size": "--group-size"}+++def checkpoint_family(repo):+ """La famiglia del checkpoint, letta dal solo config.json.++ Cinque KB decidono quale convertitore ha senso per i prossimi 300 GB, ed e'+ lo stesso appiglio che usa la guardia dentro convert_fp8_to_int4.py.++ None quando non si riesce a stabilirlo -- rete assente, huggingface_hub non+ installato, repo senza config.json. In quel caso si prosegue come prima: la+ guardia del convertitore scarica lo stesso file e rifiuta lei. Meglio+ proseguire che fermare una conversione valida per un errore di rete.+ """+ try:+ local = os.path.join(os.path.expanduser(repo), "config.json")+ if os.path.isfile(local):+ with open(local, encoding="utf-8") as handle:+ return family_for_config(json.load(handle))+ from huggingface_hub import hf_hub_download+ with open(hf_hub_download(repo, "config.json"), encoding="utf-8") as handle:+ return family_for_config(json.load(handle))+ except SystemExit:+ raise+ except Exception:+ return None++def cmd_convert(a):banner("convert")# here --model is the DESTINATION the converted weights are written to, not an@@ -1957,15 +1996,54 @@ def cmd_convert(a):f" --model <dir> is where the converted model is written")# python con torch/safetensors: l'ambiente del progetto se c'e', altrimenti quello correntepy = project_python()- base=[py, os.path.join(TOOLS,"convert_fp8_to_int4.py"),- "--repo", a.repo, "--outdir", a.model, "--ebits", str(a.ebits), "--io-bits", str(a.io_bits),- "--group-size", str(a.group_size)]- if a.xbits: base+=["--xbits",str(a.xbits)]- # passo 1: modello principale (78 layer). Resumabile: riparte dagli shard mancanti.- print(f" {C.dim}[1/2] model: {' '.join(base)}{C.r}")- rc=subprocess.call(base)- if rc!=0: sys.exit(rc)- if a.no_mtp: sys.exit(0)++ # #1368: quale convertitore, deciso dal checkpoint e non dal fatto che GLM-5.2+ # e' arrivato per primo. Prima di questo, una GLM-5.3-Flash passava per il+ # convertitore di GLM-5.2, che le quantizzava l'embedding perche' il suo nome+ # annidato non corrisponde a nessuna delle sue regole; il contenitore usciva+ # senza un errore e moriva ore dopo, dentro `coli web`.+ family = checkpoint_family(a.repo)+ script = family.converter if family else "convert_fp8_to_int4.py"+ if family and not script:+ # Famiglia nota che coli non guida: il suo convertitore ha un'altra riga+ # di comando, o non serve. La guardia dentro convert_fp8_to_int4.py tiene+ # le indicazioni per famiglia, sotto test; le si lascia dire a lei+ # invece di tenerne una seconda copia qui.+ script = "convert_fp8_to_int4.py"+ if family:+ print(f" {C.dim}checkpoint: {family.display_name} -> tools/{script}{C.r}")+ written = [flag for name, flag in _CONVERT_PRECISION_FLAGS.items()+ if flag in sys.argv and name not in family.converter_accepts]+ if written and family.converter:+ sys.exit(f"{C.yel}{' '.join(written)}: tools/{script} does not take "+ f"{'them' if len(written) > 1 else 'it'}.{C.r}\n"+ f" {family.display_name} keeps its dense weights and embedding "+ f"wide and picks the precision when the engine loads them.\n"+ f" Re-run without {' '.join(written)}.")++ # Quando la famiglia non si e' potuta stabilire si passano tutte le opzioni,+ # cioe' esattamente il comando di prima: nessuna conversione che funzionava+ # cambia comportamento perche' la rete non ha risposto.+ accepts = family.converter_accepts if family and family.converter else \+ tuple(_CONVERT_PRECISION_FLAGS)+ base = [py, os.path.join(TOOLS, script), "--repo", a.repo, "--outdir", a.model]+ if "ebits" in accepts:+ base += ["--ebits", str(a.ebits), "--io-bits", str(a.io_bits)]+ if "group_size" in accepts:+ base += ["--group-size", str(a.group_size)]+ if a.xbits and "xbits" in accepts:+ base += ["--xbits", str(a.xbits)]++ mtp_pass = family.converter_mtp_pass if family and family.converter else True+ steps = 2 if (mtp_pass and not a.no_mtp) else 1+ print(f" {C.dim}[1/{steps}] model: {' '.join(base)}{C.r}")+ rc = subprocess.call(base)+ if rc != 0: sys.exit(rc)+ if steps == 1:+ # Non tutte le famiglie hanno una testa MTP da convertire a parte, e+ # convert_glm53.py non ha proprio --mtp: chiamarlo con quel flag e' un+ # errore di argomenti dopo che il modello e' gia' stato scritto.+ sys.exit(0)# passo 2: testa MTP (layer 78). SEMPRE int8: a int4 i draft sbagliano quasi sempre# (acceptance 0-4% vs 39-59%, misurato — issue #8) e la speculazione non parte mai.mtp_cmd=list(base); i=mtp_cmd.index("--ebits"); mtp_cmd[i+1]=str(max(8,a.ebits))diff --git a/c/family_registry.py b/c/family_registry.pyindex aa4cad62..b86d36d8 100644--- a/c/family_registry.py+++ b/c/family_registry.py@@ -89,6 +89,26 @@ class FamilyDescriptor:# separate from expert_inventory prevents a fixed allocation from# being multiplied by the cache capacity.fixed_resident_inventory: object = None+ # Lo script sotto tools/ che `coli convert` puo' guidare per questa famiglia,+ # e le opzioni di `coli convert` che quello script accetta davvero.+ #+ # #1368: `coli convert` lanciava convert_fp8_to_int4.py qualunque cosa gli si+ # desse. Quel convertitore e' di GLM-5.2 e classifica i tensori per nome+ # PIATTO, mentre GLM-5.3-Flash annida il testuale sotto il wrapper vision:+ # `model.language_model.embed_tokens.weight` non corrisponde a nessuna regola+ # e cade nel fallback finale, che lo quantizza. Poi glm53.c lo legge con+ # load_f32, rifiuta l'U8, e il motore muore dentro `coli web`.+ #+ # Vuoto significa "coli non guida questa famiglia": non e' una lacuna, e'+ # che il suo convertitore ha un'altra riga di comando (convert_qwen36.py usa+ # --out e --gs, convert_inkling_int4.py non ha --repo) o non serve affatto+ # (Qwen3.8 gira sul checkpoint ufficiale). In quel caso si lascia parlare la+ # guardia del convertitore, che quelle indicazioni le ha gia' per famiglia e+ # sotto test: due copie della stessa mappa sarebbero il difetto che questa+ # correzione sta chiudendo.+ converter: str = ""+ converter_accepts: tuple = ()+ converter_mtp_pass: bool = False@dataclass(frozen=True, slots=True)@@ -888,6 +908,12 @@ FAMILIES = (# export di solo testo dichiara "glm5_next_text" al primo livello.model_types=("glm5_next", "glm5_next_text"),display_name="GLM-5.3-Flash",+ # Niente ebits/io_bits/xbits: questo convertitore tiene i densi e+ # l'embedding in BF16 e la precisione la sceglie il motore a load time+ # (GLM53_BITS). Accettare quelle opzioni per poi ignorarle sarebbe la+ # versione silenziosa dello stesso guasto.+ converter="convert_glm53.py",+ converter_accepts=("group_size",),display_scale="321B",engine_artifact="glm53",engine_aliases=(),@@ -945,6 +971,9 @@ FAMILIES = (# family loads. If a future release ever adds a real discriminator,# split this then, on evidence.display_name="GLM-5.2/5.3",+ converter="convert_fp8_to_int4.py",+ converter_accepts=("ebits", "io_bits", "xbits", "group_size"),+ converter_mtp_pass=True,display_scale="744B",engine_artifact="colibri",engine_aliases=("glm",),diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.pyindex bf43b607..84a1cda3 100644--- a/c/tests/test_cli_output.py+++ b/c/tests/test_cli_output.py@@ -262,11 +262,22 @@ class BannerModelLineTest(unittest.TestCase):self.assertNotIn("GLM-5.2", line)def test_no_model_keeps_the_generic_tagline(self):- self.assertIn("GLM-5.2", self.coli.model_banner_line(None))+ """What matters is that a tagline comes back and that it names no+ model. It used to assert the substring "GLM-5.2", which pinned the+ example rather than the property: the tagline named the flagship+ family, and #1367 renamed that family out from under it."""+ line = self.coli.model_banner_line(None)+ self.assertTrue(line.strip())+ self.assertIn("model families", line)+ for family in self.coli.all_families():+ self.assertNotIn(family.display_name, line,+ "the generic tagline names a specific model; it is "+ "printed when no model was given")def test_unreadable_model_falls_back_instead_of_raising(self):"""`coli info` banners before validating the path; it must not crash."""- self.assertIn("GLM-5.2", self.coli.model_banner_line("/nonexistent/xyz"))+ self.assertEqual(self.coli.model_banner_line("/nonexistent/xyz"),+ self.coli.model_banner_line(None))def test_size_is_reported_without_rounding_to_zero(self):small = self.line({"model_type": "olmoe"}, shard_bytes=4_200_000_000)diff --git a/c/tests/test_convert_routing.py b/c/tests/test_convert_routing.pynew file mode 100644index 00000000..5231e50b--- /dev/null+++ b/c/tests/test_convert_routing.py@@ -0,0 +1,207 @@+"""`coli convert` must pick the converter from the checkpoint, not from history.++#1368: it ran `tools/convert_fp8_to_int4.py` on whatever it was given. That+converter is GLM-5.2's, and it classifies tensors by FLAT name:++ if name in ("model.embed_tokens.weight", "lm_head.weight"): return "io"++GLM-5.3-Flash nests its language model under the vision wrapper, so its+embedding is `model.language_model.embed_tokens.weight`. It matched no rule,+fell into the final `endswith(".weight")` fallback, and was quantized. Hours+later `glm53.c` read it with load_f32, refused the U8, and the engine died+inside `coli web` with a traceback about a subprocess exiting.++Nothing along that path was wrong except the first choice.++What the tests below hold:++- the checkpoint decides the converter, and a Flash checkpoint reaches+ convert_glm53.py;+- GLM-5.2's command is byte-for-byte what it was, because the fix must not+ quietly re-tune the path that was already right;+- an option the target converter does not take is REFUSED, not dropped.+ Silently ignoring `--ebits` on a converter that has no such flag is the same+ defect wearing better manners;+- when the family cannot be determined -- no network, no huggingface_hub -- the+ old command is used unchanged, so a working conversion never fails because a+ 5 KB metadata fetch did.+"""+import importlib.machinery+import importlib.util+import json+import tempfile+import unittest+from pathlib import Path+from unittest import mock++from family_registry import FAMILIES, family_by_id+++HERE = Path(__file__).resolve().parent.parent+CLI = HERE / "coli"+++def load_cli():+ loader = importlib.machinery.SourceFileLoader("coli_convert_test", str(CLI))+ spec = importlib.util.spec_from_loader(loader.name, loader)+ module = importlib.util.module_from_spec(spec)+ loader.exec_module(module)+ return module+++class Args:+ """The subset of `coli convert`'s namespace that cmd_convert reads."""++ def __init__(self, out, **overrides):+ self.repo = "some/checkpoint"+ self.model = out+ self.ebits = 4+ self.io_bits = 8+ self.xbits = 0+ self.group_size = 64+ self.no_mtp = False+ self.__dict__.update(overrides)+++class ConvertRoutingTest(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.cli = load_cli()++ def run_convert(self, model_type, argv=("coli", "convert"), **overrides):+ """cmd_convert with the subprocess replaced: returns the commands it+ would have run, or the SystemExit it raised instead."""+ directory = tempfile.TemporaryDirectory()+ self.addCleanup(directory.cleanup)+ calls = []++ def fake_call(command):+ calls.append(command)+ return 0++ with mock.patch.object(self.cli, "subprocess") as subprocess_module, \+ mock.patch.object(self.cli, "project_python", return_value="python3"), \+ mock.patch.object(self.cli, "sys") as system, \+ mock.patch.object(self.cli, "checkpoint_family",+ return_value=(family_by_id(model_type)+ if model_type else None)):+ subprocess_module.call = fake_call+ system.argv = list(argv)+ system.exit.side_effect = SystemExit # sys.exit must RAISE, not build+ try:+ self.cli.cmd_convert(Args(directory.name, **overrides))+ except (SystemExit, TypeError):+ # cmd_convert ends by calling sys.exit; the mock raises the class.+ pass+ return calls++ @staticmethod+ def script_of(command):+ return Path(command[1]).name++ def test_a_flash_checkpoint_reaches_its_own_converter(self):+ calls = self.run_convert("glm53")+ self.assertTrue(calls, "no converter was launched at all")+ self.assertEqual(self.script_of(calls[0]), "convert_glm53.py")++ def test_the_flash_command_carries_no_precision_flags(self):+ """convert_glm53.py has no --ebits/--io-bits/--xbits: it keeps the dense+ weights and the embedding wide and the engine picks the precision at+ load time. Passing them would be an argument error after the download."""+ command = self.run_convert("glm53")[0]+ for flag in ("--ebits", "--io-bits", "--xbits"):+ self.assertNotIn(flag, command)+ self.assertIn("--group-size", command)++ def test_flash_runs_one_pass_because_it_has_no_mtp_flag(self):+ calls = self.run_convert("glm53")+ self.assertEqual(len(calls), 1,+ "convert_glm53.py has no --mtp; a second pass is an "+ "argument error after the model is already written")++ def test_glm52_is_untouched(self):+ """The regression guard. This path was already correct and the whole+ point of the change is that it stays identical."""+ calls = self.run_convert("glm")+ self.assertEqual(self.script_of(calls[0]), "convert_fp8_to_int4.py")+ for flag in ("--repo", "--outdir", "--ebits", "--io-bits", "--group-size"):+ self.assertIn(flag, calls[0])+ self.assertEqual(len(calls), 2, "the int8 MTP pass must still run")+ self.assertIn("--mtp", calls[1])+ index = calls[1].index("--ebits")+ self.assertEqual(calls[1][index + 1], "8", "the MTP head is always int8")++ def test_no_mtp_still_skips_the_second_pass_for_glm52(self):+ calls = self.run_convert("glm", no_mtp=True)+ self.assertEqual(len(calls), 1)++ def test_an_option_the_target_does_not_take_is_refused_not_dropped(self):+ calls = self.run_convert(+ "glm53", argv=("coli", "convert", "--ebits", "3"), ebits=3)+ self.assertEqual(calls, [],+ "--ebits was silently dropped and the conversion ran "+ "anyway; the user asked for a precision they did not get")++ def test_an_unresolvable_family_keeps_the_old_command(self):+ """A metadata fetch that fails is not a reason to refuse a conversion+ that would have worked. The converter's own guard downloads the same+ config.json and refuses there if the family is genuinely wrong."""+ calls = self.run_convert(None)+ self.assertEqual(self.script_of(calls[0]), "convert_fp8_to_int4.py")+ for flag in ("--ebits", "--io-bits", "--group-size"):+ self.assertIn(flag, calls[0])+ self.assertEqual(len(calls), 2)+++class ConverterDeclarationTest(unittest.TestCase):+ """A family that names a converter must also say which options it takes.++ Half a declaration is worse than none: the dispatcher would build a command+ with no precision flags at all and the user would get defaults they never+ chose, silently, which is the shape of #1368 itself.+ """++ def test_every_declared_converter_states_what_it_accepts(self):+ for family in FAMILIES:+ if not family.converter:+ self.assertEqual(+ family.converter_accepts, (),+ f"{family.id}: accepts options for a converter it does not name")+ continue+ self.assertTrue(+ family.converter.endswith(".py"),+ f"{family.id}: converter must be a script under tools/")+ self.assertTrue(+ family.converter_accepts,+ f"{family.id}: names {family.converter} but says nothing about "+ f"which coli options it takes, so every one of them is dropped")++ def test_declared_converters_exist_on_disk(self):+ for family in FAMILIES:+ if not family.converter:+ continue+ path = HERE / "tools" / family.converter+ self.assertTrue(path.is_file(),+ f"{family.id} names tools/{family.converter}, "+ f"which is not in the tree")++ def test_a_declared_converter_really_takes_what_is_claimed(self):+ """Read the target's own argparse rather than trusting the declaration.+ The two drift the moment someone renames a flag, and the failure would+ land on a user mid-conversion."""+ spelling = {"ebits": "--ebits", "io_bits": "--io-bits",+ "xbits": "--xbits", "group_size": "--group-size"}+ for family in FAMILIES:+ if not family.converter:+ continue+ source = (HERE / "tools" / family.converter).read_text(+ encoding="utf-8", errors="ignore")+ for option in family.converter_accepts:+ flag = spelling[option]+ self.assertIn(f'add_argument("{flag}"', source,+ f"{family.id} claims tools/{family.converter} takes "+ f"{flag}, but that script does not define it")+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tools/pack_python.py b/c/tools/pack_python.pyindex a6699cf3..6147dc8e 100644--- a/c/tools/pack_python.py+++ b/c/tools/pack_python.py@@ -120,6 +120,34 @@ def data_files(path):return found+def named_tools(path, src):+ """Gli script sotto tools/ nominati da una stringa, non da una chiamata.++ #1368: i convertitori per famiglia sono dichiarati in family_registry.py+ come semplici stringhe ("convert_glm53.py"), perche' e' coli a comporre il+ comando a runtime. Non c'e' nessun `os.path.join(TOOLS, "...")` da trovare,+ quindi invoked_scripts() non li vede e l'archivio uscirebbe senza il+ convertitore -- che e' precisamente #1296, un giro piu' tardi.++ Come per i file di dati, si richiede che il file ESISTA sotto tools/: una+ stringa che somiglia a un nome di script ma non lo e' non produce nulla."""+ try:+ tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"))+ except SyntaxError:+ return set()+ found = set()+ for node in ast.walk(tree):+ if not (isinstance(node, ast.Constant) and isinstance(node.value, str)):+ continue+ name = node.value+ if not re.fullmatch(r"[a-z0-9_]+\.py", name):+ continue+ candidate = src / "tools" / name+ if candidate.is_file():+ found.add(candidate)+ return found++def needed(src):"""Tutto cio' che coli raggiunge, come percorsi sotto <c-dir>.@@ -143,6 +171,11 @@ def needed(src):while queue:path = queue.pop()data |= data_files(path)+ for candidate in named_tools(path, src):+ if candidate in script_paths:+ continue+ script_paths.add(candidate)+ queue.append(candidate)for script in invoked_scripts(path):if script in scripts:continue