Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(convert): pick the converter from the checkpoint (#1368)

JustVuggd4d11efce39a
diff --git a/c/coli b/c/coli
index 6cbdeb2e..77f86a1c 100755
--- a/c/coli
+++ b/c/coli
@@ -69,7 +69,8 @@ except ModuleNotFoundError:
_version = "unknown"
from family_registry import (FamilyConfigError, UnknownFamilyError, all_families,
- family_by_id, resolve_model, tuning_replay_prompt)
+ family_by_id, family_for_config, resolve_model,
+ tuning_replay_prompt)
_TUNE_ROTATION_PROMPT = (
"A service has high median throughput but poor tail latency. Explain how "
@@ -174,7 +175,13 @@ def model_banner_line(model):
this runs before every command, and a banner may not cost a scan of a
400 GB checkpoint.
"""
- generic = "GLM-5.2 · 744B MoE · int4 · streaming CPU"
+ # Senza un modello non si nomina un modello. Diceva "GLM-5.2 · 744B", che
+ # era la famiglia di punta quando e' stata scritta e che dal #1367 non si
+ # chiama nemmeno piu' cosi'; una riga che invecchia perche' cita un caso
+ # invece di descrivere il programma. Il conto viene dal registry: aggiungere
+ # una famiglia lo aggiorna, dimenticarsene non lo sbaglia.
+ generic = (f"{len(_BANNER_MODELS)} model families · "
+ f"MoE experts streamed from disk · CPU")
if not model:
return generic
try:
@@ -1948,6 +1955,38 @@ def cmd_bench(a):
print(f" {C.dim}decode is disk-bound: this takes HOURS on slow hardware. Raise --limit on faster machines.{C.r}\n")
sys.exit(subprocess.call(cmd, env=e))
+#: I flag di `coli convert` che scelgono la precisione, e il loro nome sulla
+#: riga di comando. Serve a sapere se l'utente li ha SCRITTI: un default non
+#: passato al convertitore non fa danno, uno scritto e ignorato si'.
+_CONVERT_PRECISION_FLAGS = {"ebits": "--ebits", "io_bits": "--io-bits",
+ "xbits": "--xbits", "group_size": "--group-size"}
+
+
+def checkpoint_family(repo):
+ """La famiglia del checkpoint, letta dal solo config.json.
+
+ Cinque KB decidono quale convertitore ha senso per i prossimi 300 GB, ed e'
+ lo stesso appiglio che usa la guardia dentro convert_fp8_to_int4.py.
+
+ None quando non si riesce a stabilirlo -- rete assente, huggingface_hub non
+ installato, repo senza config.json. In quel caso si prosegue come prima: la
+ guardia del convertitore scarica lo stesso file e rifiuta lei. Meglio
+ proseguire che fermare una conversione valida per un errore di rete.
+ """
+ try:
+ local = os.path.join(os.path.expanduser(repo), "config.json")
+ if os.path.isfile(local):
+ with open(local, encoding="utf-8") as handle:
+ return family_for_config(json.load(handle))
+ from huggingface_hub import hf_hub_download
+ with open(hf_hub_download(repo, "config.json"), encoding="utf-8") as handle:
+ return family_for_config(json.load(handle))
+ except SystemExit:
+ raise
+ except Exception:
+ return None
+
+
def cmd_convert(a):
banner("convert")
# here --model is the DESTINATION the converted weights are written to, not an
@@ -1957,15 +1996,54 @@ def cmd_convert(a):
f" --model <dir> is where the converted model is written")
# python con torch/safetensors: l'ambiente del progetto se c'e', altrimenti quello corrente
py = project_python()
- base=[py, os.path.join(TOOLS,"convert_fp8_to_int4.py"),
- "--repo", a.repo, "--outdir", a.model, "--ebits", str(a.ebits), "--io-bits", str(a.io_bits),
- "--group-size", str(a.group_size)]
- if a.xbits: base+=["--xbits",str(a.xbits)]
- # passo 1: modello principale (78 layer). Resumabile: riparte dagli shard mancanti.
- print(f" {C.dim}[1/2] model: {' '.join(base)}{C.r}")
- rc=subprocess.call(base)
- if rc!=0: sys.exit(rc)
- if a.no_mtp: sys.exit(0)
+
+ # #1368: quale convertitore, deciso dal checkpoint e non dal fatto che GLM-5.2
+ # e' arrivato per primo. Prima di questo, una GLM-5.3-Flash passava per il
+ # convertitore di GLM-5.2, che le quantizzava l'embedding perche' il suo nome
+ # annidato non corrisponde a nessuna delle sue regole; il contenitore usciva
+ # senza un errore e moriva ore dopo, dentro `coli web`.
+ family = checkpoint_family(a.repo)
+ script = family.converter if family else "convert_fp8_to_int4.py"
+ if family and not script:
+ # Famiglia nota che coli non guida: il suo convertitore ha un'altra riga
+ # di comando, o non serve. La guardia dentro convert_fp8_to_int4.py tiene
+ # le indicazioni per famiglia, sotto test; le si lascia dire a lei
+ # invece di tenerne una seconda copia qui.
+ script = "convert_fp8_to_int4.py"
+ if family:
+ print(f" {C.dim}checkpoint: {family.display_name} -> tools/{script}{C.r}")
+ written = [flag for name, flag in _CONVERT_PRECISION_FLAGS.items()
+ if flag in sys.argv and name not in family.converter_accepts]
+ if written and family.converter:
+ sys.exit(f"{C.yel}{' '.join(written)}: tools/{script} does not take "
+ f"{'them' if len(written) > 1 else 'it'}.{C.r}\n"
+ f" {family.display_name} keeps its dense weights and embedding "
+ f"wide and picks the precision when the engine loads them.\n"
+ f" Re-run without {' '.join(written)}.")
+
+ # Quando la famiglia non si e' potuta stabilire si passano tutte le opzioni,
+ # cioe' esattamente il comando di prima: nessuna conversione che funzionava
+ # cambia comportamento perche' la rete non ha risposto.
+ accepts = family.converter_accepts if family and family.converter else \
+ tuple(_CONVERT_PRECISION_FLAGS)
+ base = [py, os.path.join(TOOLS, script), "--repo", a.repo, "--outdir", a.model]
+ if "ebits" in accepts:
+ base += ["--ebits", str(a.ebits), "--io-bits", str(a.io_bits)]
+ if "group_size" in accepts:
+ base += ["--group-size", str(a.group_size)]
+ if a.xbits and "xbits" in accepts:
+ base += ["--xbits", str(a.xbits)]
+
+ mtp_pass = family.converter_mtp_pass if family and family.converter else True
+ steps = 2 if (mtp_pass and not a.no_mtp) else 1
+ print(f" {C.dim}[1/{steps}] model: {' '.join(base)}{C.r}")
+ rc = subprocess.call(base)
+ if rc != 0: sys.exit(rc)
+ if steps == 1:
+ # Non tutte le famiglie hanno una testa MTP da convertire a parte, e
+ # convert_glm53.py non ha proprio --mtp: chiamarlo con quel flag e' un
+ # errore di argomenti dopo che il modello e' gia' stato scritto.
+ sys.exit(0)
# passo 2: testa MTP (layer 78). SEMPRE int8: a int4 i draft sbagliano quasi sempre
# (acceptance 0-4% vs 39-59%, misurato — issue #8) e la speculazione non parte mai.
mtp_cmd=list(base); i=mtp_cmd.index("--ebits"); mtp_cmd[i+1]=str(max(8,a.ebits))
diff --git a/c/family_registry.py b/c/family_registry.py
index aa4cad62..b86d36d8 100644
--- a/c/family_registry.py
+++ b/c/family_registry.py
@@ -89,6 +89,26 @@ class FamilyDescriptor:
# separate from expert_inventory prevents a fixed allocation from
# being multiplied by the cache capacity.
fixed_resident_inventory: object = None
+ # Lo script sotto tools/ che `coli convert` puo' guidare per questa famiglia,
+ # e le opzioni di `coli convert` che quello script accetta davvero.
+ #
+ # #1368: `coli convert` lanciava convert_fp8_to_int4.py qualunque cosa gli si
+ # desse. Quel convertitore e' di GLM-5.2 e classifica i tensori per nome
+ # PIATTO, mentre GLM-5.3-Flash annida il testuale sotto il wrapper vision:
+ # `model.language_model.embed_tokens.weight` non corrisponde a nessuna regola
+ # e cade nel fallback finale, che lo quantizza. Poi glm53.c lo legge con
+ # load_f32, rifiuta l'U8, e il motore muore dentro `coli web`.
+ #
+ # Vuoto significa "coli non guida questa famiglia": non e' una lacuna, e'
+ # che il suo convertitore ha un'altra riga di comando (convert_qwen36.py usa
+ # --out e --gs, convert_inkling_int4.py non ha --repo) o non serve affatto
+ # (Qwen3.8 gira sul checkpoint ufficiale). In quel caso si lascia parlare la
+ # guardia del convertitore, che quelle indicazioni le ha gia' per famiglia e
+ # sotto test: due copie della stessa mappa sarebbero il difetto che questa
+ # correzione sta chiudendo.
+ converter: str = ""
+ converter_accepts: tuple = ()
+ converter_mtp_pass: bool = False
@dataclass(frozen=True, slots=True)
@@ -888,6 +908,12 @@ FAMILIES = (
# export di solo testo dichiara "glm5_next_text" al primo livello.
model_types=("glm5_next", "glm5_next_text"),
display_name="GLM-5.3-Flash",
+ # Niente ebits/io_bits/xbits: questo convertitore tiene i densi e
+ # l'embedding in BF16 e la precisione la sceglie il motore a load time
+ # (GLM53_BITS). Accettare quelle opzioni per poi ignorarle sarebbe la
+ # versione silenziosa dello stesso guasto.
+ converter="convert_glm53.py",
+ converter_accepts=("group_size",),
display_scale="321B",
engine_artifact="glm53",
engine_aliases=(),
@@ -945,6 +971,9 @@ FAMILIES = (
# family loads. If a future release ever adds a real discriminator,
# split this then, on evidence.
display_name="GLM-5.2/5.3",
+ converter="convert_fp8_to_int4.py",
+ converter_accepts=("ebits", "io_bits", "xbits", "group_size"),
+ converter_mtp_pass=True,
display_scale="744B",
engine_artifact="colibri",
engine_aliases=("glm",),
diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.py
index bf43b607..84a1cda3 100644
--- a/c/tests/test_cli_output.py
+++ b/c/tests/test_cli_output.py
@@ -262,11 +262,22 @@ class BannerModelLineTest(unittest.TestCase):
self.assertNotIn("GLM-5.2", line)
def test_no_model_keeps_the_generic_tagline(self):
- self.assertIn("GLM-5.2", self.coli.model_banner_line(None))
+ """What matters is that a tagline comes back and that it names no
+ model. It used to assert the substring "GLM-5.2", which pinned the
+ example rather than the property: the tagline named the flagship
+ family, and #1367 renamed that family out from under it."""
+ line = self.coli.model_banner_line(None)
+ self.assertTrue(line.strip())
+ self.assertIn("model families", line)
+ for family in self.coli.all_families():
+ self.assertNotIn(family.display_name, line,
+ "the generic tagline names a specific model; it is "
+ "printed when no model was given")
def test_unreadable_model_falls_back_instead_of_raising(self):
"""`coli info` banners before validating the path; it must not crash."""
- self.assertIn("GLM-5.2", self.coli.model_banner_line("/nonexistent/xyz"))
+ self.assertEqual(self.coli.model_banner_line("/nonexistent/xyz"),
+ self.coli.model_banner_line(None))
def test_size_is_reported_without_rounding_to_zero(self):
small = self.line({"model_type": "olmoe"}, shard_bytes=4_200_000_000)
diff --git a/c/tests/test_convert_routing.py b/c/tests/test_convert_routing.py
new file mode 100644
index 00000000..5231e50b
--- /dev/null
+++ b/c/tests/test_convert_routing.py
@@ -0,0 +1,207 @@
+"""`coli convert` must pick the converter from the checkpoint, not from history.
+
+#1368: it ran `tools/convert_fp8_to_int4.py` on whatever it was given. That
+converter is GLM-5.2's, and it classifies tensors by FLAT name:
+
+ if name in ("model.embed_tokens.weight", "lm_head.weight"): return "io"
+
+GLM-5.3-Flash nests its language model under the vision wrapper, so its
+embedding is `model.language_model.embed_tokens.weight`. It matched no rule,
+fell into the final `endswith(".weight")` fallback, and was quantized. Hours
+later `glm53.c` read it with load_f32, refused the U8, and the engine died
+inside `coli web` with a traceback about a subprocess exiting.
+
+Nothing along that path was wrong except the first choice.
+
+What the tests below hold:
+
+- the checkpoint decides the converter, and a Flash checkpoint reaches
+ convert_glm53.py;
+- GLM-5.2's command is byte-for-byte what it was, because the fix must not
+ quietly re-tune the path that was already right;
+- an option the target converter does not take is REFUSED, not dropped.
+ Silently ignoring `--ebits` on a converter that has no such flag is the same
+ defect wearing better manners;
+- when the family cannot be determined -- no network, no huggingface_hub -- the
+ old command is used unchanged, so a working conversion never fails because a
+ 5 KB metadata fetch did.
+"""
+import importlib.machinery
+import importlib.util
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from family_registry import FAMILIES, family_by_id
+
+
+HERE = Path(__file__).resolve().parent.parent
+CLI = HERE / "coli"
+
+
+def load_cli():
+ loader = importlib.machinery.SourceFileLoader("coli_convert_test", str(CLI))
+ spec = importlib.util.spec_from_loader(loader.name, loader)
+ module = importlib.util.module_from_spec(spec)
+ loader.exec_module(module)
+ return module
+
+
+class Args:
+ """The subset of `coli convert`'s namespace that cmd_convert reads."""
+
+ def __init__(self, out, **overrides):
+ self.repo = "some/checkpoint"
+ self.model = out
+ self.ebits = 4
+ self.io_bits = 8
+ self.xbits = 0
+ self.group_size = 64
+ self.no_mtp = False
+ self.__dict__.update(overrides)
+
+
+class ConvertRoutingTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.cli = load_cli()
+
+ def run_convert(self, model_type, argv=("coli", "convert"), **overrides):
+ """cmd_convert with the subprocess replaced: returns the commands it
+ would have run, or the SystemExit it raised instead."""
+ directory = tempfile.TemporaryDirectory()
+ self.addCleanup(directory.cleanup)
+ calls = []
+
+ def fake_call(command):
+ calls.append(command)
+ return 0
+
+ with mock.patch.object(self.cli, "subprocess") as subprocess_module, \
+ mock.patch.object(self.cli, "project_python", return_value="python3"), \
+ mock.patch.object(self.cli, "sys") as system, \
+ mock.patch.object(self.cli, "checkpoint_family",
+ return_value=(family_by_id(model_type)
+ if model_type else None)):
+ subprocess_module.call = fake_call
+ system.argv = list(argv)
+ system.exit.side_effect = SystemExit # sys.exit must RAISE, not build
+ try:
+ self.cli.cmd_convert(Args(directory.name, **overrides))
+ except (SystemExit, TypeError):
+ # cmd_convert ends by calling sys.exit; the mock raises the class.
+ pass
+ return calls
+
+ @staticmethod
+ def script_of(command):
+ return Path(command[1]).name
+
+ def test_a_flash_checkpoint_reaches_its_own_converter(self):
+ calls = self.run_convert("glm53")
+ self.assertTrue(calls, "no converter was launched at all")
+ self.assertEqual(self.script_of(calls[0]), "convert_glm53.py")
+
+ def test_the_flash_command_carries_no_precision_flags(self):
+ """convert_glm53.py has no --ebits/--io-bits/--xbits: it keeps the dense
+ weights and the embedding wide and the engine picks the precision at
+ load time. Passing them would be an argument error after the download."""
+ command = self.run_convert("glm53")[0]
+ for flag in ("--ebits", "--io-bits", "--xbits"):
+ self.assertNotIn(flag, command)
+ self.assertIn("--group-size", command)
+
+ def test_flash_runs_one_pass_because_it_has_no_mtp_flag(self):
+ calls = self.run_convert("glm53")
+ self.assertEqual(len(calls), 1,
+ "convert_glm53.py has no --mtp; a second pass is an "
+ "argument error after the model is already written")
+
+ def test_glm52_is_untouched(self):
+ """The regression guard. This path was already correct and the whole
+ point of the change is that it stays identical."""
+ calls = self.run_convert("glm")
+ self.assertEqual(self.script_of(calls[0]), "convert_fp8_to_int4.py")
+ for flag in ("--repo", "--outdir", "--ebits", "--io-bits", "--group-size"):
+ self.assertIn(flag, calls[0])
+ self.assertEqual(len(calls), 2, "the int8 MTP pass must still run")
+ self.assertIn("--mtp", calls[1])
+ index = calls[1].index("--ebits")
+ self.assertEqual(calls[1][index + 1], "8", "the MTP head is always int8")
+
+ def test_no_mtp_still_skips_the_second_pass_for_glm52(self):
+ calls = self.run_convert("glm", no_mtp=True)
+ self.assertEqual(len(calls), 1)
+
+ def test_an_option_the_target_does_not_take_is_refused_not_dropped(self):
+ calls = self.run_convert(
+ "glm53", argv=("coli", "convert", "--ebits", "3"), ebits=3)
+ self.assertEqual(calls, [],
+ "--ebits was silently dropped and the conversion ran "
+ "anyway; the user asked for a precision they did not get")
+
+ def test_an_unresolvable_family_keeps_the_old_command(self):
+ """A metadata fetch that fails is not a reason to refuse a conversion
+ that would have worked. The converter's own guard downloads the same
+ config.json and refuses there if the family is genuinely wrong."""
+ calls = self.run_convert(None)
+ self.assertEqual(self.script_of(calls[0]), "convert_fp8_to_int4.py")
+ for flag in ("--ebits", "--io-bits", "--group-size"):
+ self.assertIn(flag, calls[0])
+ self.assertEqual(len(calls), 2)
+
+
+class ConverterDeclarationTest(unittest.TestCase):
+ """A family that names a converter must also say which options it takes.
+
+ Half a declaration is worse than none: the dispatcher would build a command
+ with no precision flags at all and the user would get defaults they never
+ chose, silently, which is the shape of #1368 itself.
+ """
+
+ def test_every_declared_converter_states_what_it_accepts(self):
+ for family in FAMILIES:
+ if not family.converter:
+ self.assertEqual(
+ family.converter_accepts, (),
+ f"{family.id}: accepts options for a converter it does not name")
+ continue
+ self.assertTrue(
+ family.converter.endswith(".py"),
+ f"{family.id}: converter must be a script under tools/")
+ self.assertTrue(
+ family.converter_accepts,
+ f"{family.id}: names {family.converter} but says nothing about "
+ f"which coli options it takes, so every one of them is dropped")
+
+ def test_declared_converters_exist_on_disk(self):
+ for family in FAMILIES:
+ if not family.converter:
+ continue
+ path = HERE / "tools" / family.converter
+ self.assertTrue(path.is_file(),
+ f"{family.id} names tools/{family.converter}, "
+ f"which is not in the tree")
+
+ def test_a_declared_converter_really_takes_what_is_claimed(self):
+ """Read the target's own argparse rather than trusting the declaration.
+ The two drift the moment someone renames a flag, and the failure would
+ land on a user mid-conversion."""
+ spelling = {"ebits": "--ebits", "io_bits": "--io-bits",
+ "xbits": "--xbits", "group_size": "--group-size"}
+ for family in FAMILIES:
+ if not family.converter:
+ continue
+ source = (HERE / "tools" / family.converter).read_text(
+ encoding="utf-8", errors="ignore")
+ for option in family.converter_accepts:
+ flag = spelling[option]
+ self.assertIn(f'add_argument("{flag}"', source,
+ f"{family.id} claims tools/{family.converter} takes "
+ f"{flag}, but that script does not define it")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/pack_python.py b/c/tools/pack_python.py
index a6699cf3..6147dc8e 100644
--- a/c/tools/pack_python.py
+++ b/c/tools/pack_python.py
@@ -120,6 +120,34 @@ def data_files(path):
return found
+def named_tools(path, src):
+ """Gli script sotto tools/ nominati da una stringa, non da una chiamata.
+
+ #1368: i convertitori per famiglia sono dichiarati in family_registry.py
+ come semplici stringhe ("convert_glm53.py"), perche' e' coli a comporre il
+ comando a runtime. Non c'e' nessun `os.path.join(TOOLS, "...")` da trovare,
+ quindi invoked_scripts() non li vede e l'archivio uscirebbe senza il
+ convertitore -- che e' precisamente #1296, un giro piu' tardi.
+
+ Come per i file di dati, si richiede che il file ESISTA sotto tools/: una
+ stringa che somiglia a un nome di script ma non lo e' non produce nulla."""
+ try:
+ tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"))
+ except SyntaxError:
+ return set()
+ found = set()
+ for node in ast.walk(tree):
+ if not (isinstance(node, ast.Constant) and isinstance(node.value, str)):
+ continue
+ name = node.value
+ if not re.fullmatch(r"[a-z0-9_]+\.py", name):
+ continue
+ candidate = src / "tools" / name
+ if candidate.is_file():
+ found.add(candidate)
+ return found
+
+
def needed(src):
"""Tutto cio' che coli raggiunge, come percorsi sotto <c-dir>.
@@ -143,6 +171,11 @@ def needed(src):
while queue:
path = queue.pop()
data |= data_files(path)
+ for candidate in named_tools(path, src):
+ if candidate in script_paths:
+ continue
+ script_paths.add(candidate)
+ queue.append(candidate)
for script in invoked_scripts(path):
if script in scripts:
continue