Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
build(tools): package the data files a reached module opens
JustVuggeb32e6c2f7ed
diff --git a/c/tests/test_pack_python.py b/c/tests/test_pack_python.pyindex d5e544e0..dc0aaacc 100644--- a/c/tests/test_pack_python.py+++ b/c/tests/test_pack_python.py@@ -1,9 +1,18 @@"""tools/pack_python.py's needed() must compute the complete, real set of-Python files a release archive has to contain: every file coli reaches,-whether by import (followed to closure) or by subprocess invocation-(matched by its own `os.path.join(TOOLS, "<script>.py")` call sites) ---and, for a subprocess-invoked script, every file THAT script itself-reaches by import, followed the same way coli's own imports are.+files a release archive has to contain: every file coli reaches, by any of+the three kinds of edge, and everything those files reach in turn.++ - import, followed to closure from `coli`+ - subprocess, matched on the `os.path.join(TOOLS, "<script>.py")` call+ shape -- and the script is then walked like any other reached file, so+ what IT imports is followed too+ - data, a file with a data suffix opened next to a module it belongs to++Each edge was added after a published archive was found missing a file:+the hand-written list (#1296), the subprocess boundary, and the data file+(both #1359). The recurring defect is never the value of one case; it is+that the set is computed one way and the truth is another, so each test+below names the real edge it defends rather than an invented one.Checks enumerated from the source (`tools/pack_python.py`, read in fullbefore writing this module):@@ -16,16 +25,25 @@ before writing this module):-- ast sees them regardless of where in the file they appear.- `invoked_scripts`: only the exact `TOOLS, "<snake_case>.py"` call-siteshape is matched; nothing else in the file is mistaken for one.+- `data_files`: a string literal naming a file that EXISTS next to the+ module and carries a suffix from `DATA_SUFFIXES`; a plain name only, so+ a path or a name nothing matches yields nothing, and a source file+ merely mentioned in a message is not mistaken for data.- `needed`: the import closure starting from `coli`; a script reached bysubprocess is itself added to the queue so its own imports are- followed to the same closure, not merely added as a leaf; a script- invoked but missing from `tools/` raises `SystemExit` before anything- is returned.+ followed to the same closure, not merely added as a leaf; data files+ are collected along the same walk as leaves; a script invoked but+ missing from `tools/` raises `SystemExit` before anything is returned.-No real Colibri source tree is read here -- every case builds a small,-disposable fixture tree with real files."""+All but one case builds a small, disposable fixture tree with real files.+The exception is deliberate and marked: one test asserts against this+checkout's own `c/` directory, because a fixture can only defend edges+someone already thought of, and the two files #1359 was about are the+evidence that the real tree grows shapes the fixtures do not have."""+import contextlibimport importlib.util+import ioimport pathlibimport tempfileimport unittest@@ -131,14 +149,104 @@ class PackPythonNeededTests(unittest.TestCase):found = PACK.invoked_scripts(path)self.assertEqual(found, {"real_one.py"})- def test_engine_evidence_is_needed_by_the_real_tree(self):- """Every other test in this module builds a disposable fixture; this- is the one real-tree assertion, run against this checkout's own- c/ directory so a future tool import that the fixture-based tests- cannot see (because they never touch the real tree) is still- caught here."""+ def test_the_real_tree_carries_the_case_that_started_this(self):+ """Every other test above builds a disposable fixture; this is the one+ real-tree assertion, run against this checkout's own c/ directory so a+ future tool import the fixtures cannot see is still caught here.++ It asserts the two files #1359 was actually about, rather than a name+ invented for the test: coli launches tools/convert_fp8_to_int4.py as a+ subprocess, that script imports iq3_pack inside quant_e8(), and+ iq3_pack opens iq3xxs_grid.json next to itself. Both were absent from+ every published archive. Assert the real pair and the test dies with+ the real bug if either edge is ever dropped again."""paths = PACK.needed(HERE.parent)- self.assertIn(TOOLS / "engine_evidence.py", paths)+ self.assertIn(TOOLS / "iq3_pack.py", paths,+ "reached only through a subprocess-launched script's import")+ self.assertIn(TOOLS / "iq3xxs_grid.json", paths,+ "reached only as a data file opened next to iq3_pack.py")+++class PackPythonDataFileTests(unittest.TestCase):+ """#1359 left a second edge open after the import one was closed: packaging+ iq3_pack.py still does not make `--xbits e8` work from an archive, because+ iq3_pack.py opens a sibling JSON and the packer copied only .py files."""++ def make_tree(self, root, data_name="grid.json"):+ root = pathlib.Path(root)+ (root / "tools").mkdir()+ (root / "coli").write_text(+ "import os\nimport reader\n"+ "TOOLS = os.path.join(os.path.dirname(__file__), 'tools')\n")+ (root / "reader.py").write_text(+ "import os, json\n"+ "def load():\n"+ " p = os.path.join(os.path.dirname(__file__), %r)\n"+ " return json.load(open(p))\n" % data_name)+ (root / data_name).write_text("[1, 2, 3]\n")+ return root++ def test_a_data_file_opened_next_to_a_module_is_packaged(self):+ with tempfile.TemporaryDirectory() as tmp:+ root = self.make_tree(tmp)+ names = {path.name for path in PACK.needed(root)}+ self.assertIn("reader.py", names)+ self.assertIn("grid.json", names,+ "a module's sibling data file is part of what the "+ "archive must contain, not an optional extra")++ def test_without_data_files_the_module_ships_but_cannot_run(self):+ # Negative control: replay the pre-fix shape (walk the same graph, but+ # never collect data siblings) against the identical fixture. If this+ # ever stops dropping grid.json, the fixture no longer isolates the fix.+ with tempfile.TemporaryDirectory() as tmp:+ root = self.make_tree(tmp)+ original, PACK.data_files = PACK.data_files, lambda path: set()+ try:+ names = {path.name for path in PACK.needed(root)}+ finally:+ PACK.data_files = original+ self.assertIn("reader.py", names)+ self.assertNotIn("grid.json", names)++ def test_a_source_file_named_in_a_message_is_not_packaged(self):+ """The allowlist earns its keep here. coli names colibri.c and+ family_registry.py names five engine sources, none of which belong in+ a release archive; a rule of "any existing sibling that is not .py"+ would have shipped all of them."""+ with tempfile.TemporaryDirectory() as tmp:+ root = pathlib.Path(tmp)+ (root / "engine.c").write_text("int main(void){return 0;}\n")+ (root / "talker.py").write_text(+ 'MSG = "build it first: cc engine.c"\n'+ 'NAME = "engine.c"\n')+ found = PACK.data_files(root / "talker.py")+ self.assertEqual(found, set())++ def test_a_name_that_is_not_a_real_sibling_is_ignored(self):+ with tempfile.TemporaryDirectory() as tmp:+ root = pathlib.Path(tmp)+ (root / "talker.py").write_text(+ 'A = "absent.json"\n' # nothing of that name exists+ 'B = "sub/dir/real.json"\n') # a path, not a plain sibling+ (root / "sub").mkdir()+ (root / "sub" / "dir").mkdir()+ (root / "sub" / "dir" / "real.json").write_text("{}\n")+ found = PACK.data_files(root / "talker.py")+ self.assertEqual(found, set())++ def test_the_summary_line_counts_data_files_separately(self):+ """A display derived from a set whose meaning changed is how #856 hid+ for a release: the engine's own self-report agreed with the bug."""+ with tempfile.TemporaryDirectory() as tmp:+ root = self.make_tree(tmp)+ dist = pathlib.Path(tmp) / "dist"+ buf = io.StringIO()+ with contextlib.redirect_stdout(buf):+ PACK.main(["pack_python.py", str(root), str(dist)])+ # coli is the walk's root and is copied by the workflow, not by+ # needed(), so the fixture's one packaged module is reader.py.+ self.assertIn("1 Python files and 1 data files", buf.getvalue())if __name__ == "__main__":diff --git a/c/tools/pack_python.py b/c/tools/pack_python.pyindex 38f7c36d..a6699cf3 100644--- a/c/tools/pack_python.py+++ b/c/tools/pack_python.py@@ -17,9 +17,33 @@ rispondeva quindi che mancavano delle dipendenze all'utente, mentre il file nonera stato spedito. Un import cercato con una regex, o a occhio, non lo trova:per questo qui si usa ast.-Due modi di raggiungere un file, ed entrambi contano:+Tre modi di raggiungere un file, e contano tutti:- import, seguiti in chiusura a partire da coli- - subprocess, cioe' os.path.join(TOOLS, "qualcosa.py")+ - subprocess, cioe' os.path.join(TOOLS, "qualcosa.py"), seguiti a loro volta+ - dati letti accanto al modulo, cioe' iq3_pack.py che apre iq3xxs_grid.json++Il terzo e' arrivato per ultimo (#1359) e insegna la stessa cosa dei primi due:+il difetto non e' mai il valore di un caso, e' che l'insieme viene calcolato in+un modo e la verita' e' un altro. Ogni volta il file mancante c'era da sempre;+mancava un tipo di arco nel grafo.++Sui dati, questa e' la regola e i suoi limiti dichiarati. Si spedisce una+stringa-letterale che nomina un file ESISTENTE accanto al modulo e con una+estensione da dati (DATA_SUFFIXES). Quindi:+ - non trova un percorso costruito a pezzi, o un nome calcolato a runtime;+ - non spedisce un sorgente citato in un messaggio -- e ce ne sono molti:+ `coli` nomina colibri.c, family_registry.py nomina cinque motori. Nessuno+ di questi va nell'archivio, ed e' il motivo per cui qui c'e' una lista di+ estensioni e non "tutto cio' che non e' .py".+Sbaglia quindi per eccesso, mai per difetto: al peggio spedisce qualche KB in+piu'. E' la direzione giusta per un packer, dove sotto-spedire rompe una+release e sovra-spedire no.++Un caso volutamente fuori: tools/iq3_encode.c, che iq3_pack.py compila come+libreria condivisa se la trova. Il suo stesso docstring lo dice opzionale --+senza, il percorso numpy funziona 25 volte piu' lento -- e servirebbe un+compilatore sulla macchina di chi scarica. Spedirlo sarebbe una decisione da+prendere, non un effetto collaterale di un'euristica.Uso:pack_python.py <c-dir> <dist-dir> copia@@ -62,6 +86,40 @@ def invoked_scripts(path):return set(re.findall(r'TOOLS,\s*"([a-z0-9_]+\.py)"', text))+#: Estensioni che un modulo apre come dati. Deliberatamente una lista, non+#: "tutto cio' che non e' .py": vedi il docstring in testa.+DATA_SUFFIXES = frozenset({+ ".json", ".txt", ".csv", ".tsv", ".model", ".bin", ".dat",+ ".cfg", ".ini", ".yaml", ".yml",+})+++def data_files(path):+ """I file di dati che questo modulo apre accanto a se'.++ iq3_pack.py fa `os.path.join(os.path.dirname(__file__), "iq3xxs_grid.json")`+ e senza quel file il suo encode muore appena viene chiamato (#1359).++ Si richiede che il file ESISTA gia' accanto al modulo, cosi' una stringa che+ somiglia a un nome di file ma non lo e' non produce nulla: e' il vincolo che+ tiene onesta un'euristica su stringhe letterali."""+ try:+ tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"))+ except SyntaxError:+ return set()+ found = set()+ for node in ast.walk(tree):+ if not (isinstance(node, ast.Constant) and isinstance(node.value, str)):+ continue+ name = node.value+ if "/" in name or "\\" in name: # solo nomi semplici, non percorsi+ continue+ sibling = path.parent / name+ if sibling.suffix.lower() in DATA_SUFFIXES and sibling.is_file():+ found.add(sibling)+ return found++def needed(src):"""Tutto cio' che coli raggiunge, come percorsi sotto <c-dir>.@@ -75,12 +133,16 @@ def needed(src):script that needs it was. The script paths below are pushed onto thesame import-following queue as every other reached file, so theirimports close the same way coli's own do.++ I file di dati letti accanto a un modulo si raccolgono lungo la stessa+ visita: sono foglie, non hanno archi uscenti, quindi non entrano in coda."""local = local_modules(src)reached, queue = set(), [src / "coli"]- scripts, script_paths = set(), set()+ scripts, script_paths, data = set(), set(), set()while queue:path = queue.pop()+ data |= data_files(path)for script in invoked_scripts(path):if script in scripts:continue@@ -96,6 +158,7 @@ def needed(src):queue.append(local[name])paths = {local[name] for name in reached}paths |= script_paths+ paths |= datareturn sorted(paths)@@ -121,8 +184,12 @@ def main(argv):for name in missing:print(f" {name}")return 1- print(f"{len(files)} Python files reachable from coli "- f"({'all present' if check else 'copied'})")+ # Contati separatamente perche' l'insieme non e' piu' solo Python: dire+ # "N Python files" mentre se ne spediscono anche di dati e' la stessa+ # bugia di display che in #856 fece concordare l'autodiagnosi col bug.+ py = sum(1 for path in files if path.suffix == ".py")+ print(f"{py} Python files and {len(files) - py} data files reachable "+ f"from coli ({'all present' if check else 'copied'})")return 0