Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

build(tools): package the data files a reached module opens

JustVuggeb32e6c2f7ed
diff --git a/c/tests/test_pack_python.py b/c/tests/test_pack_python.py
index d5e544e0..dc0aaacc 100644
--- a/c/tests/test_pack_python.py
+++ b/c/tests/test_pack_python.py
@@ -1,9 +1,18 @@
"""tools/pack_python.py's needed() must compute the complete, real set of
-Python files a release archive has to contain: every file coli reaches,
-whether by import (followed to closure) or by subprocess invocation
-(matched by its own `os.path.join(TOOLS, "<script>.py")` call sites) --
-and, for a subprocess-invoked script, every file THAT script itself
-reaches by import, followed the same way coli's own imports are.
+files a release archive has to contain: every file coli reaches, by any of
+the three kinds of edge, and everything those files reach in turn.
+
+ - import, followed to closure from `coli`
+ - subprocess, matched on the `os.path.join(TOOLS, "<script>.py")` call
+ shape -- and the script is then walked like any other reached file, so
+ what IT imports is followed too
+ - data, a file with a data suffix opened next to a module it belongs to
+
+Each edge was added after a published archive was found missing a file:
+the hand-written list (#1296), the subprocess boundary, and the data file
+(both #1359). The recurring defect is never the value of one case; it is
+that the set is computed one way and the truth is another, so each test
+below names the real edge it defends rather than an invented one.
Checks enumerated from the source (`tools/pack_python.py`, read in full
before writing this module):
@@ -16,16 +25,25 @@ before writing this module):
-- ast sees them regardless of where in the file they appear.
- `invoked_scripts`: only the exact `TOOLS, "<snake_case>.py"` call-site
shape is matched; nothing else in the file is mistaken for one.
+- `data_files`: a string literal naming a file that EXISTS next to the
+ module and carries a suffix from `DATA_SUFFIXES`; a plain name only, so
+ a path or a name nothing matches yields nothing, and a source file
+ merely mentioned in a message is not mistaken for data.
- `needed`: the import closure starting from `coli`; a script reached by
subprocess is itself added to the queue so its own imports are
- followed to the same closure, not merely added as a leaf; a script
- invoked but missing from `tools/` raises `SystemExit` before anything
- is returned.
+ followed to the same closure, not merely added as a leaf; data files
+ are collected along the same walk as leaves; a script invoked but
+ missing from `tools/` raises `SystemExit` before anything is returned.
-No real Colibri source tree is read here -- every case builds a small,
-disposable fixture tree with real files."""
+All but one case builds a small, disposable fixture tree with real files.
+The exception is deliberate and marked: one test asserts against this
+checkout's own `c/` directory, because a fixture can only defend edges
+someone already thought of, and the two files #1359 was about are the
+evidence that the real tree grows shapes the fixtures do not have."""
+import contextlib
import importlib.util
+import io
import pathlib
import tempfile
import unittest
@@ -131,14 +149,104 @@ class PackPythonNeededTests(unittest.TestCase):
found = PACK.invoked_scripts(path)
self.assertEqual(found, {"real_one.py"})
- def test_engine_evidence_is_needed_by_the_real_tree(self):
- """Every other test in this module builds a disposable fixture; this
- is the one real-tree assertion, run against this checkout's own
- c/ directory so a future tool import that the fixture-based tests
- cannot see (because they never touch the real tree) is still
- caught here."""
+ def test_the_real_tree_carries_the_case_that_started_this(self):
+ """Every other test above builds a disposable fixture; this is the one
+ real-tree assertion, run against this checkout's own c/ directory so a
+ future tool import the fixtures cannot see is still caught here.
+
+ It asserts the two files #1359 was actually about, rather than a name
+ invented for the test: coli launches tools/convert_fp8_to_int4.py as a
+ subprocess, that script imports iq3_pack inside quant_e8(), and
+ iq3_pack opens iq3xxs_grid.json next to itself. Both were absent from
+ every published archive. Assert the real pair and the test dies with
+ the real bug if either edge is ever dropped again."""
paths = PACK.needed(HERE.parent)
- self.assertIn(TOOLS / "engine_evidence.py", paths)
+ self.assertIn(TOOLS / "iq3_pack.py", paths,
+ "reached only through a subprocess-launched script's import")
+ self.assertIn(TOOLS / "iq3xxs_grid.json", paths,
+ "reached only as a data file opened next to iq3_pack.py")
+
+
+class PackPythonDataFileTests(unittest.TestCase):
+ """#1359 left a second edge open after the import one was closed: packaging
+ iq3_pack.py still does not make `--xbits e8` work from an archive, because
+ iq3_pack.py opens a sibling JSON and the packer copied only .py files."""
+
+ def make_tree(self, root, data_name="grid.json"):
+ root = pathlib.Path(root)
+ (root / "tools").mkdir()
+ (root / "coli").write_text(
+ "import os\nimport reader\n"
+ "TOOLS = os.path.join(os.path.dirname(__file__), 'tools')\n")
+ (root / "reader.py").write_text(
+ "import os, json\n"
+ "def load():\n"
+ " p = os.path.join(os.path.dirname(__file__), %r)\n"
+ " return json.load(open(p))\n" % data_name)
+ (root / data_name).write_text("[1, 2, 3]\n")
+ return root
+
+ def test_a_data_file_opened_next_to_a_module_is_packaged(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = self.make_tree(tmp)
+ names = {path.name for path in PACK.needed(root)}
+ self.assertIn("reader.py", names)
+ self.assertIn("grid.json", names,
+ "a module's sibling data file is part of what the "
+ "archive must contain, not an optional extra")
+
+ def test_without_data_files_the_module_ships_but_cannot_run(self):
+ # Negative control: replay the pre-fix shape (walk the same graph, but
+ # never collect data siblings) against the identical fixture. If this
+ # ever stops dropping grid.json, the fixture no longer isolates the fix.
+ with tempfile.TemporaryDirectory() as tmp:
+ root = self.make_tree(tmp)
+ original, PACK.data_files = PACK.data_files, lambda path: set()
+ try:
+ names = {path.name for path in PACK.needed(root)}
+ finally:
+ PACK.data_files = original
+ self.assertIn("reader.py", names)
+ self.assertNotIn("grid.json", names)
+
+ def test_a_source_file_named_in_a_message_is_not_packaged(self):
+ """The allowlist earns its keep here. coli names colibri.c and
+ family_registry.py names five engine sources, none of which belong in
+ a release archive; a rule of "any existing sibling that is not .py"
+ would have shipped all of them."""
+ with tempfile.TemporaryDirectory() as tmp:
+ root = pathlib.Path(tmp)
+ (root / "engine.c").write_text("int main(void){return 0;}\n")
+ (root / "talker.py").write_text(
+ 'MSG = "build it first: cc engine.c"\n'
+ 'NAME = "engine.c"\n')
+ found = PACK.data_files(root / "talker.py")
+ self.assertEqual(found, set())
+
+ def test_a_name_that_is_not_a_real_sibling_is_ignored(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = pathlib.Path(tmp)
+ (root / "talker.py").write_text(
+ 'A = "absent.json"\n' # nothing of that name exists
+ 'B = "sub/dir/real.json"\n') # a path, not a plain sibling
+ (root / "sub").mkdir()
+ (root / "sub" / "dir").mkdir()
+ (root / "sub" / "dir" / "real.json").write_text("{}\n")
+ found = PACK.data_files(root / "talker.py")
+ self.assertEqual(found, set())
+
+ def test_the_summary_line_counts_data_files_separately(self):
+ """A display derived from a set whose meaning changed is how #856 hid
+ for a release: the engine's own self-report agreed with the bug."""
+ with tempfile.TemporaryDirectory() as tmp:
+ root = self.make_tree(tmp)
+ dist = pathlib.Path(tmp) / "dist"
+ buf = io.StringIO()
+ with contextlib.redirect_stdout(buf):
+ PACK.main(["pack_python.py", str(root), str(dist)])
+ # coli is the walk's root and is copied by the workflow, not by
+ # needed(), so the fixture's one packaged module is reader.py.
+ self.assertIn("1 Python files and 1 data files", buf.getvalue())
if __name__ == "__main__":
diff --git a/c/tools/pack_python.py b/c/tools/pack_python.py
index 38f7c36d..a6699cf3 100644
--- a/c/tools/pack_python.py
+++ b/c/tools/pack_python.py
@@ -17,9 +17,33 @@ rispondeva quindi che mancavano delle dipendenze all'utente, mentre il file non
era stato spedito. Un import cercato con una regex, o a occhio, non lo trova:
per questo qui si usa ast.
-Due modi di raggiungere un file, ed entrambi contano:
+Tre modi di raggiungere un file, e contano tutti:
- import, seguiti in chiusura a partire da coli
- - subprocess, cioe' os.path.join(TOOLS, "qualcosa.py")
+ - subprocess, cioe' os.path.join(TOOLS, "qualcosa.py"), seguiti a loro volta
+ - dati letti accanto al modulo, cioe' iq3_pack.py che apre iq3xxs_grid.json
+
+Il terzo e' arrivato per ultimo (#1359) e insegna la stessa cosa dei primi due:
+il difetto non e' mai il valore di un caso, e' che l'insieme viene calcolato in
+un modo e la verita' e' un altro. Ogni volta il file mancante c'era da sempre;
+mancava un tipo di arco nel grafo.
+
+Sui dati, questa e' la regola e i suoi limiti dichiarati. Si spedisce una
+stringa-letterale che nomina un file ESISTENTE accanto al modulo e con una
+estensione da dati (DATA_SUFFIXES). Quindi:
+ - non trova un percorso costruito a pezzi, o un nome calcolato a runtime;
+ - non spedisce un sorgente citato in un messaggio -- e ce ne sono molti:
+ `coli` nomina colibri.c, family_registry.py nomina cinque motori. Nessuno
+ di questi va nell'archivio, ed e' il motivo per cui qui c'e' una lista di
+ estensioni e non "tutto cio' che non e' .py".
+Sbaglia quindi per eccesso, mai per difetto: al peggio spedisce qualche KB in
+piu'. E' la direzione giusta per un packer, dove sotto-spedire rompe una
+release e sovra-spedire no.
+
+Un caso volutamente fuori: tools/iq3_encode.c, che iq3_pack.py compila come
+libreria condivisa se la trova. Il suo stesso docstring lo dice opzionale --
+senza, il percorso numpy funziona 25 volte piu' lento -- e servirebbe un
+compilatore sulla macchina di chi scarica. Spedirlo sarebbe una decisione da
+prendere, non un effetto collaterale di un'euristica.
Uso:
pack_python.py <c-dir> <dist-dir> copia
@@ -62,6 +86,40 @@ def invoked_scripts(path):
return set(re.findall(r'TOOLS,\s*"([a-z0-9_]+\.py)"', text))
+#: Estensioni che un modulo apre come dati. Deliberatamente una lista, non
+#: "tutto cio' che non e' .py": vedi il docstring in testa.
+DATA_SUFFIXES = frozenset({
+ ".json", ".txt", ".csv", ".tsv", ".model", ".bin", ".dat",
+ ".cfg", ".ini", ".yaml", ".yml",
+})
+
+
+def data_files(path):
+ """I file di dati che questo modulo apre accanto a se'.
+
+ iq3_pack.py fa `os.path.join(os.path.dirname(__file__), "iq3xxs_grid.json")`
+ e senza quel file il suo encode muore appena viene chiamato (#1359).
+
+ Si richiede che il file ESISTA gia' accanto al modulo, cosi' una stringa che
+ somiglia a un nome di file ma non lo e' non produce nulla: e' il vincolo che
+ tiene onesta un'euristica su stringhe letterali."""
+ try:
+ tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"))
+ except SyntaxError:
+ return set()
+ found = set()
+ for node in ast.walk(tree):
+ if not (isinstance(node, ast.Constant) and isinstance(node.value, str)):
+ continue
+ name = node.value
+ if "/" in name or "\\" in name: # solo nomi semplici, non percorsi
+ continue
+ sibling = path.parent / name
+ if sibling.suffix.lower() in DATA_SUFFIXES and sibling.is_file():
+ found.add(sibling)
+ return found
+
+
def needed(src):
"""Tutto cio' che coli raggiunge, come percorsi sotto <c-dir>.
@@ -75,12 +133,16 @@ def needed(src):
script that needs it was. The script paths below are pushed onto the
same import-following queue as every other reached file, so their
imports close the same way coli's own do.
+
+ I file di dati letti accanto a un modulo si raccolgono lungo la stessa
+ visita: sono foglie, non hanno archi uscenti, quindi non entrano in coda.
"""
local = local_modules(src)
reached, queue = set(), [src / "coli"]
- scripts, script_paths = set(), set()
+ scripts, script_paths, data = set(), set(), set()
while queue:
path = queue.pop()
+ data |= data_files(path)
for script in invoked_scripts(path):
if script in scripts:
continue
@@ -96,6 +158,7 @@ def needed(src):
queue.append(local[name])
paths = {local[name] for name in reached}
paths |= script_paths
+ paths |= data
return sorted(paths)
@@ -121,8 +184,12 @@ def main(argv):
for name in missing:
print(f" {name}")
return 1
- print(f"{len(files)} Python files reachable from coli "
- f"({'all present' if check else 'copied'})")
+ # Contati separatamente perche' l'insieme non e' piu' solo Python: dire
+ # "N Python files" mentre se ne spediscono anche di dati e' la stessa
+ # bugia di display che in #856 fece concordare l'autodiagnosi col bug.
+ py = sum(1 for path in files if path.suffix == ".py")
+ print(f"{py} Python files and {len(files) - py} data files reachable "
+ f"from coli ({'all present' if check else 'copied'})")
return 0