Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

build(tools): follow the imports of subprocess-launched scripts

monotophic6bb0113b4656
diff --git a/c/tests/test_pack_python.py b/c/tests/test_pack_python.py
new file mode 100644
index 00000000..d5e544e0
--- /dev/null
+++ b/c/tests/test_pack_python.py
@@ -0,0 +1,145 @@
+"""tools/pack_python.py's needed() must compute the complete, real set of
+Python files a release archive has to contain: every file coli reaches,
+whether by import (followed to closure) or by subprocess invocation
+(matched by its own `os.path.join(TOOLS, "<script>.py")` call sites) --
+and, for a subprocess-invoked script, every file THAT script itself
+reaches by import, followed the same way coli's own imports are.
+
+Checks enumerated from the source (`tools/pack_python.py`, read in full
+before writing this module):
+
+- `local_modules`: root `*.py` files take priority over a `tools/*.py`
+ file of the same stem (root added first, `tools/` only via
+ `setdefault`).
+- `imports_of`: both `import x` and `from x import y` are seen, at any
+ nesting (inside a function, after a `sys.path.insert`, inside a `try`)
+ -- ast sees them regardless of where in the file they appear.
+- `invoked_scripts`: only the exact `TOOLS, "<snake_case>.py"` call-site
+ shape is matched; nothing else in the file is mistaken for one.
+- `needed`: the import closure starting from `coli`; a script reached by
+ subprocess is itself added to the queue so its own imports are
+ followed to the same closure, not merely added as a leaf; a script
+ invoked but missing from `tools/` raises `SystemExit` before anything
+ is returned.
+
+No real Colibri source tree is read here -- every case builds a small,
+disposable fixture tree with real files."""
+
+import importlib.util
+import pathlib
+import tempfile
+import unittest
+
+
+HERE = pathlib.Path(__file__).resolve().parent
+TOOLS = HERE.parent / "tools"
+
+_spec = importlib.util.spec_from_file_location(
+ "pack_python_under_test", TOOLS / "pack_python.py")
+PACK = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(PACK)
+
+
+class PackPythonNeededTests(unittest.TestCase):
+ def make_tree(self, root):
+ """A minimal coli-shaped tree: coli imports `foo` and invokes
+ tools/bar.py as a subprocess; bar.py itself imports tools/baz.py,
+ a module coli never mentions by name anywhere."""
+ root = pathlib.Path(root)
+ (root / "tools").mkdir()
+ (root / "coli").write_text(
+ "import os\n"
+ "import foo\n"
+ "TOOLS = os.path.join(os.path.dirname(__file__), 'tools')\n"
+ "CMD = [PY, os.path.join(TOOLS,\"bar.py\")]\n")
+ (root / "foo.py").write_text("X = 1\n")
+ (root / "tools" / "bar.py").write_text("import baz\nX = 2\n")
+ (root / "tools" / "baz.py").write_text("Y = 3\n")
+ return root
+
+ def test_a_subprocess_reached_script_pulls_in_its_own_imports(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = self.make_tree(tmp)
+ paths = PACK.needed(root)
+ names = {path.name for path in paths}
+ self.assertIn("foo.py", names) # reached by ordinary import
+ self.assertIn("bar.py", names) # reached by subprocess
+ self.assertIn(
+ "baz.py", names,
+ "bar.py's own import of baz must be followed, not dropped "
+ "at the subprocess boundary")
+
+ def test_removing_the_subprocess_recursion_drops_the_transitive_import(self):
+ # Mechanical proof the fix above is load-bearing: replay the
+ # PRE-FIX shape (a subprocess-reached script is added to the
+ # result directly, but its own imports are never walked) against
+ # the identical fixture, and confirm baz.py is silently dropped.
+ def needed_without_recursion(src):
+ local = PACK.local_modules(src)
+ reached, queue = set(), [src / "coli"]
+ scripts = set()
+ while queue:
+ path = queue.pop()
+ scripts |= PACK.invoked_scripts(path)
+ for name in PACK.imports_of(path):
+ if name in local and name not in reached:
+ reached.add(name)
+ queue.append(local[name])
+ paths = {local[name] for name in reached}
+ for script in sorted(scripts):
+ candidate = src / "tools" / script
+ if not candidate.exists():
+ raise SystemExit(
+ f"FAIL: coli invokes tools/{script}, which does not exist")
+ paths.add(candidate)
+ return sorted(paths)
+
+ with tempfile.TemporaryDirectory() as tmp:
+ root = self.make_tree(tmp)
+ names = {path.name for path in needed_without_recursion(root)}
+ self.assertIn("bar.py", names)
+ self.assertNotIn(
+ "baz.py", names,
+ "this replay of the pre-fix code must reproduce the defect "
+ "(baz.py missing) -- if it doesn't, the fixture no longer "
+ "isolates what the fix changed")
+
+ def test_missing_invoked_script_fails_closed(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = self.make_tree(tmp)
+ (root / "tools" / "bar.py").unlink()
+ with self.assertRaisesRegex(SystemExit, r"tools/bar\.py"):
+ PACK.needed(root)
+
+ def test_root_module_shadows_a_same_named_tools_module(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = pathlib.Path(tmp)
+ (root / "tools").mkdir()
+ (root / "shared.py").write_text("ROOT = 1\n")
+ (root / "tools" / "shared.py").write_text("TOOLS = 1\n")
+ local = PACK.local_modules(root)
+ self.assertEqual(local["shared"].read_text(), "ROOT = 1\n")
+
+ def test_invoked_scripts_matches_only_the_exact_call_shape(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = pathlib.Path(tmp)
+ path = root / "sample.py"
+ path.write_text(
+ 'a = os.path.join(TOOLS,"real_one.py")\n'
+ 'b = "TOOLS, \\"not_a_call.py\\""\n'
+ 'c = os.path.join(OTHER,"wrong_var.py")\n')
+ found = PACK.invoked_scripts(path)
+ self.assertEqual(found, {"real_one.py"})
+
+ def test_engine_evidence_is_needed_by_the_real_tree(self):
+ """Every other test in this module builds a disposable fixture; this
+ is the one real-tree assertion, run against this checkout's own
+ c/ directory so a future tool import that the fixture-based tests
+ cannot see (because they never touch the real tree) is still
+ caught here."""
+ paths = PACK.needed(HERE.parent)
+ self.assertIn(TOOLS / "engine_evidence.py", paths)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tools/pack_python.py b/c/tools/pack_python.py
index 018ddb3f..38f7c36d 100644
--- a/c/tools/pack_python.py
+++ b/c/tools/pack_python.py
@@ -63,23 +63,39 @@ def invoked_scripts(path):
def needed(src):
- """Tutto cio' che coli raggiunge, come percorsi sotto <c-dir>."""
+ """Tutto cio' che coli raggiunge, come percorsi sotto <c-dir>.
+
+ A subprocess-launched script is reached the same way an imported
+ module is: its own imports are followed too, not just its own file.
+ #1296's own precedent -- a script invoked by coli missing from the
+ archive -- had a sibling shape one level deeper that #1296 itself did
+ not catch: a script invoked by coli imports a module that neither
+ coli nor any file coli imports ever mentions by name, so that module
+ was never added to `reached` and never copied, even though the
+ script that needs it was. The script paths below are pushed onto the
+ same import-following queue as every other reached file, so their
+ imports close the same way coli's own do.
+ """
local = local_modules(src)
reached, queue = set(), [src / "coli"]
- scripts = set()
+ scripts, script_paths = set(), set()
while queue:
path = queue.pop()
- scripts |= invoked_scripts(path)
+ for script in invoked_scripts(path):
+ if script in scripts:
+ continue
+ scripts.add(script)
+ candidate = src / "tools" / script
+ if not candidate.exists():
+ raise SystemExit(f"FAIL: coli invokes tools/{script}, which does not exist")
+ script_paths.add(candidate)
+ queue.append(candidate)
for name in imports_of(path):
if name in local and name not in reached:
reached.add(name)
queue.append(local[name])
paths = {local[name] for name in reached}
- for script in sorted(scripts):
- candidate = src / "tools" / script
- if not candidate.exists():
- raise SystemExit(f"FAIL: coli invokes tools/{script}, which does not exist")
- paths.add(candidate)
+ paths |= script_paths
return sorted(paths)