Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
build(release): make XDNA release inputs reproducible
Kenneth-Javier Ortigoza7f06a2068acf
diff --git a/c/tests/test_clean_ownership.py b/c/tests/test_clean_ownership.pynew file mode 100644index 00000000..167c2707--- /dev/null+++ b/c/tests/test_clean_ownership.py@@ -0,0 +1,153 @@+"""Owner for what `make clean` is responsible for removing.++There was no test here, and the cost of that showed up in release engineering:+tools/clean.py carried a hand-written list of engine binaries that had never+gained qwen36. `make clean` left the old qwen36 binary in place, and the release+Package step copies whatever sits at c/qwen36 without asking where it came from,+so a locally staged release could ship an engine that was never rebuilt. CI did+not see it because CI starts from an empty runner.++The fix was to derive the engine names from family_registry instead of listing+them, and the point of this file is to keep that derivation honest: the first+test compares the two sets mechanically, so adding a family to the registry+cannot silently create a new uncleanable build product.++clean.py is executed in a sandbox rather than in the checkout -- running the real+thing against the real tree during `make check` would delete the binaries the+rest of the suite is using.+"""+import os+import shutil+import subprocess+import sys+import tempfile+import unittest++HERE = os.path.dirname(os.path.abspath(__file__))+C_DIR = os.path.dirname(HERE)+CLEAN = os.path.join(C_DIR, "tools", "clean.py")++sys.path.insert(0, C_DIR)+from family_registry import all_families # noqa: E402+++def packaged_engines():+ """What a release archive must contain -- the release job's own source.++ .github/workflows/release.yml verifies the packaged archive against exactly+ this registry, so it is the authority on which binaries get published.+ """+ return sorted({f.engine_artifact for f in all_families()})+++class CleanOwnershipTest(unittest.TestCase):+ def _sandbox(self, files=(), dirs=()):+ """A throwaway c/ with clean.py, its import, and some build products."""+ tmp = tempfile.mkdtemp()+ self.addCleanup(shutil.rmtree, tmp, True)+ os.makedirs(os.path.join(tmp, "tools"))+ os.makedirs(os.path.join(tmp, "tests"))+ shutil.copy(CLEAN, os.path.join(tmp, "tools", "clean.py"))+ shutil.copy(os.path.join(C_DIR, "family_registry.py"), tmp)+ for rel in files:+ path = os.path.join(tmp, rel)+ os.makedirs(os.path.dirname(path), exist_ok=True)+ with open(path, "wb") as fh:+ fh.write(b"build product")+ for rel in dirs:+ os.makedirs(os.path.join(tmp, rel), exist_ok=True)+ return tmp++ def _run_clean(self, tmp):+ return subprocess.run([sys.executable, os.path.join("tools", "clean.py")],+ cwd=tmp, capture_output=True, text=True, timeout=120)++ # ---- the invariant -----------------------------------------------------++ def test_every_packaged_engine_is_owned_by_clean(self):+ """PACKAGED_BUT_NOT_CLEANED must be 0, by construction not by vigilance."""+ import importlib.util+ spec = importlib.util.spec_from_file_location("coli_clean", CLEAN)+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ owned = set(mod.FILES)+ missing = [e for e in packaged_engines()+ if e not in owned or e + ".exe" not in owned]+ self.assertEqual(missing, [], "release-packaged but not cleaned: %s" % missing)++ def test_qwen36_specifically(self):+ """The one that was actually missing. Pinned by name so it stays fixed."""+ self.assertIn("qwen36", packaged_engines())+ import importlib.util+ spec = importlib.util.spec_from_file_location("coli_clean", CLEAN)+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ self.assertIn("qwen36", mod.FILES)+ self.assertIn("qwen36.exe", mod.FILES)++ def test_importing_the_module_is_inert(self):+ """Reading FILES must not delete anything.++ This is not hypothetical: the first version of the test above imported+ clean.py with exec_module while `make check` had cwd=c/, and the+ module-level delete loop ran against the real tree, removing the+ binaries the rest of the suite was about to use. The action now lives+ behind __main__.+ """+ import importlib.util+ tmp = self._sandbox(files=["colibri.exe", "qwen36.exe"])+ cwd = os.getcwd()+ os.chdir(tmp)+ try:+ spec = importlib.util.spec_from_file_location("coli_clean_inert", CLEAN)+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ finally:+ os.chdir(cwd)+ for rel in ("colibri.exe", "qwen36.exe"):+ self.assertTrue(os.path.exists(os.path.join(tmp, rel)),+ "importing clean.py deleted %s" % rel)++ # ---- behaviour ---------------------------------------------------------++ def test_clean_removes_engine_binaries(self):+ names = [e + ext for e in packaged_engines() for ext in ("", ".exe")]+ tmp = self._sandbox(files=names)+ self._run_clean(tmp)+ survivors = [n for n in names if os.path.exists(os.path.join(tmp, n))]+ self.assertEqual(survivors, [], "survived clean: %s" % survivors)++ def test_clean_removes_the_physical_probe(self):+ """A stale probe reports PASS for code that is no longer in the tree."""+ tmp = self._sandbox(files=["tests/xdna_physical_probe",+ "tests/xdna_physical_probe.exe"])+ self._run_clean(tmp)+ for rel in ("tests/xdna_physical_probe", "tests/xdna_physical_probe.exe"):+ self.assertFalse(os.path.exists(os.path.join(tmp, rel)), rel)++ def test_clean_does_not_delete_sources_or_unrelated_files(self):+ keep = ["tests/xdna_physical_probe.c", "tests/test_something.c",+ "tests/test_fixture.json", "tests/notes.md", "colibri.c",+ "my_notes.txt", "tests/data.bin", "tools/clean.py"]+ tmp = self._sandbox(files=[k for k in keep if k != "tools/clean.py"],+ dirs=["tests/fixtures"])+ self._run_clean(tmp)+ for rel in keep:+ self.assertTrue(os.path.exists(os.path.join(tmp, rel)),+ "clean deleted %s" % rel)+ self.assertTrue(os.path.isdir(os.path.join(tmp, "tests", "fixtures")))++ def test_clean_is_idempotent(self):+ tmp = self._sandbox(files=["colibri.exe", "qwen36.exe",+ "tests/test_x.exe"])+ first = self._run_clean(tmp)+ self.assertEqual(first.returncode, 0, first.stderr)+ before = sorted(os.listdir(tmp))+ second = self._run_clean(tmp)+ self.assertEqual(second.returncode, 0, second.stderr)+ self.assertEqual(sorted(os.listdir(tmp)), before)+ self.assertIn("removed 0", second.stdout)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/test_xdna_package.py b/c/tests/test_xdna_package.pyindex d4a3a733..ec11471f 100644--- a/c/tests/test_xdna_package.py+++ b/c/tests/test_xdna_package.py@@ -209,5 +209,93 @@ class TestSidecarReleaseProcess(unittest.TestCase):self.assertIn(flag, out)++class TestDeterministicArchive(unittest.TestCase):+ """The published sha256 must be a property of the CONTENT, not the builder.++ ZipFile.write() stores each file's mtime and derives create_system and the+ external attributes from the host, so two maintainers staging identical+ bytes produced different archives. N8-A6-R1 and N8-F0 built the same five+ files and published dcf30127... and 74551ed2... for them.+ """++ @classmethod+ def setUpClass(cls):+ cls.tool = load_tool()++ def _tree(self, tmp, name, mtime, helper=b"synthetic-helper"):+ """A staging tree whose files carry a chosen mtime."""+ tool = self.tool+ adir, hname = tool.artifact_dir_name(), tool.helper_name()+ root = os.path.join(tmp, name)+ os.makedirs(os.path.join(root, adir), exist_ok=True)+ with open(os.path.join(root, hname), "wb") as f:+ f.write(helper)+ for artifact, _ in tool.required_files():+ with open(os.path.join(root, adir, artifact), "wb") as f:+ f.write(b"synthetic-" + artifact.encode())+ for dirpath, _dirs, files in os.walk(root):+ for f in files:+ os.utime(os.path.join(dirpath, f), (mtime, mtime))+ return root++ def _build(self, tmp, root, out):+ tool = self.tool+ path = os.path.join(tmp, out)+ tool.write_archive(path, root, tool.artifact_dir_name(),+ tool.helper_name(), tool.required_files())+ with open(path, "rb") as fh:+ return hashlib.sha256(fh.read()).hexdigest(), path++ def test_archive_is_identical_across_staging_paths_and_mtimes(self):+ with tempfile.TemporaryDirectory() as tmp:+ a = self._tree(tmp, "stage-a", 1_566_000_000) # 2019+ b = self._tree(tmp, "deeper/stage-b", 1_956_528_000) # 2032+ ha, _ = self._build(tmp, a, "a.zip")+ hb, _ = self._build(tmp, b, "b.zip")+ self.assertEqual(ha, hb,+ "archive hash still depends on staging mtime/path")++ def test_a_changed_member_byte_changes_the_archive(self):+ """Reproducibility must not mean insensitivity."""+ with tempfile.TemporaryDirectory() as tmp:+ a = self._tree(tmp, "same", 1_566_000_000)+ b = self._tree(tmp, "diff", 1_566_000_000, helper=b"synthetic-helpeR")+ ha, _ = self._build(tmp, a, "a.zip")+ hb, _ = self._build(tmp, b, "b.zip")+ self.assertNotEqual(ha, hb)++ def test_member_metadata_is_pinned_not_inherited(self):+ with tempfile.TemporaryDirectory() as tmp:+ root = self._tree(tmp, "stage", 1_566_000_000)+ _, path = self._build(tmp, root, "a.zip")+ with zipfile.ZipFile(path) as z:+ for info in z.infolist():+ self.assertEqual(info.date_time, self.tool.ZIP_EPOCH)+ self.assertEqual(info.create_system, 0)+ self.assertEqual(info.external_attr,+ self.tool.ZIP_MODE << 16)+ self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED)++ def test_member_order_is_sorted_not_filesystem_order(self):+ with tempfile.TemporaryDirectory() as tmp:+ root = self._tree(tmp, "stage", 1_566_000_000)+ _, path = self._build(tmp, root, "a.zip")+ with zipfile.ZipFile(path) as z:+ stored = [i.filename for i in z.infolist()]+ self.assertEqual(stored, sorted(stored))++ def test_deterministic_archive_still_holds_exactly_the_asset_set(self):+ with tempfile.TemporaryDirectory() as tmp:+ root = self._tree(tmp, "stage", 1_566_000_000)+ _, path = self._build(tmp, root, "a.zip")+ adir = self.tool.artifact_dir_name()+ expected = sorted(+ [self.tool.helper_name()]+ + ["%s/%s" % (adir, n) for n, _ in self.tool.required_files()])+ with zipfile.ZipFile(path) as z:+ self.assertEqual(sorted(z.namelist()), expected)++if __name__ == "__main__":unittest.main()diff --git a/c/tools/build_xdna_package.py b/c/tools/build_xdna_package.pyindex 04b46181..75440460 100644--- a/c/tools/build_xdna_package.py+++ b/c/tools/build_xdna_package.py@@ -236,6 +236,56 @@ def verify_release(archive, tag_expected=None):return problems+# ---- the reproducible archive ------------------------------------------+#+# Two maintainers with the same helper and the same qualified artifacts must+# produce the same archive, byte for byte -- otherwise the published sha256 is+# a property of whoever built it rather than of what is inside.+#+# ZipFile.write() defeats that on its own: it reads each file's mtime and+# stores it in the local header and the central directory, and it derives+# create_system and external_attr from the host OS and the file mode. So the+# same bytes staged from two directories produced two different archives --+# which is exactly what happened between N8-A6-R1 (dcf30127...) and N8-F0+# (74551ed2...), for content that was identical.+#+# Everything that varies is therefore pinned:+ZIP_EPOCH = (1980, 1, 1, 0, 0, 0) # the earliest a ZIP timestamp can express+ZIP_MODE = 0o644 # not the staging file's mode+ZIP_LEVEL = 9 # explicit, not the zlib default+++def zip_members(root, adir, hname, need):+ """(archive_path, source_path) in a fixed order, independent of the fs."""+ members = [(hname, os.path.join(root, hname))]+ for name, _ in need:+ members.append(("%s/%s" % (adir, name), os.path.join(root, adir, name)))+ return sorted(members)+++def write_archive(path, root, adir, hname, need):+ """Write the sidecar archive deterministically.++ Depends on nothing but the member paths and the member bytes: not the+ staging mtimes, not the staging path, not the host OS, not the locale,+ not the order the filesystem happens to return.++ The one residual assumption is that zlib emits the same deflate stream for+ the same input at the same level. That is true in practice and stable+ across the versions this project builds against, but it is an assumption,+ not a guarantee, so it is written down rather than left implicit.+ """+ with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED,+ compresslevel=ZIP_LEVEL) as z:+ for arcname, src in zip_members(root, adir, hname, need):+ info = zipfile.ZipInfo(arcname, date_time=ZIP_EPOCH)+ info.compress_type = zipfile.ZIP_DEFLATED+ info.create_system = 0 # always MS-DOS, never host+ info.external_attr = ZIP_MODE << 16+ with open(src, "rb") as fh:+ z.writestr(info, fh.read())++def main():ap = argparse.ArgumentParser(description=__doc__,formatter_class=argparse.RawDescriptionHelpFormatter)@@ -315,10 +365,7 @@ def main():print(" %s/%-34s %s" % (adir, name, h))if a.zip:- with zipfile.ZipFile(a.zip, "w", zipfile.ZIP_DEFLATED) as z:- z.write(os.path.join(a.out, hname), hname)- for name, _ in need:- z.write(os.path.join(a.out, adir, name), "%s/%s" % (adir, name))+ write_archive(a.zip, a.out, adir, hname, need)print("zip written: %s (%d bytes)" % (a.zip, os.path.getsize(a.zip)))if a.release is not None:@@ -326,10 +373,7 @@ def main():os.makedirs(a.dist, exist_ok=True)stem = asset_stem(tag)archive = os.path.join(a.dist, stem + ".zip")- with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as z:- z.write(os.path.join(a.out, hname), hname)- for name, _ in need:- z.write(os.path.join(a.out, adir, name), "%s/%s" % (adir, name))+ write_archive(archive, a.out, adir, hname, need)manifest = os.path.join(a.dist, stem + ".manifest.txt")with open(manifest, "w", encoding="utf-8", newline="\n") as f:f.write(manifest_text(tag, a.out, adir, hname, need, archive))diff --git a/c/tools/clean.py b/c/tools/clean.pyindex 4a791003..d30877a4 100644--- a/c/tools/clean.py+++ b/c/tools/clean.py@@ -7,13 +7,27 @@ Works from cmd.exe, PowerShell, Git Bash, or MSYS2 — no `rm` or POSIXimport globimport osimport shutil+import sys++C_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+if C_DIR not in sys.path: # `python tools/clean.py` puts tools/ first+ sys.path.insert(0, C_DIR)+from family_registry import all_families++# The engines are DERIVED, not listed. family_registry is what the release job+# already trusts to decide which engines an archive must contain (release.yml,+# "Verify the packaged archive actually runs"), so deriving from it makes one+# fact true by construction: nothing can be published that clean does not own.+#+# The hand-written list had qwen36 missing. `make clean` therefore left the old+# qwen36 binary in place, and the release Package step copies whatever sits at+# c/qwen36 without asking where it came from -- so a locally staged release+# could ship an engine that was never rebuilt. CI never saw it (fresh runners);+# a maintainer packaging on their own machine would have.+ENGINES = sorted({family.engine_artifact for family in all_families()})# Files (relative to c/) to remove if present.-FILES = [- "colibri", "colibri.exe",- "inkling", "inkling.exe",- "kimi_k3", "kimi_k3.exe",- "olmoe", "olmoe.exe",+FILES = [name for engine in ENGINES for name in (engine, engine + ".exe")] + ["glm", "glm.exe", # pre-rename name of the colibri engine"iobench", "iobench.exe","backend_cuda.o", "backend_loader.o",@@ -41,28 +55,44 @@ FILES = [# repo tracks under tests/ carries one of these extensions, and directories# (tests/fixtures/) are skipped by the isfile() check. Object files are not in# it, so COLI_V4_UNIT_*.o is removed by the same rule rather than a second one.+#+# tests/*_probe* is here for the same reason. The XDNA physical qualification+# probe is built by an explicit `make tests/xdna_physical_probe$(EXE)` and+# matched none of the test_/bench_/fuzz_ prefixes, so it survived every clean.+# A stale probe does not just waste space -- it is the owner that PRODUCES+# physical execution evidence, and a stale one reports PASS for code that is no+# longer in the tree.ARTIFACT_GLOBS = ["tests/test_*", "tests/bench_*", "tests/fuzz_*",- "COLI_V4_UNIT_*.o"]+ "tests/*_probe*", "COLI_V4_UNIT_*.o"]KEEP_EXT = (".c", ".h", ".cc", ".cpp", ".cu", ".mm", ".py", ".txt", ".json",".md", ".bin", ".sh", ".toml", ".yml", ".yaml")# Directories to remove.DIRS = ["tests/__pycache__", "build/ownership"]-removed = 0-for f in FILES:- if os.path.exists(f):- os.remove(f)- removed += 1-for pattern in ARTIFACT_GLOBS:- for f in glob.glob(pattern):- if not os.path.isfile(f): # tests/fixtures/ and friends- continue- if f.endswith(KEEP_EXT): # never a source file- continue- os.remove(f)- removed += 1-for d in DIRS:- if os.path.isdir(d):- shutil.rmtree(d)- removed += 1-print(f"clean: removed {removed} files/dirs")+def clean():+ """Remove everything above, relative to the current directory."""+ removed = 0+ for f in FILES:+ if os.path.exists(f):+ os.remove(f)+ removed += 1+ for pattern in ARTIFACT_GLOBS:+ for f in glob.glob(pattern):+ if not os.path.isfile(f): # tests/fixtures/ and friends+ continue+ if f.endswith(KEEP_EXT): # never a source file+ continue+ os.remove(f)+ removed += 1+ for d in DIRS:+ if os.path.isdir(d):+ shutil.rmtree(d)+ removed += 1+ return removed+++# Guarded, so that reading FILES (which tests/test_clean_ownership.py does, to+# check it against the release registry) cannot delete the build products the+# rest of the suite is running against. Importing this module must be inert.+if __name__ == "__main__":+ print(f"clean: removed {clean()} files/dirs")diff --git a/docs/xdna.md b/docs/xdna.mdindex d1b0b2e8..c5b19cbc 100644--- a/docs/xdna.md+++ b/docs/xdna.md@@ -157,6 +157,11 @@ The name keeps the core archive's stem and adds `-xdna`, so it sits beside`colibri-<tag>-windows-x86_64.zip`, is matched by the release job's own`sha256sum colibri-*`, and cannot be mistaken for the core download.+The archive is reproducible: the same helper and the same qualified artifacts+produce the same bytes, and therefore the same sha256, wherever they were staged+from and whatever mtimes those files carry. If you rebuild the sidecar and the+checksum moves, something in the *content* changed — that is a signal, not noise.+Before publishing, re-verify the built assets:```bash@@ -181,5 +186,5 @@ about the default path changes.- It is not enabled by having the hardware, the helper, or the package.- It does not choose between the NPU and the normal path on speed or cost.-- No speed claim is made here; none has been measured in a comparable way.+- No speed claim is made here.- It does not extend to other models or other operations.