Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

build(release): make XDNA release inputs reproducible

Kenneth-Javier Ortigoza7f06a2068acf
diff --git a/c/tests/test_clean_ownership.py b/c/tests/test_clean_ownership.py
new file mode 100644
index 00000000..167c2707
--- /dev/null
+++ b/c/tests/test_clean_ownership.py
@@ -0,0 +1,153 @@
+"""Owner for what `make clean` is responsible for removing.
+
+There was no test here, and the cost of that showed up in release engineering:
+tools/clean.py carried a hand-written list of engine binaries that had never
+gained qwen36. `make clean` left the old qwen36 binary in place, and the release
+Package step copies whatever sits at c/qwen36 without asking where it came from,
+so a locally staged release could ship an engine that was never rebuilt. CI did
+not see it because CI starts from an empty runner.
+
+The fix was to derive the engine names from family_registry instead of listing
+them, and the point of this file is to keep that derivation honest: the first
+test compares the two sets mechanically, so adding a family to the registry
+cannot silently create a new uncleanable build product.
+
+clean.py is executed in a sandbox rather than in the checkout -- running the real
+thing against the real tree during `make check` would delete the binaries the
+rest of the suite is using.
+"""
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+C_DIR = os.path.dirname(HERE)
+CLEAN = os.path.join(C_DIR, "tools", "clean.py")
+
+sys.path.insert(0, C_DIR)
+from family_registry import all_families # noqa: E402
+
+
+def packaged_engines():
+ """What a release archive must contain -- the release job's own source.
+
+ .github/workflows/release.yml verifies the packaged archive against exactly
+ this registry, so it is the authority on which binaries get published.
+ """
+ return sorted({f.engine_artifact for f in all_families()})
+
+
+class CleanOwnershipTest(unittest.TestCase):
+ def _sandbox(self, files=(), dirs=()):
+ """A throwaway c/ with clean.py, its import, and some build products."""
+ tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, tmp, True)
+ os.makedirs(os.path.join(tmp, "tools"))
+ os.makedirs(os.path.join(tmp, "tests"))
+ shutil.copy(CLEAN, os.path.join(tmp, "tools", "clean.py"))
+ shutil.copy(os.path.join(C_DIR, "family_registry.py"), tmp)
+ for rel in files:
+ path = os.path.join(tmp, rel)
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "wb") as fh:
+ fh.write(b"build product")
+ for rel in dirs:
+ os.makedirs(os.path.join(tmp, rel), exist_ok=True)
+ return tmp
+
+ def _run_clean(self, tmp):
+ return subprocess.run([sys.executable, os.path.join("tools", "clean.py")],
+ cwd=tmp, capture_output=True, text=True, timeout=120)
+
+ # ---- the invariant -----------------------------------------------------
+
+ def test_every_packaged_engine_is_owned_by_clean(self):
+ """PACKAGED_BUT_NOT_CLEANED must be 0, by construction not by vigilance."""
+ import importlib.util
+ spec = importlib.util.spec_from_file_location("coli_clean", CLEAN)
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ owned = set(mod.FILES)
+ missing = [e for e in packaged_engines()
+ if e not in owned or e + ".exe" not in owned]
+ self.assertEqual(missing, [], "release-packaged but not cleaned: %s" % missing)
+
+ def test_qwen36_specifically(self):
+ """The one that was actually missing. Pinned by name so it stays fixed."""
+ self.assertIn("qwen36", packaged_engines())
+ import importlib.util
+ spec = importlib.util.spec_from_file_location("coli_clean", CLEAN)
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ self.assertIn("qwen36", mod.FILES)
+ self.assertIn("qwen36.exe", mod.FILES)
+
+ def test_importing_the_module_is_inert(self):
+ """Reading FILES must not delete anything.
+
+ This is not hypothetical: the first version of the test above imported
+ clean.py with exec_module while `make check` had cwd=c/, and the
+ module-level delete loop ran against the real tree, removing the
+ binaries the rest of the suite was about to use. The action now lives
+ behind __main__.
+ """
+ import importlib.util
+ tmp = self._sandbox(files=["colibri.exe", "qwen36.exe"])
+ cwd = os.getcwd()
+ os.chdir(tmp)
+ try:
+ spec = importlib.util.spec_from_file_location("coli_clean_inert", CLEAN)
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ finally:
+ os.chdir(cwd)
+ for rel in ("colibri.exe", "qwen36.exe"):
+ self.assertTrue(os.path.exists(os.path.join(tmp, rel)),
+ "importing clean.py deleted %s" % rel)
+
+ # ---- behaviour ---------------------------------------------------------
+
+ def test_clean_removes_engine_binaries(self):
+ names = [e + ext for e in packaged_engines() for ext in ("", ".exe")]
+ tmp = self._sandbox(files=names)
+ self._run_clean(tmp)
+ survivors = [n for n in names if os.path.exists(os.path.join(tmp, n))]
+ self.assertEqual(survivors, [], "survived clean: %s" % survivors)
+
+ def test_clean_removes_the_physical_probe(self):
+ """A stale probe reports PASS for code that is no longer in the tree."""
+ tmp = self._sandbox(files=["tests/xdna_physical_probe",
+ "tests/xdna_physical_probe.exe"])
+ self._run_clean(tmp)
+ for rel in ("tests/xdna_physical_probe", "tests/xdna_physical_probe.exe"):
+ self.assertFalse(os.path.exists(os.path.join(tmp, rel)), rel)
+
+ def test_clean_does_not_delete_sources_or_unrelated_files(self):
+ keep = ["tests/xdna_physical_probe.c", "tests/test_something.c",
+ "tests/test_fixture.json", "tests/notes.md", "colibri.c",
+ "my_notes.txt", "tests/data.bin", "tools/clean.py"]
+ tmp = self._sandbox(files=[k for k in keep if k != "tools/clean.py"],
+ dirs=["tests/fixtures"])
+ self._run_clean(tmp)
+ for rel in keep:
+ self.assertTrue(os.path.exists(os.path.join(tmp, rel)),
+ "clean deleted %s" % rel)
+ self.assertTrue(os.path.isdir(os.path.join(tmp, "tests", "fixtures")))
+
+ def test_clean_is_idempotent(self):
+ tmp = self._sandbox(files=["colibri.exe", "qwen36.exe",
+ "tests/test_x.exe"])
+ first = self._run_clean(tmp)
+ self.assertEqual(first.returncode, 0, first.stderr)
+ before = sorted(os.listdir(tmp))
+ second = self._run_clean(tmp)
+ self.assertEqual(second.returncode, 0, second.stderr)
+ self.assertEqual(sorted(os.listdir(tmp)), before)
+ self.assertIn("removed 0", second.stdout)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_xdna_package.py b/c/tests/test_xdna_package.py
index d4a3a733..ec11471f 100644
--- a/c/tests/test_xdna_package.py
+++ b/c/tests/test_xdna_package.py
@@ -209,5 +209,93 @@ class TestSidecarReleaseProcess(unittest.TestCase):
self.assertIn(flag, out)
+
+class TestDeterministicArchive(unittest.TestCase):
+ """The published sha256 must be a property of the CONTENT, not the builder.
+
+ ZipFile.write() stores each file's mtime and derives create_system and the
+ external attributes from the host, so two maintainers staging identical
+ bytes produced different archives. N8-A6-R1 and N8-F0 built the same five
+ files and published dcf30127... and 74551ed2... for them.
+ """
+
+ @classmethod
+ def setUpClass(cls):
+ cls.tool = load_tool()
+
+ def _tree(self, tmp, name, mtime, helper=b"synthetic-helper"):
+ """A staging tree whose files carry a chosen mtime."""
+ tool = self.tool
+ adir, hname = tool.artifact_dir_name(), tool.helper_name()
+ root = os.path.join(tmp, name)
+ os.makedirs(os.path.join(root, adir), exist_ok=True)
+ with open(os.path.join(root, hname), "wb") as f:
+ f.write(helper)
+ for artifact, _ in tool.required_files():
+ with open(os.path.join(root, adir, artifact), "wb") as f:
+ f.write(b"synthetic-" + artifact.encode())
+ for dirpath, _dirs, files in os.walk(root):
+ for f in files:
+ os.utime(os.path.join(dirpath, f), (mtime, mtime))
+ return root
+
+ def _build(self, tmp, root, out):
+ tool = self.tool
+ path = os.path.join(tmp, out)
+ tool.write_archive(path, root, tool.artifact_dir_name(),
+ tool.helper_name(), tool.required_files())
+ with open(path, "rb") as fh:
+ return hashlib.sha256(fh.read()).hexdigest(), path
+
+ def test_archive_is_identical_across_staging_paths_and_mtimes(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ a = self._tree(tmp, "stage-a", 1_566_000_000) # 2019
+ b = self._tree(tmp, "deeper/stage-b", 1_956_528_000) # 2032
+ ha, _ = self._build(tmp, a, "a.zip")
+ hb, _ = self._build(tmp, b, "b.zip")
+ self.assertEqual(ha, hb,
+ "archive hash still depends on staging mtime/path")
+
+ def test_a_changed_member_byte_changes_the_archive(self):
+ """Reproducibility must not mean insensitivity."""
+ with tempfile.TemporaryDirectory() as tmp:
+ a = self._tree(tmp, "same", 1_566_000_000)
+ b = self._tree(tmp, "diff", 1_566_000_000, helper=b"synthetic-helpeR")
+ ha, _ = self._build(tmp, a, "a.zip")
+ hb, _ = self._build(tmp, b, "b.zip")
+ self.assertNotEqual(ha, hb)
+
+ def test_member_metadata_is_pinned_not_inherited(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = self._tree(tmp, "stage", 1_566_000_000)
+ _, path = self._build(tmp, root, "a.zip")
+ with zipfile.ZipFile(path) as z:
+ for info in z.infolist():
+ self.assertEqual(info.date_time, self.tool.ZIP_EPOCH)
+ self.assertEqual(info.create_system, 0)
+ self.assertEqual(info.external_attr,
+ self.tool.ZIP_MODE << 16)
+ self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED)
+
+ def test_member_order_is_sorted_not_filesystem_order(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = self._tree(tmp, "stage", 1_566_000_000)
+ _, path = self._build(tmp, root, "a.zip")
+ with zipfile.ZipFile(path) as z:
+ stored = [i.filename for i in z.infolist()]
+ self.assertEqual(stored, sorted(stored))
+
+ def test_deterministic_archive_still_holds_exactly_the_asset_set(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ root = self._tree(tmp, "stage", 1_566_000_000)
+ _, path = self._build(tmp, root, "a.zip")
+ adir = self.tool.artifact_dir_name()
+ expected = sorted(
+ [self.tool.helper_name()]
+ + ["%s/%s" % (adir, n) for n, _ in self.tool.required_files()])
+ with zipfile.ZipFile(path) as z:
+ self.assertEqual(sorted(z.namelist()), expected)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/c/tools/build_xdna_package.py b/c/tools/build_xdna_package.py
index 04b46181..75440460 100644
--- a/c/tools/build_xdna_package.py
+++ b/c/tools/build_xdna_package.py
@@ -236,6 +236,56 @@ def verify_release(archive, tag_expected=None):
return problems
+# ---- the reproducible archive ------------------------------------------
+#
+# Two maintainers with the same helper and the same qualified artifacts must
+# produce the same archive, byte for byte -- otherwise the published sha256 is
+# a property of whoever built it rather than of what is inside.
+#
+# ZipFile.write() defeats that on its own: it reads each file's mtime and
+# stores it in the local header and the central directory, and it derives
+# create_system and external_attr from the host OS and the file mode. So the
+# same bytes staged from two directories produced two different archives --
+# which is exactly what happened between N8-A6-R1 (dcf30127...) and N8-F0
+# (74551ed2...), for content that was identical.
+#
+# Everything that varies is therefore pinned:
+ZIP_EPOCH = (1980, 1, 1, 0, 0, 0) # the earliest a ZIP timestamp can express
+ZIP_MODE = 0o644 # not the staging file's mode
+ZIP_LEVEL = 9 # explicit, not the zlib default
+
+
+def zip_members(root, adir, hname, need):
+ """(archive_path, source_path) in a fixed order, independent of the fs."""
+ members = [(hname, os.path.join(root, hname))]
+ for name, _ in need:
+ members.append(("%s/%s" % (adir, name), os.path.join(root, adir, name)))
+ return sorted(members)
+
+
+def write_archive(path, root, adir, hname, need):
+ """Write the sidecar archive deterministically.
+
+ Depends on nothing but the member paths and the member bytes: not the
+ staging mtimes, not the staging path, not the host OS, not the locale,
+ not the order the filesystem happens to return.
+
+ The one residual assumption is that zlib emits the same deflate stream for
+ the same input at the same level. That is true in practice and stable
+ across the versions this project builds against, but it is an assumption,
+ not a guarantee, so it is written down rather than left implicit.
+ """
+ with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED,
+ compresslevel=ZIP_LEVEL) as z:
+ for arcname, src in zip_members(root, adir, hname, need):
+ info = zipfile.ZipInfo(arcname, date_time=ZIP_EPOCH)
+ info.compress_type = zipfile.ZIP_DEFLATED
+ info.create_system = 0 # always MS-DOS, never host
+ info.external_attr = ZIP_MODE << 16
+ with open(src, "rb") as fh:
+ z.writestr(info, fh.read())
+
+
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
@@ -315,10 +365,7 @@ def main():
print(" %s/%-34s %s" % (adir, name, h))
if a.zip:
- with zipfile.ZipFile(a.zip, "w", zipfile.ZIP_DEFLATED) as z:
- z.write(os.path.join(a.out, hname), hname)
- for name, _ in need:
- z.write(os.path.join(a.out, adir, name), "%s/%s" % (adir, name))
+ write_archive(a.zip, a.out, adir, hname, need)
print("zip written: %s (%d bytes)" % (a.zip, os.path.getsize(a.zip)))
if a.release is not None:
@@ -326,10 +373,7 @@ def main():
os.makedirs(a.dist, exist_ok=True)
stem = asset_stem(tag)
archive = os.path.join(a.dist, stem + ".zip")
- with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as z:
- z.write(os.path.join(a.out, hname), hname)
- for name, _ in need:
- z.write(os.path.join(a.out, adir, name), "%s/%s" % (adir, name))
+ write_archive(archive, a.out, adir, hname, need)
manifest = os.path.join(a.dist, stem + ".manifest.txt")
with open(manifest, "w", encoding="utf-8", newline="\n") as f:
f.write(manifest_text(tag, a.out, adir, hname, need, archive))
diff --git a/c/tools/clean.py b/c/tools/clean.py
index 4a791003..d30877a4 100644
--- a/c/tools/clean.py
+++ b/c/tools/clean.py
@@ -7,13 +7,27 @@ Works from cmd.exe, PowerShell, Git Bash, or MSYS2 — no `rm` or POSIX
import glob
import os
import shutil
+import sys
+
+C_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if C_DIR not in sys.path: # `python tools/clean.py` puts tools/ first
+ sys.path.insert(0, C_DIR)
+from family_registry import all_families
+
+# The engines are DERIVED, not listed. family_registry is what the release job
+# already trusts to decide which engines an archive must contain (release.yml,
+# "Verify the packaged archive actually runs"), so deriving from it makes one
+# fact true by construction: nothing can be published that clean does not own.
+#
+# The hand-written list had qwen36 missing. `make clean` therefore left the old
+# qwen36 binary in place, and the release Package step copies whatever sits at
+# c/qwen36 without asking where it came from -- so a locally staged release
+# could ship an engine that was never rebuilt. CI never saw it (fresh runners);
+# a maintainer packaging on their own machine would have.
+ENGINES = sorted({family.engine_artifact for family in all_families()})
# Files (relative to c/) to remove if present.
-FILES = [
- "colibri", "colibri.exe",
- "inkling", "inkling.exe",
- "kimi_k3", "kimi_k3.exe",
- "olmoe", "olmoe.exe",
+FILES = [name for engine in ENGINES for name in (engine, engine + ".exe")] + [
"glm", "glm.exe", # pre-rename name of the colibri engine
"iobench", "iobench.exe",
"backend_cuda.o", "backend_loader.o",
@@ -41,28 +55,44 @@ FILES = [
# repo tracks under tests/ carries one of these extensions, and directories
# (tests/fixtures/) are skipped by the isfile() check. Object files are not in
# it, so COLI_V4_UNIT_*.o is removed by the same rule rather than a second one.
+#
+# tests/*_probe* is here for the same reason. The XDNA physical qualification
+# probe is built by an explicit `make tests/xdna_physical_probe$(EXE)` and
+# matched none of the test_/bench_/fuzz_ prefixes, so it survived every clean.
+# A stale probe does not just waste space -- it is the owner that PRODUCES
+# physical execution evidence, and a stale one reports PASS for code that is no
+# longer in the tree.
ARTIFACT_GLOBS = ["tests/test_*", "tests/bench_*", "tests/fuzz_*",
- "COLI_V4_UNIT_*.o"]
+ "tests/*_probe*", "COLI_V4_UNIT_*.o"]
KEEP_EXT = (".c", ".h", ".cc", ".cpp", ".cu", ".mm", ".py", ".txt", ".json",
".md", ".bin", ".sh", ".toml", ".yml", ".yaml")
# Directories to remove.
DIRS = ["tests/__pycache__", "build/ownership"]
-removed = 0
-for f in FILES:
- if os.path.exists(f):
- os.remove(f)
- removed += 1
-for pattern in ARTIFACT_GLOBS:
- for f in glob.glob(pattern):
- if not os.path.isfile(f): # tests/fixtures/ and friends
- continue
- if f.endswith(KEEP_EXT): # never a source file
- continue
- os.remove(f)
- removed += 1
-for d in DIRS:
- if os.path.isdir(d):
- shutil.rmtree(d)
- removed += 1
-print(f"clean: removed {removed} files/dirs")
+def clean():
+ """Remove everything above, relative to the current directory."""
+ removed = 0
+ for f in FILES:
+ if os.path.exists(f):
+ os.remove(f)
+ removed += 1
+ for pattern in ARTIFACT_GLOBS:
+ for f in glob.glob(pattern):
+ if not os.path.isfile(f): # tests/fixtures/ and friends
+ continue
+ if f.endswith(KEEP_EXT): # never a source file
+ continue
+ os.remove(f)
+ removed += 1
+ for d in DIRS:
+ if os.path.isdir(d):
+ shutil.rmtree(d)
+ removed += 1
+ return removed
+
+
+# Guarded, so that reading FILES (which tests/test_clean_ownership.py does, to
+# check it against the release registry) cannot delete the build products the
+# rest of the suite is running against. Importing this module must be inert.
+if __name__ == "__main__":
+ print(f"clean: removed {clean()} files/dirs")
diff --git a/docs/xdna.md b/docs/xdna.md
index d1b0b2e8..c5b19cbc 100644
--- a/docs/xdna.md
+++ b/docs/xdna.md
@@ -157,6 +157,11 @@ The name keeps the core archive's stem and adds `-xdna`, so it sits beside
`colibri-<tag>-windows-x86_64.zip`, is matched by the release job's own
`sha256sum colibri-*`, and cannot be mistaken for the core download.
+The archive is reproducible: the same helper and the same qualified artifacts
+produce the same bytes, and therefore the same sha256, wherever they were staged
+from and whatever mtimes those files carry. If you rebuild the sidecar and the
+checksum moves, something in the *content* changed — that is a signal, not noise.
+
Before publishing, re-verify the built assets:
```bash
@@ -181,5 +186,5 @@ about the default path changes.
- It is not enabled by having the hardware, the helper, or the package.
- It does not choose between the NPU and the normal path on speed or cost.
-- No speed claim is made here; none has been measured in a comparable way.
+- No speed claim is made here.
- It does not extend to other models or other operations.