Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
"""Header prerequisites are generated by the compiler, and must stay that way.
`make` tracks file timestamps, not #include graphs. A header an engine includes
but that is absent from its rule's prerequisites produces the worst kind of
build result: `make` reports success, changes nothing, and leaves a STALE
binary that looks freshly built (#1284).
#1284 closed that by listing every header by hand and checking the lists here.
The lists then became the most edited line in the Makefile -- 27 commits by 16
authors in 30 days, and a merge conflict for every open PR touching the same
rule (#1741). So the lists are gone: `-MMD -MP` in CFLAGS makes the compiler
write `<target>.d` naming each header that compile actually read, and the
Makefile includes those files. What this file checks is that the generated
dependencies really are complete, for three ways they could silently not be:
1. A recipe that compiles more than one .c in one command. GCC then writes
only the LAST unit's dependencies, so the others' headers are missing from
the .d with nothing to show for it. A helper source a test links is
therefore its own object (the tests/*.o rules), and every rule in
AUTODEP_BINS is checked to compile exactly one unit, with -MMD, by asking
make for the commands it would run.
2. A binary built without a .d, e.g. before this change. Most rules do not
depend on .build-config, so nothing else would force the rebuild that writes
one. The Makefile makes each target depend on its own .d for that reason;
the coverage test below fails on any built target whose .d is missing.
3. A .d that does not cover what its source includes. Every unconditional
`#include "..."` of each built target's source must appear in its .d, and
for the engines `make -q -W <header>` must actually answer "rebuild".
Includes under #if are not required here: `uring.h` is read on Linux only,
and a Windows build that does not depend on it is correct. What stands
behind them instead is point 1, checked in every build flavour the host
accepts: a compile that reads a conditional header is a one-unit compile
with -MMD, so the header is in that flavour's .d.
The engine list is still DERIVED from the Makefile, never kept here: anything
matching `NAME$(EXE):` with a `NAME.c` beside it is an engine, and the family
registry is the cross-check that the parse found them all.
"""
import contextlib
import os
import re
import shlex
import shutil
import subprocess
import unittest
from pathlib import Path
from family_registry import FAMILIES
C_DIR = Path(__file__).resolve().parent.parent
MAKE = shutil.which("make")
RULE_RE = re.compile(r"(?m)^([A-Za-z0-9_]+)\$\(EXE\):[ \t]*(.*)$")
INCLUDE_RE = re.compile(r'^[ \t]*#[ \t]*include[ \t]*"([^"]+)"')
COND_OPEN_RE = re.compile(r"^[ \t]*#[ \t]*if")
COND_CLOSE_RE = re.compile(r"^[ \t]*#[ \t]*endif")
SOURCE_RE = re.compile(r"\.(c|cc|cpp|m)$")
# deepseek_v4 is built by Makefile.deepseek-v4 through a phony delegating
# target, so it has no `NAME$(EXE):` rule here. test_family_registry pins that.
SEPARATE_MAKEFILE = {"deepseek-v4"}
def _joined_makefile():
text = (C_DIR / "Makefile").read_text(encoding="utf-8")
return re.sub(r"\\\r?\n[ \t]*", " ", text).replace("\r", "")
def _engine_rules():
"""Every target built from a single same-named .c, taken from the Makefile."""
rules = {}
for name, prereqs in RULE_RE.findall(_joined_makefile()):
if (C_DIR / f"{name}.c").exists():
rules[name] = prereqs.split("#", 1)[0].split()
return rules
@contextlib.contextmanager
def _build_config_kept():
"""Leave .build-config exactly as it was, contents and timestamp.
The Makefile rewrites .build-config while it parses, whenever the flags
differ from the last build's. Every make call below parses, and some pass
CUDA=1 or CUDA_DLL=1 on purpose, so without this the suite would switch
the tree's recorded configuration and the next `make` would relink
everything that depends on it.
"""
path = C_DIR / ".build-config"
before = path.read_bytes() if path.exists() else None
stamp = path.stat() if before is not None else None
try:
yield
finally:
if before is None:
if path.exists():
path.unlink()
elif not path.exists() or path.read_bytes() != before or \
path.stat().st_mtime_ns != stamp.st_mtime_ns:
path.write_bytes(before)
os.utime(path, ns=(stamp.st_atime_ns, stamp.st_mtime_ns))
def _make(*args):
"""Run make in c/, leaving the tree's build configuration untouched."""
with _build_config_kept():
return subprocess.run([MAKE, "--no-print-directory", *args], cwd=C_DIR,
text=True, encoding="utf-8", errors="replace",
capture_output=True, timeout=600)
def _make_vars(*names):
"""Makefile variables as make expands them, {name: words}, in one call."""
# `make --eval` needs GNU Make 3.82; macOS ships 3.81, which does read an
# extra makefile from stdin.
with _build_config_kept():
proc = subprocess.run([MAKE, "-s", "--no-print-directory", "-f", "Makefile",
"-f", "-", *(f"print-{n}" for n in names)], cwd=C_DIR,
text=True, input="print-%: ; @echo $*=$($*)\n",
capture_output=True, timeout=300)
if proc.returncode != 0:
raise AssertionError(f"could not read {' '.join(names)} from the Makefile:\n"
f"{proc.stderr}")
values = {}
for line in proc.stdout.splitlines():
name, sep, value = line.partition("=")
if sep and name in names:
values[name] = value.split()
return values
def _make_var(name):
return _make_vars(name).get(name, [])
RULE_LINE_RE = re.compile(r"(?m)^([^\s#:=][^:=\n]*?):(?![=:])(.*)$")
VAR_RE = re.compile(r"\$\(([A-Za-z0-9_]+)\)")
def _rules_by_target():
"""Each rule's prerequisites as written, keyed by the file make builds.
Rules are spelled `colibri$(EXE):`, `$(SEGMENT_BUILD_DIR)/glm.o:`,
`$(RANSLIB):` or `$(SEGMENT_V4_OBJS): $(SEGMENT_BUILD_DIR)/%.o: ...`, so
the target side is expanded by asking make for those variables, and a
static pattern rule gives its prerequisites to each object it names. The
prerequisites keep their spelling: a hand-listed header is a plain word.
"""
# `$(foreach b,...,$(eval ...))` lines generate rules; they are not rules.
lines = [(t, r) for t, r in RULE_LINE_RE.findall(_joined_makefile())
if not re.search(r"\$\([a-z-]+ ", t)]
names = sorted({v for target, _ in lines for v in VAR_RE.findall(target)})
values = _make_vars(*names)
rules = {}
for target, rest in lines:
prereqs = rest.split("#", 1)[0]
pattern, sep, after = prereqs.partition(":")
if sep and "%" in pattern:
prereqs = after
expanded = VAR_RE.sub(lambda m: " ".join(values.get(m.group(1), [])), target)
for name in expanded.split():
rules.setdefault(name, []).extend(prereqs.split())
return rules
def _commands(targets, *variables):
"""The compile command make WOULD run for each target, keyed by its -o.
-B as well as -n: an already-built target otherwise prints no recipe.
"""
proc = _make("-Bn", *targets, *variables)
if proc.returncode != 0:
raise AssertionError(f"make -Bn {' '.join(variables)} failed:\n"
f"{proc.stderr[-2000:]}")
commands = {}
for line in re.sub(r"\\\r?\n", " ", proc.stdout).splitlines():
try:
words = shlex.split(line)
except ValueError:
continue
if "-o" in words and words.index("-o") + 1 < len(words):
commands[words[words.index("-o") + 1]] = words
return commands
# Every build switch that changes what a compile reads, with the define that
# shows it took effect. HIP=1 gets a fixed architecture, as the CI syntax job
# does, because the default asks for a GPU the host may not have.
FLAVOURS = ((("CUDA=1",), "-DCOLI_CUDA"),
(("CUDA_DLL=1",), "-DCOLI_CUDA"),
(("HIP=1", "HIP_ARCH=gfx1100"), "-DCOLI_CUDA"),
(("HIP_DLL=1",), "-DCOLI_HIP_DLL"),
(("VK=1",), "-DCOLI_VULKAN"),
(("XDNA=1",), "-DCOLI_XDNA"),
(("METAL=1",), "-DCOLI_METAL"))
def _accepted_flavours(exe):
"""The build flavours this host's make accepts, as variable tuples.
Headers under #if are read only in the flavour that turns them on
(backend_xdna.h with XDNA=1, backend_vulkan.h with VK=1, qwen36_tier.o
only with CUDA or HIP), so the one-unit check has to see each of them.
Several are refused off their platform, with wording that differs per
platform, so test the fact rather than the message, the way
test_makefile_cuda_scope does: does colibri then get the define?
"""
accepted = []
for variables, define in FLAVOURS:
proc = _make("-Bn", "colibri" + exe, *variables)
if proc.returncode == 0 and define in proc.stdout:
accepted.append(variables)
return accepted
def _dep_file(target):
return C_DIR / (os.path.splitext(target)[0] + ".d")
def _raw_deps(dep_file):
"""The prerequisites of the first rule in a .d file, spelled as make sees
them: a test's include of "../st.h" is `tests/../st.h`, not `st.h`."""
text = dep_file.read_text(encoding="utf-8", errors="replace")
text = re.sub(r"\\\r?\n", " ", text)
first = text.split("\n", 1)[0]
_, _, deps = first.partition(": ")
return deps.split()
def _deps_in(dep_file):
"""The same prerequisites, normalised to paths relative to c/."""
return {os.path.normpath(d) for d in _raw_deps(dep_file)}
def _unconditional_includes(source):
"""`#include "..."` lines of a source that no #if/#ifdef encloses."""
depth, found = 0, []
for line in source.read_text(encoding="utf-8", errors="replace").splitlines():
if COND_OPEN_RE.match(line):
depth += 1
elif COND_CLOSE_RE.match(line):
depth = max(0, depth - 1)
elif depth == 0:
m = INCLUDE_RE.match(line)
if m:
found.append(m.group(1))
return found
def _source_of(target, rules):
"""The .c a rule compiles: the first source among its prerequisites."""
for word in rules.get(target, []):
if SOURCE_RE.search(word):
return C_DIR / word
return None
class EngineListTest(unittest.TestCase):
def test_the_engine_list_is_derived_and_not_empty(self):
"""A parse that matches nothing would make every other check vacuous."""
rules = _engine_rules()
self.assertTrue(rules, "no `NAME$(EXE):` rule with a matching NAME.c was "
"found -- the Makefile rule syntax changed and "
"this file now checks nothing")
for family in FAMILIES:
if family.build_target in SEPARATE_MAKEFILE:
continue
with self.subTest(family=family.id):
self.assertIn(
family.build_target, rules,
f"{family.id}: registered family whose build target is not "
f"a checkable `{family.build_target}$(EXE):` rule")
@unittest.skipUnless(MAKE, "make is not installed")
class GeneratedDepsWiringTest(unittest.TestCase):
"""What the Makefile asks for, checked without compiling anything."""
@classmethod
def setUpClass(cls):
cls.exe = (_make_var("EXE") or [""])[0]
cls.bins = _make_var("AUTODEP_BINS")
cls.engines = _make_var("ENGINE_RULES")
def test_every_engine_gets_generated_deps(self):
self.assertTrue(self.bins, "AUTODEP_BINS is empty")
for name in _engine_rules():
with self.subTest(engine=name):
self.assertIn(name + self.exe, self.bins,
f"{name}: an engine rule outside AUTODEP_BINS keeps "
f"no dependencies at all once its list is gone")
self.assertEqual(sorted(self.engines), sorted(_engine_rules()),
"the Makefile's ENGINE_RULES and this file's derivation "
"disagree about which rules are engines")
def test_no_generated_rule_lists_headers_by_hand(self):
"""A hand-listed header on these rules is redundant and brings back the
merge conflicts #1741 removed. Bite: add `st.h` to `colibri$(EXE):`,
or to `$(SEGMENT_BUILD_DIR)/glm.o:`."""
rules = _rules_by_target()
problems = []
for target in self.bins:
if target not in rules:
problems.append(f"{target}: no rule found for it, so its prerequisites "
f"cannot be checked")
continue
listed = [w for w in rules[target]
if re.search(r"\.(h|inc)$", w) and not w.startswith("$")]
if listed:
problems.append(f"{target}: {' '.join(listed)}")
self.assertEqual(problems, [],
"these rules get their headers from -MMD and should "
"not list them by hand:\n " + "\n ".join(problems))
def _one_unit_problems(self, *variables):
commands = _commands(self.bins, *variables)
problems = []
for target in self.bins:
words = commands.get(target)
if words is None:
problems.append(f"{target}: make printed no command producing it")
continue
units = sorted({w for w in words if SOURCE_RE.search(w)})
if "-MMD" not in words:
problems.append(f"{target}: compiled without -MMD, so no .d is "
f"written (is it built with $(CFLAGS)?)")
elif len(units) != 1:
problems.append(f"{target}: compiles {len(units)} units in one "
f"command ({' '.join(units)}); GCC writes only the "
f"last one's dependencies. Compile the extra "
f"sources to objects, as the tests/*.o rules do")
return problems
def test_every_generated_rule_compiles_one_unit_with_mmd(self):
"""Point 1 of the module docstring, default build. Bite: put
segment_runtime.c back on test_segment_runtime's command line."""
problems = self._one_unit_problems()
self.assertEqual(problems, [], "\n " + "\n ".join(problems))
def test_every_generated_rule_compiles_one_unit_in_every_build_flavour(self):
"""The same in every flavour this host accepts (VK=1, XDNA=1, the GPU
builds, ...). A header under #if reaches a .d only through the compile
that reads it, so this is what stands behind the conditional includes
the coverage test below cannot require. Bite: put qwen36_tier.c back
on qwen36's command line; the GPU flavour then fails."""
flavours = _accepted_flavours(self.exe)
if not flavours:
self.skipTest("this host accepts no build flavour beyond the default")
for variables in flavours:
with self.subTest(flavour=" ".join(variables)):
problems = self._one_unit_problems(*variables)
self.assertEqual(problems, [], "\n " + "\n ".join(problems))
def test_the_checks_leave_the_build_config_alone(self):
"""Parsing with another flavour rewrites .build-config; the checks
above do exactly that, and must not change which configuration the
tree says it was built with. Bite: drop the `with _build_config_kept()`
from `_make` and this fails whenever a flavour is accepted here."""
path = C_DIR / ".build-config"
before = (path.read_bytes(), path.stat().st_mtime_ns) if path.exists() else None
_accepted_flavours(self.exe)
after = (path.read_bytes(), path.stat().st_mtime_ns) if path.exists() else None
self.assertEqual(after, before, ".build-config was changed by a make call "
"that only meant to read the Makefile")
@unittest.skipUnless(MAKE, "make is not installed")
class GeneratedDepsCoverageTest(unittest.TestCase):
"""After a build: every built target's .d exists and covers its includes."""
@classmethod
def setUpClass(cls):
cls.exe = (_make_var("EXE") or [""])[0]
cls.built = [t for t in _make_var("AUTODEP_BINS") if (C_DIR / t).exists()]
if not cls.built:
raise unittest.SkipTest("nothing in AUTODEP_BINS is built here; run "
"this after `make check`")
def test_every_built_target_has_a_dep_file_covering_its_includes(self):
rules = _rules_by_target()
problems = []
for target in self.built:
dep = _dep_file(target)
if not dep.exists():
problems.append(f"{target}: built, but has no {dep.name}; its "
f"headers are untracked until it is rebuilt")
continue
source = _source_of(target, rules)
if source is None:
problems.append(f"{target}: no source found in its rule")
continue
deps = _deps_in(dep)
for inc in _unconditional_includes(source):
path = source.parent / inc
if not path.exists():
continue
rel = os.path.normpath(os.path.relpath(path, C_DIR))
if rel not in deps:
problems.append(f"{target}: {dep.name} does not name {rel}, "
f"which {source.name} includes")
self.assertEqual(problems, [], "\n " + "\n ".join(problems))
def test_make_rebuilds_a_target_when_its_header_changes(self):
"""The end-to-end form of #1284's reproduction, with nothing touched:
`make -q -W <header>` pretends the header is newer. Engines first, then
tests, since `make check` builds every test but few engines. Bite:
remove the `-include` line at the end of the Makefile and every target
answers "up to date"."""
engines = [n + self.exe for n in _engine_rules()]
candidates = [t for t in engines if t in self.built] + \
[t for t in self.built if t.startswith("tests/")][:6]
checked, problems = 0, []
for target in candidates:
dep = _dep_file(target)
if not dep.exists():
continue
# `make check` builds colibri with ARCH=portable and runs the
# tests with the default ARCH, so .build-config can legitimately
# make a target out of date already. A -W answer means nothing then.
if _make("-q", target).returncode != 0:
continue
# -W matches a prerequisite by name, so use the .d's own spelling.
headers = [h for h in _raw_deps(dep) if h.endswith(".h")][:2]
for header in headers:
checked += 1
if _make("-q", "-W", header, target).returncode != 1:
problems.append(f"{target}: `make -q -W {header}` says up to "
f"date, though {dep.name} names it")
if not checked:
self.skipTest("no target is built and up to date here")
self.assertEqual(problems, [], "\n " + "\n ".join(problems))
if __name__ == "__main__":
unittest.main()