Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
19.3 KB
Raw Blame
"""Header prerequisites are generated by the compiler, and must stay that way.

`make` tracks file timestamps, not #include graphs. A header an engine includes
but that is absent from its rule's prerequisites produces the worst kind of
build result: `make` reports success, changes nothing, and leaves a STALE
binary that looks freshly built (#1284).

#1284 closed that by listing every header by hand and checking the lists here.
The lists then became the most edited line in the Makefile -- 27 commits by 16
authors in 30 days, and a merge conflict for every open PR touching the same
rule (#1741). So the lists are gone: `-MMD -MP` in CFLAGS makes the compiler
write `<target>.d` naming each header that compile actually read, and the
Makefile includes those files. What this file checks is that the generated
dependencies really are complete, for three ways they could silently not be:

1. A recipe that compiles more than one .c in one command. GCC then writes
   only the LAST unit's dependencies, so the others' headers are missing from
   the .d with nothing to show for it. A helper source a test links is
   therefore its own object (the tests/*.o rules), and every rule in
   AUTODEP_BINS is checked to compile exactly one unit, with -MMD, by asking
   make for the commands it would run.

2. A binary built without a .d, e.g. before this change. Most rules do not
   depend on .build-config, so nothing else would force the rebuild that writes
   one. The Makefile makes each target depend on its own .d for that reason;
   the coverage test below fails on any built target whose .d is missing.

3. A .d that does not cover what its source includes. Every unconditional
   `#include "..."` of each built target's source must appear in its .d, and
   for the engines `make -q -W <header>` must actually answer "rebuild".
   Includes under #if are not required here: `uring.h` is read on Linux only,
   and a Windows build that does not depend on it is correct. What stands
   behind them instead is point 1, checked in every build flavour the host
   accepts: a compile that reads a conditional header is a one-unit compile
   with -MMD, so the header is in that flavour's .d.

The engine list is still DERIVED from the Makefile, never kept here: anything
matching `NAME$(EXE):` with a `NAME.c` beside it is an engine, and the family
registry is the cross-check that the parse found them all.
"""
import contextlib
import os
import re
import shlex
import shutil
import subprocess
import unittest
from pathlib import Path

from family_registry import FAMILIES

C_DIR = Path(__file__).resolve().parent.parent
MAKE = shutil.which("make")

RULE_RE = re.compile(r"(?m)^([A-Za-z0-9_]+)\$\(EXE\):[ \t]*(.*)$")
INCLUDE_RE = re.compile(r'^[ \t]*#[ \t]*include[ \t]*"([^"]+)"')
COND_OPEN_RE = re.compile(r"^[ \t]*#[ \t]*if")
COND_CLOSE_RE = re.compile(r"^[ \t]*#[ \t]*endif")
SOURCE_RE = re.compile(r"\.(c|cc|cpp|m)$")

# deepseek_v4 is built by Makefile.deepseek-v4 through a phony delegating
# target, so it has no `NAME$(EXE):` rule here. test_family_registry pins that.
SEPARATE_MAKEFILE = {"deepseek-v4"}


def _joined_makefile():
    text = (C_DIR / "Makefile").read_text(encoding="utf-8")
    return re.sub(r"\\\r?\n[ \t]*", " ", text).replace("\r", "")


def _engine_rules():
    """Every target built from a single same-named .c, taken from the Makefile."""
    rules = {}
    for name, prereqs in RULE_RE.findall(_joined_makefile()):
        if (C_DIR / f"{name}.c").exists():
            rules[name] = prereqs.split("#", 1)[0].split()
    return rules


@contextlib.contextmanager
def _build_config_kept():
    """Leave .build-config exactly as it was, contents and timestamp.

    The Makefile rewrites .build-config while it parses, whenever the flags
    differ from the last build's. Every make call below parses, and some pass
    CUDA=1 or CUDA_DLL=1 on purpose, so without this the suite would switch
    the tree's recorded configuration and the next `make` would relink
    everything that depends on it.
    """
    path = C_DIR / ".build-config"
    before = path.read_bytes() if path.exists() else None
    stamp = path.stat() if before is not None else None
    try:
        yield
    finally:
        if before is None:
            if path.exists():
                path.unlink()
        elif not path.exists() or path.read_bytes() != before or \
                path.stat().st_mtime_ns != stamp.st_mtime_ns:
            path.write_bytes(before)
            os.utime(path, ns=(stamp.st_atime_ns, stamp.st_mtime_ns))


def _make(*args):
    """Run make in c/, leaving the tree's build configuration untouched."""
    with _build_config_kept():
        return subprocess.run([MAKE, "--no-print-directory", *args], cwd=C_DIR,
                              text=True, encoding="utf-8", errors="replace",
                              capture_output=True, timeout=600)


def _make_vars(*names):
    """Makefile variables as make expands them, {name: words}, in one call."""
    # `make --eval` needs GNU Make 3.82; macOS ships 3.81, which does read an
    # extra makefile from stdin.
    with _build_config_kept():
        proc = subprocess.run([MAKE, "-s", "--no-print-directory", "-f", "Makefile",
                               "-f", "-", *(f"print-{n}" for n in names)], cwd=C_DIR,
                              text=True, input="print-%: ; @echo $*=$($*)\n",
                              capture_output=True, timeout=300)
    if proc.returncode != 0:
        raise AssertionError(f"could not read {' '.join(names)} from the Makefile:\n"
                             f"{proc.stderr}")
    values = {}
    for line in proc.stdout.splitlines():
        name, sep, value = line.partition("=")
        if sep and name in names:
            values[name] = value.split()
    return values


def _make_var(name):
    return _make_vars(name).get(name, [])


RULE_LINE_RE = re.compile(r"(?m)^([^\s#:=][^:=\n]*?):(?![=:])(.*)$")
VAR_RE = re.compile(r"\$\(([A-Za-z0-9_]+)\)")


def _rules_by_target():
    """Each rule's prerequisites as written, keyed by the file make builds.

    Rules are spelled `colibri$(EXE):`, `$(SEGMENT_BUILD_DIR)/glm.o:`,
    `$(RANSLIB):` or `$(SEGMENT_V4_OBJS): $(SEGMENT_BUILD_DIR)/%.o: ...`, so
    the target side is expanded by asking make for those variables, and a
    static pattern rule gives its prerequisites to each object it names. The
    prerequisites keep their spelling: a hand-listed header is a plain word.
    """
    # `$(foreach b,...,$(eval ...))` lines generate rules; they are not rules.
    lines = [(t, r) for t, r in RULE_LINE_RE.findall(_joined_makefile())
             if not re.search(r"\$\([a-z-]+ ", t)]
    names = sorted({v for target, _ in lines for v in VAR_RE.findall(target)})
    values = _make_vars(*names)
    rules = {}
    for target, rest in lines:
        prereqs = rest.split("#", 1)[0]
        pattern, sep, after = prereqs.partition(":")
        if sep and "%" in pattern:
            prereqs = after
        expanded = VAR_RE.sub(lambda m: " ".join(values.get(m.group(1), [])), target)
        for name in expanded.split():
            rules.setdefault(name, []).extend(prereqs.split())
    return rules


def _commands(targets, *variables):
    """The compile command make WOULD run for each target, keyed by its -o.

    -B as well as -n: an already-built target otherwise prints no recipe.
    """
    proc = _make("-Bn", *targets, *variables)
    if proc.returncode != 0:
        raise AssertionError(f"make -Bn {' '.join(variables)} failed:\n"
                             f"{proc.stderr[-2000:]}")
    commands = {}
    for line in re.sub(r"\\\r?\n", " ", proc.stdout).splitlines():
        try:
            words = shlex.split(line)
        except ValueError:
            continue
        if "-o" in words and words.index("-o") + 1 < len(words):
            commands[words[words.index("-o") + 1]] = words
    return commands


# Every build switch that changes what a compile reads, with the define that
# shows it took effect. HIP=1 gets a fixed architecture, as the CI syntax job
# does, because the default asks for a GPU the host may not have.
FLAVOURS = ((("CUDA=1",), "-DCOLI_CUDA"),
            (("CUDA_DLL=1",), "-DCOLI_CUDA"),
            (("HIP=1", "HIP_ARCH=gfx1100"), "-DCOLI_CUDA"),
            (("HIP_DLL=1",), "-DCOLI_HIP_DLL"),
            (("VK=1",), "-DCOLI_VULKAN"),
            (("XDNA=1",), "-DCOLI_XDNA"),
            (("METAL=1",), "-DCOLI_METAL"))


def _accepted_flavours(exe):
    """The build flavours this host's make accepts, as variable tuples.

    Headers under #if are read only in the flavour that turns them on
    (backend_xdna.h with XDNA=1, backend_vulkan.h with VK=1, qwen36_tier.o
    only with CUDA or HIP), so the one-unit check has to see each of them.
    Several are refused off their platform, with wording that differs per
    platform, so test the fact rather than the message, the way
    test_makefile_cuda_scope does: does colibri then get the define?
    """
    accepted = []
    for variables, define in FLAVOURS:
        proc = _make("-Bn", "colibri" + exe, *variables)
        if proc.returncode == 0 and define in proc.stdout:
            accepted.append(variables)
    return accepted


def _dep_file(target):
    return C_DIR / (os.path.splitext(target)[0] + ".d")


def _raw_deps(dep_file):
    """The prerequisites of the first rule in a .d file, spelled as make sees
    them: a test's include of "../st.h" is `tests/../st.h`, not `st.h`."""
    text = dep_file.read_text(encoding="utf-8", errors="replace")
    text = re.sub(r"\\\r?\n", " ", text)
    first = text.split("\n", 1)[0]
    _, _, deps = first.partition(": ")
    return deps.split()


def _deps_in(dep_file):
    """The same prerequisites, normalised to paths relative to c/."""
    return {os.path.normpath(d) for d in _raw_deps(dep_file)}


def _unconditional_includes(source):
    """`#include "..."` lines of a source that no #if/#ifdef encloses."""
    depth, found = 0, []
    for line in source.read_text(encoding="utf-8", errors="replace").splitlines():
        if COND_OPEN_RE.match(line):
            depth += 1
        elif COND_CLOSE_RE.match(line):
            depth = max(0, depth - 1)
        elif depth == 0:
            m = INCLUDE_RE.match(line)
            if m:
                found.append(m.group(1))
    return found


def _source_of(target, rules):
    """The .c a rule compiles: the first source among its prerequisites."""
    for word in rules.get(target, []):
        if SOURCE_RE.search(word):
            return C_DIR / word
    return None


class EngineListTest(unittest.TestCase):
    def test_the_engine_list_is_derived_and_not_empty(self):
        """A parse that matches nothing would make every other check vacuous."""
        rules = _engine_rules()
        self.assertTrue(rules, "no `NAME$(EXE):` rule with a matching NAME.c was "
                               "found -- the Makefile rule syntax changed and "
                               "this file now checks nothing")
        for family in FAMILIES:
            if family.build_target in SEPARATE_MAKEFILE:
                continue
            with self.subTest(family=family.id):
                self.assertIn(
                    family.build_target, rules,
                    f"{family.id}: registered family whose build target is not "
                    f"a checkable `{family.build_target}$(EXE):` rule")


@unittest.skipUnless(MAKE, "make is not installed")
class GeneratedDepsWiringTest(unittest.TestCase):
    """What the Makefile asks for, checked without compiling anything."""

    @classmethod
    def setUpClass(cls):
        cls.exe = (_make_var("EXE") or [""])[0]
        cls.bins = _make_var("AUTODEP_BINS")
        cls.engines = _make_var("ENGINE_RULES")

    def test_every_engine_gets_generated_deps(self):
        self.assertTrue(self.bins, "AUTODEP_BINS is empty")
        for name in _engine_rules():
            with self.subTest(engine=name):
                self.assertIn(name + self.exe, self.bins,
                              f"{name}: an engine rule outside AUTODEP_BINS keeps "
                              f"no dependencies at all once its list is gone")
        self.assertEqual(sorted(self.engines), sorted(_engine_rules()),
                         "the Makefile's ENGINE_RULES and this file's derivation "
                         "disagree about which rules are engines")

    def test_no_generated_rule_lists_headers_by_hand(self):
        """A hand-listed header on these rules is redundant and brings back the
        merge conflicts #1741 removed. Bite: add `st.h` to `colibri$(EXE):`,
        or to `$(SEGMENT_BUILD_DIR)/glm.o:`."""
        rules = _rules_by_target()
        problems = []
        for target in self.bins:
            if target not in rules:
                problems.append(f"{target}: no rule found for it, so its prerequisites "
                                f"cannot be checked")
                continue
            listed = [w for w in rules[target]
                      if re.search(r"\.(h|inc)$", w) and not w.startswith("$")]
            if listed:
                problems.append(f"{target}: {' '.join(listed)}")
        self.assertEqual(problems, [],
                         "these rules get their headers from -MMD and should "
                         "not list them by hand:\n  " + "\n  ".join(problems))

    def _one_unit_problems(self, *variables):
        commands = _commands(self.bins, *variables)
        problems = []
        for target in self.bins:
            words = commands.get(target)
            if words is None:
                problems.append(f"{target}: make printed no command producing it")
                continue
            units = sorted({w for w in words if SOURCE_RE.search(w)})
            if "-MMD" not in words:
                problems.append(f"{target}: compiled without -MMD, so no .d is "
                                f"written (is it built with $(CFLAGS)?)")
            elif len(units) != 1:
                problems.append(f"{target}: compiles {len(units)} units in one "
                                f"command ({' '.join(units)}); GCC writes only the "
                                f"last one's dependencies. Compile the extra "
                                f"sources to objects, as the tests/*.o rules do")
        return problems

    def test_every_generated_rule_compiles_one_unit_with_mmd(self):
        """Point 1 of the module docstring, default build. Bite: put
        segment_runtime.c back on test_segment_runtime's command line."""
        problems = self._one_unit_problems()
        self.assertEqual(problems, [], "\n  " + "\n  ".join(problems))

    def test_every_generated_rule_compiles_one_unit_in_every_build_flavour(self):
        """The same in every flavour this host accepts (VK=1, XDNA=1, the GPU
        builds, ...). A header under #if reaches a .d only through the compile
        that reads it, so this is what stands behind the conditional includes
        the coverage test below cannot require. Bite: put qwen36_tier.c back
        on qwen36's command line; the GPU flavour then fails."""
        flavours = _accepted_flavours(self.exe)
        if not flavours:
            self.skipTest("this host accepts no build flavour beyond the default")
        for variables in flavours:
            with self.subTest(flavour=" ".join(variables)):
                problems = self._one_unit_problems(*variables)
                self.assertEqual(problems, [], "\n  " + "\n  ".join(problems))

    def test_the_checks_leave_the_build_config_alone(self):
        """Parsing with another flavour rewrites .build-config; the checks
        above do exactly that, and must not change which configuration the
        tree says it was built with. Bite: drop the `with _build_config_kept()`
        from `_make` and this fails whenever a flavour is accepted here."""
        path = C_DIR / ".build-config"
        before = (path.read_bytes(), path.stat().st_mtime_ns) if path.exists() else None
        _accepted_flavours(self.exe)
        after = (path.read_bytes(), path.stat().st_mtime_ns) if path.exists() else None
        self.assertEqual(after, before, ".build-config was changed by a make call "
                                        "that only meant to read the Makefile")


@unittest.skipUnless(MAKE, "make is not installed")
class GeneratedDepsCoverageTest(unittest.TestCase):
    """After a build: every built target's .d exists and covers its includes."""

    @classmethod
    def setUpClass(cls):
        cls.exe = (_make_var("EXE") or [""])[0]
        cls.built = [t for t in _make_var("AUTODEP_BINS") if (C_DIR / t).exists()]
        if not cls.built:
            raise unittest.SkipTest("nothing in AUTODEP_BINS is built here; run "
                                    "this after `make check`")

    def test_every_built_target_has_a_dep_file_covering_its_includes(self):
        rules = _rules_by_target()
        problems = []
        for target in self.built:
            dep = _dep_file(target)
            if not dep.exists():
                problems.append(f"{target}: built, but has no {dep.name}; its "
                                f"headers are untracked until it is rebuilt")
                continue
            source = _source_of(target, rules)
            if source is None:
                problems.append(f"{target}: no source found in its rule")
                continue
            deps = _deps_in(dep)
            for inc in _unconditional_includes(source):
                path = source.parent / inc
                if not path.exists():
                    continue
                rel = os.path.normpath(os.path.relpath(path, C_DIR))
                if rel not in deps:
                    problems.append(f"{target}: {dep.name} does not name {rel}, "
                                    f"which {source.name} includes")
        self.assertEqual(problems, [], "\n  " + "\n  ".join(problems))

    def test_make_rebuilds_a_target_when_its_header_changes(self):
        """The end-to-end form of #1284's reproduction, with nothing touched:
        `make -q -W <header>` pretends the header is newer. Engines first, then
        tests, since `make check` builds every test but few engines. Bite:
        remove the `-include` line at the end of the Makefile and every target
        answers "up to date"."""
        engines = [n + self.exe for n in _engine_rules()]
        candidates = [t for t in engines if t in self.built] + \
                     [t for t in self.built if t.startswith("tests/")][:6]
        checked, problems = 0, []
        for target in candidates:
            dep = _dep_file(target)
            if not dep.exists():
                continue
            # `make check` builds colibri with ARCH=portable and runs the
            # tests with the default ARCH, so .build-config can legitimately
            # make a target out of date already. A -W answer means nothing then.
            if _make("-q", target).returncode != 0:
                continue
            # -W matches a prerequisite by name, so use the .d's own spelling.
            headers = [h for h in _raw_deps(dep) if h.endswith(".h")][:2]
            for header in headers:
                checked += 1
                if _make("-q", "-W", header, target).returncode != 1:
                    problems.append(f"{target}: `make -q -W {header}` says up to "
                                    f"date, though {dep.name} names it")
        if not checked:
            self.skipTest("no target is built and up to date here")
        self.assertEqual(problems, [], "\n  " + "\n  ".join(problems))


if __name__ == "__main__":
    unittest.main()