Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
11.1 KB
Raw Blame
"""A link that calls the Vulkan backend must link $(VK_OBJ).

Under VK=1, CFLAGS carries -DCOLI_VULKAN. A translation unit that includes an
engine with Vulkan hooks (colibri.c, kimi_k3.c, ...) then calls coli_vk_*, and
the link fails unless the rule also links $(VK_OBJ). That variable is empty
unless VK=1, so the default build and every default CI job stay green while
`make test-c VK=1` is broken. #1728 fixed the 43 rules that
`make -k test-c VK=1` reported. Four more gates (test_xdna_qt_state,
test_xdna_failure, test_logprob_status, test_ablate_mode) arrived in pull
requests merged after it, and nine on-demand rules outside TEST_BINS -- the
benches, the XDNA physical probe, the e8x4g64 loader harness -- were never
built by test-c, so no such run could report them.

Whether a link needs the backend is a preprocessor question, not a textual
one: code can pick CUDA over Vulkan with `#elif`, a hook can be an #ifdef that
only reads an environment variable, and a test can define the coli_vk_*
functions itself to fake the device. So nothing here reads #if lines. It asks
make for the expanded VK=1 link commands (`make -Bn`), preprocesses each .c on
a line with that line's own flags, and looks at what survives: a link needs
$(VK_OBJ) when the code calls a function backend_vulkan.h declares and does
not define it. The rule must then link $(VK_OBJ) and list it as a
prerequisite, or a clean build can reach the link before backend_vulkan.o
exists (#1728 found six rules like that). Nothing is hand-listed: the rules
come from the Makefile and the API from backend_vulkan.h.
"""
import os
import re
import shutil
import subprocess
import unittest
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

C_DIR = Path(__file__).resolve().parent.parent
MAKE = shutil.which("make")

ASSIGN_RE = re.compile(
    r"^(?:override[ \t]+)?[A-Za-z0-9_]+[ \t]*(?:\+|\?|!|::?)?=")
RULE_RE = re.compile(r"^([^\t#][^=]*?)[ \t]*:(?![=:])[ \t]*(.*)$")
# Lines that are not rules even when they contain a colon, e.g.
# `$(warning mixed HIP_ARCH list: ...)`. A rule may still START with `$(`.
DIRECTIVE_RE = re.compile(
    r"^(?:ifeq|ifneq|ifdef|ifndef|else|endif|(?:-|s)?include|export"
    r"|unexport|vpath)\b|^\$\((?:error|warning|info|file|shell|eval|call)\b")
LINEMARKER_RE = re.compile(r'^#\s+\d+\s+"([^"]+)"')
STRING_RE = re.compile(r'"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'')
NAME_RE = re.compile(r"\bcoli_vk_\w+")
DEF_RE = re.compile(r"\b(coli_vk_\w+)\s*\([^;{}()]*\)\s*\{")
# Link-only and dependency-file arguments; the rest of a link line is what
# the compile saw. -MMD/-MF would make `-E` write .d files beside the sources.
SKIP_WORD = re.compile(r"^(-l|-L|-Wl,|-static|-shared|-flto|-fuse-ld|-MMD$|-MD$|-MP$)")
SKIP_PAIR = {"-o", "-MF", "-MT", "-MQ", "-framework"}
INPUT_SUFFIXES = (".c", ".o", ".a", ".so", ".dll", ".lib", ".dylib")


def _link_rules():
    """{target: normal prerequisites} for every rule with a $(CC) link recipe.

    Prerequisites are the one thing `make -n` does not print. Continuations
    are folded first, and define/endef bodies skipped.
    """
    text = (C_DIR / "Makefile").read_text(encoding="utf-8")
    text = re.sub(r"\\\n[ \t]*", " ", text)
    rules, current, in_define = {}, None, False
    for line in text.splitlines():
        if in_define:
            in_define = not line.startswith("endef")
            continue
        if line.startswith("define "):
            in_define, current = True, None
            continue
        if line.startswith("\t"):
            words = line.split()
            if current and "$(CC)" in words and "-c" not in words:
                for target in current[0]:
                    rules[target] = current[1]
            continue
        stripped = line.split("#", 1)[0].strip()
        if not stripped or DIRECTIVE_RE.match(stripped):
            continue
        if ASSIGN_RE.match(stripped):
            current = None
            continue
        rule = RULE_RE.match(stripped)
        if rule:
            prereqs = rule.group(2).split(";", 1)[0].split("|", 1)[0].split()
            current = (rule.group(1).split(), prereqs)
    return rules


def _backend_api():
    text = (C_DIR / "backend_vulkan.h").read_text(encoding="utf-8")
    return set(re.findall(r"\bcoli_vk_\w+", text))


def _make(*args):
    return subprocess.run([MAKE, "--no-print-directory", *args], cwd=C_DIR,
                          capture_output=True, text=True, errors="replace",
                          timeout=600)


def _unresolved(words, api):
    """Backend functions the link line's sources call without defining them,
    or None when this host cannot preprocess them (and so cannot build them
    either; the platforms that can are where the line gets checked)."""
    cc, flags, sources, i = words[0], [], [], 1
    while i < len(words):
        w = words[i]
        if w in SKIP_PAIR:
            i += 2
            continue
        if w.endswith(INPUT_SUFFIXES):
            if w.endswith(".c"):
                sources.append(w)
        elif w != "-c" and not SKIP_WORD.match(w):
            flags.append(w)
        i += 1
    called, defined = set(), set()
    for source in sources:
        r = subprocess.run([cc, "-E", *flags, source], cwd=C_DIR,
                           capture_output=True, text=True, errors="replace",
                           timeout=300)
        if r.returncode != 0:
            return None
        # Keep only this tree's own code: system headers cannot call the
        # backend, and backend_vulkan.h only declares it.
        kept, keep, own = [], True, {}
        for line in r.stdout.splitlines():
            marker = LINEMARKER_RE.match(line)
            if marker:
                path = marker.group(1)
                if path not in own:
                    resolved = (C_DIR / path).resolve()
                    own[path] = (not path.startswith("<")
                                 and resolved.is_relative_to(C_DIR)
                                 and resolved.name != "backend_vulkan.h")
                keep = own[path]
            elif keep:
                kept.append(line)
        code = STRING_RE.sub('""', "\n".join(kept))
        called |= set(NAME_RE.findall(code)) & api
        defined |= set(DEF_RE.findall(code)) & api
    return called - defined


@unittest.skipUnless(MAKE, "make is required")
class MakefileVkObjTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        # A dry run still rewrites .build-config at parse time; put it back so
        # running this test does not make the next real build relink.
        stamp = C_DIR / ".build-config"
        saved = (stamp.read_bytes(), stamp.stat()) if stamp.exists() else None
        try:
            # $(EXE) from make's own database. Not --eval: GNU Make 3.81,
            # macOS's /usr/bin/make, predates it; -p is older than that.
            database = _make("-pn", "VK=1", ".build-config").stdout
            exe = re.search(r"(?m)^EXE :?= ?(\S*)", database)
            if exe is None:
                raise AssertionError("make -pn printed no EXE variable")
            cls.rules = {t.replace("$(EXE)", exe.group(1)): p
                         for t, p in _link_rules().items()
                         if "$(" not in t.replace("$(EXE)", "") and "%" not in t}
            dry = _make("-Bnk", "VK=1", *sorted(cls.rules))
        finally:
            if saved is None:
                stamp.unlink(missing_ok=True)
            else:
                stamp.write_bytes(saved[0])
                os.utime(stamp, ns=(saved[1].st_atime_ns, saved[1].st_mtime_ns))
        # A link line is any printed command that compiles a .c into an output
        # without -c -- also for rules that write a differently named file
        # (fuzz-rans) -- and the compiler is whatever make put first on it.
        # `make -n` prints recipe continuations as written, so fold them first.
        lines = {}
        for line in re.sub(r"\\\n[ \t]*", " ", dry.stdout).splitlines():
            words = line.split()
            if ("-o" in words and "-c" not in words
                    and words.index("-o") + 1 < len(words)
                    and any(w.endswith(".c") for w in words)):
                lines[words[words.index("-o") + 1]] = words
        compilers = {words[0] for words in lines.values()}
        if lines and not any(shutil.which(c) for c in compilers):
            raise unittest.SkipTest(f"{', '.join(sorted(compilers))} is required")
        cls.links = {t: w for t, w in lines.items() if shutil.which(w[0])}
        api = _backend_api()
        with ThreadPoolExecutor(max_workers=os.cpu_count() or 4) as pool:
            results = dict(zip(cls.links, pool.map(
                lambda words: _unresolved(words, api), cls.links.values())))
        cls.needs = {t: names for t, names in results.items() if names}
        cls.unchecked = sorted(t for t, names in results.items() if names is None)

    def test_the_scan_is_not_vacuous(self):
        """A broken dry run or preprocess would pass the check below silently.

        Cross-check against the engines: a NAME.c that calls a backend function
        and has a `NAME$(EXE):` rule compiles that call directly, so the scan
        must report the rule as calling the backend.
        """
        api = _backend_api()
        self.assertTrue(api, "backend_vulkan.h declares no coli_vk_*")
        self.assertTrue(self.links, "make -Bn VK=1 printed no link line")
        call = re.compile(r"\b(%s)\s*\(" % "|".join(sorted(api)))
        engines = sorted(
            t for t in self.links
            if "/" not in t and (C_DIR / (t.split(".")[0] + ".c")).is_file()
            and call.search((C_DIR / (t.split(".")[0] + ".c")).read_text(
                encoding="utf-8", errors="replace")))
        self.assertTrue(engines, "no engine source calls the Vulkan backend")
        for target in engines:
            self.assertNotIn(target, self.unchecked,
                             f"{target} could not be preprocessed here")
            self.assertIn(target, self.needs,
                          f"{target} calls the backend in its source, yet the "
                          f"scan found no call -- the dry run or preprocess "
                          f"is broken")
        self.assertTrue(any(t.startswith("tests/") for t in self.needs),
                        "the scan found no test that calls the backend")

    def test_every_link_that_calls_vulkan_links_vk_obj(self):
        missing = []
        for target, names in sorted(self.needs.items()):
            example = sorted(names)[0]
            if "backend_vulkan.o" not in self.links[target]:
                missing.append(f"{target}: not on the link line (calls {example})")
            if target not in self.rules:
                missing.append(f"{target}: its rule is not named after it; "
                               f"list $(VK_OBJ) as that rule's prerequisite")
            elif "$(VK_OBJ)" not in self.rules[target]:
                missing.append(f"{target}: $(VK_OBJ) not a prerequisite")
        self.assertFalse(
            missing,
            "under VK=1 these links call coli_vk_* without defining it, so they "
            "must link backend_vulkan.o:\n  " + "\n  ".join(missing))


if __name__ == "__main__":
    unittest.main()