Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
build: generate header prerequisites for the rest of the gcc rules
Kenneth-Javier Ortigozafbeef6aa6734
diff --git a/c/Makefile b/c/Makefileindex 3c240493..058c4643 100644--- a/c/Makefile+++ b/c/Makefile@@ -1024,7 +1024,7 @@ elseRANSLIB = tools/librans_c.soendifrans: $(RANSLIB)-$(RANSLIB): tools/rans_ctypes.c rans.h+$(RANSLIB): tools/rans_ctypes.c$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)cuda-test: tests/test_mxfp4_expert_cuda.cu backend_cuda.cu backend_cuda.h fp8_format.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu tests/test_cuda_fmt_trap_cuda.cu tests/test_alloc_footprint_cuda.cu tests/test_cuda_init_failure.cu@@ -1312,7 +1312,8 @@ tests/test_qwen36_tokenizer$(EXE): tests/test_qwen36_tokenizer.c qwen36.c $(QWEN$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_OBJ) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)# Reproducible local timing evidence; intentionally not a noisy CI perf gate.-tests/bench_qwen36_dense_batch$(EXE): tests/bench_qwen36_dense_batch.c gsgemv.h qgemv.h sse41_kernels.h qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_OBJ) $(CUDA_OBJ)+tests/bench_qwen36_dense_batch$(EXE): tests/bench_qwen36_dense_batch.c qwen36.c \+ $(QWEN36_TIER_OBJ) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_OBJ) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)inkling$(EXE): inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ)@@ -1414,8 +1415,8 @@ tests/test_xdna_failure$(EXE): tests/test_xdna_failure.c colibri.c tests/backend## M-list defaults to 1,32,64 (the M64 bucket); pass 65,130,256 to qualify# the M256 bucket through the same owner.-tests/xdna_physical_probe$(EXE): tests/xdna_physical_probe.c colibri.c backend_xdna.c backend_xdna.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h- $(CC) $(CFLAGS) -DCOLI_XDNA $< backend_xdna.c -o $@ $(LDFLAGS)+tests/xdna_physical_probe$(EXE): tests/xdna_physical_probe.c colibri.c tests/backend_xdna_lane.o+ $(CC) $(CFLAGS) -DCOLI_XDNA $< tests/backend_xdna_lane.o -o $@ $(LDFLAGS)tests/test_json$(EXE): tests/test_json.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)@@ -1513,7 +1514,7 @@ tests/test_st_f16_bf16_simd_sse41$(EXE): tests/test_st_f16_bf16_simd.c# bf16_to_f32_bulk/f16_to_f32_bulk vs the scalar per-element reference, NOT a# test gate. Build on demand: make tests/bench_st_f16_bf16_simd ARCH=native-tests/bench_st_f16_bf16_simd$(EXE): tests/bench_st_f16_bf16_simd.c st.h json.h compat.h+tests/bench_st_f16_bf16_simd$(EXE): tests/bench_st_f16_bf16_simd.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)# Synthetic peak-RSS proxy for qwen36.c's dense-int8-during-load change@@ -1628,7 +1629,8 @@ tests/test_inkling_shared_batch$(EXE): tests/test_inkling_shared_batch.c inkling# Reproducible local A/B for the shared-expert prefill path. This is timing# evidence, not a CI gate: build and run it explicitly on the target CPU.-tests/bench_inkling_shared_batch$(EXE): tests/bench_inkling_shared_batch.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)+tests/bench_inkling_shared_batch$(EXE): tests/bench_inkling_shared_batch.c inkling.c \+ $(INK_CUDA_OBJ) $(METAL_OBJ)$(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)tests/test_inkling_cache_index$(EXE): tests/test_inkling_cache_index.c inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ)@@ -1685,7 +1687,7 @@ tests/test_topp$(EXE): tests/test_topp.c colibri.c $(VK_OBJ)# bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test# gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp-tests/bench_topp$(EXE): tests/bench_topp.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h+tests/bench_topp$(EXE): tests/bench_topp.c colibri.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c $(VK_OBJ)@@ -1827,58 +1829,37 @@ SEGMENT_RUNTIME_LIB = $(SEGMENT_BUILD_DIR)/libcolibri_segment_edge.a$(SEGMENT_BUILD_DIR):mkdir -p $@-$(SEGMENT_BUILD_DIR)/glm.o: colibri.c sse41_kernels.h oracle.h segment_runtime.h edge_runtime.h \- segment_adapters.h edge_adapters.h segment_adapter_internal.h \- edge_adapter_internal.h edge_tok_internal.h st.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_BUILD_DIR)/glm.o: colibri.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -DCOLIBRI_NO_MAIN -c colibri.c -o $@-$(SEGMENT_BUILD_DIR)/glm53.o: glm53.c sse41_kernels.h segment_runtime.h edge_runtime.h \- segment_adapters.h edge_adapters.h segment_adapter_internal.h \- edge_adapter_internal.h st.h quant.h fp8_format.h tok.h hyper_connections.h \- delta_attention.h sparse_index.h vision_tower.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_BUILD_DIR)/glm53.o: glm53.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -DGLM53_NO_MAIN -c glm53.c -o $@-$(SEGMENT_BUILD_DIR)/inkling.o: inkling.c segment_runtime.h edge_runtime.h \- segment_adapters.h edge_adapters.h segment_adapter_internal.h \- edge_adapter_internal.h edge_tok_internal.h st.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_BUILD_DIR)/inkling.o: inkling.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -DINKLING_NO_MAIN -c inkling.c -o $@-$(SEGMENT_BUILD_DIR)/kimi.o: kimi_k3.c sse41_kernels.h segment_runtime.h edge_runtime.h \- segment_adapters.h edge_adapters.h segment_adapter_internal.h \- edge_adapter_internal.h edge_tok_internal.h st.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_BUILD_DIR)/kimi.o: kimi_k3.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -DKIMI_K3_NO_MAIN -c kimi_k3.c -o $@-$(SEGMENT_BUILD_DIR)/olmoe.o: olmoe.c sse41_kernels.h segment_runtime.h edge_runtime.h \- segment_adapters.h edge_adapters.h segment_adapter_internal.h \- edge_adapter_internal.h edge_tok_internal.h st.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_BUILD_DIR)/olmoe.o: olmoe.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -DOLMOE_NO_MAIN -c olmoe.c -o $@-$(SEGMENT_BUILD_DIR)/qwen36.o: qwen36.c gsgemv.h qgemv.h sse41_kernels.h segment_runtime.h edge_runtime.h \- segment_adapters.h edge_adapters.h segment_adapter_internal.h \- edge_adapter_internal.h st.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_BUILD_DIR)/qwen36.o: qwen36.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -DQWEN36_NO_MAIN -c qwen36.c -o $@-$(SEGMENT_BUILD_DIR)/qwen38.o: qwen38.c sse41_kernels.h qwen38_core.h kv_prefix.h segment_runtime.h \- segment_adapters.h segment_adapter_internal.h edge_runtime.h \- edge_adapters.h edge_adapter_internal.h st.h json.h compat.h quant.h fp8_format.h \- qwen38_nfc.h qwen38_nfc_tables.h route_trace.h tok_unicode.h \- tok_unicode_o200k.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_BUILD_DIR)/qwen38.o: qwen38.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -DQWEN38_NO_MAIN -c qwen38.c -o $@-$(SEGMENT_V4_OBJS): $(SEGMENT_BUILD_DIR)/%.o: deepseek_v4.c sse41_kernels.h deepseek_v4.h \- deepseek_v4_internal.h segment_runtime.h segment_adapters.h \- edge_runtime.h edge_adapters.h segment_adapter_internal.h \- edge_adapter_internal.h edge_tok_internal.h st.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_V4_OBJS): $(SEGMENT_BUILD_DIR)/%.o: deepseek_v4.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -D$* -c deepseek_v4.c -o $@-$(SEGMENT_V4_REGISTRY_OBJ): expert_store_registry.c expert_store_registry.h \- expert_store.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_V4_REGISTRY_OBJ): expert_store_registry.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -c expert_store_registry.c -o $@-$(SEGMENT_BUILD_DIR)/segment_runtime.o: segment_runtime.c segment_runtime.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_BUILD_DIR)/segment_runtime.o: segment_runtime.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -c segment_runtime.c -o $@-$(SEGMENT_BUILD_DIR)/edge_runtime.o: edge_runtime.c edge_runtime.h | $(SEGMENT_BUILD_DIR)+$(SEGMENT_BUILD_DIR)/edge_runtime.o: edge_runtime.c | $(SEGMENT_BUILD_DIR)$(CC) $(SEGMENT_CPU_CFLAGS) -c edge_runtime.c -o $@$(SEGMENT_RUNTIME_LIB): $(SEGMENT_RUNTIME_OBJS) $(SEGMENT_ALL_OBJS)@@ -2017,22 +1998,22 @@ V4_OWNERSHIP_OBJS = \$(V4_OWN_DIR):mkdir -p $(V4_OWN_DIR)-$(V4_OWN_DIR)/COLI_V4_UNIT_RUNTIME.o: deepseek_v4.c sse41_kernels.h deepseek_v4.h deepseek_v4_internal.h | $(V4_OWN_DIR)+$(V4_OWN_DIR)/COLI_V4_UNIT_RUNTIME.o: deepseek_v4.c | $(V4_OWN_DIR)$(CC) $(V4_OWN_CFLAGS) -DCOLI_V4_UNIT_RUNTIME -c deepseek_v4.c -o $@-$(V4_OWN_DIR)/COLI_V4_UNIT_CONFIG.o: deepseek_v4.c sse41_kernels.h deepseek_v4.h deepseek_v4_internal.h | $(V4_OWN_DIR)+$(V4_OWN_DIR)/COLI_V4_UNIT_CONFIG.o: deepseek_v4.c | $(V4_OWN_DIR)$(CC) $(V4_OWN_CFLAGS) -DCOLI_V4_UNIT_CONFIG -c deepseek_v4.c -o $@-$(V4_OWN_DIR)/COLI_V4_UNIT_ST.o: deepseek_v4.c sse41_kernels.h deepseek_v4.h deepseek_v4_internal.h st.h | $(V4_OWN_DIR)+$(V4_OWN_DIR)/COLI_V4_UNIT_ST.o: deepseek_v4.c | $(V4_OWN_DIR)$(CC) $(V4_OWN_CFLAGS) -DCOLI_V4_UNIT_ST -c deepseek_v4.c -o $@-$(V4_OWN_DIR)/COLI_V4_UNIT_NATIVE_QUANT.o: deepseek_v4.c sse41_kernels.h deepseek_v4.h deepseek_v4_internal.h quant.h fp8_format.h idot.h | $(V4_OWN_DIR)+$(V4_OWN_DIR)/COLI_V4_UNIT_NATIVE_QUANT.o: deepseek_v4.c | $(V4_OWN_DIR)$(CC) $(V4_OWN_CFLAGS) -DCOLI_V4_UNIT_NATIVE_QUANT -c deepseek_v4.c -o $@# The engine (RUNTIME unit) opens its expert store through the pluggable# backend registry, so any test linking RUNTIME also links the registry object# (which defines coli_expert_store_backend_open_selected + registers "auto").-$(V4_OWN_DIR)/expert_store_registry.o: expert_store_registry.c expert_store_registry.h expert_store.h | $(V4_OWN_DIR)+$(V4_OWN_DIR)/expert_store_registry.o: expert_store_registry.c | $(V4_OWN_DIR)$(CC) $(V4_OWN_CFLAGS) -c expert_store_registry.c -o $@tests/test_v4_ownership$(EXE): tests/test_v4_ownership.c $(V4_OWNERSHIP_OBJS)@@ -2213,12 +2194,12 @@ tests/test_ssd_probe$(EXE): tests/test_ssd_probe.c colibri.c $(VK_OBJ)# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356),# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select-tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h+tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)# bench_router_select is a microbenchmark (duplicate-prefix scan vs marked-score scan),# not a test gate. Build on demand: make tests/bench_router_select.-tests/bench_router_select$(EXE): tests/bench_router_select.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h+tests/bench_router_select$(EXE): tests/bench_router_select.c colibri.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)# bench_indexer_allocations: microbenchmark (DeepSeek V4 indexer malloc vs persistent arena scratch), NOT a test gate.@@ -2228,23 +2209,23 @@ tests/bench_indexer_allocations$(EXE): tests/bench_indexer_allocations.c# bench_idot: microbenchmark (single-acc vs independent-acc AVX-VNNI idot), NOT a test gate.# Build on demand on an AVX-VNNI CPU: make tests/bench_idot ARCH=native-tests/bench_idot$(EXE): tests/bench_idot.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h+tests/bench_idot$(EXE): tests/bench_idot.c colibri.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)# bench_i4p_gidot: microbenchmark (per-row vs multi-row/AMX K1b grouped planar IDOT), NOT a test gate.# Build on demand: make tests/bench_i4p_gidot ARCH=native-tests/bench_i4p_gidot$(EXE): tests/bench_i4p_gidot.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h+tests/bench_i4p_gidot$(EXE): tests/bench_i4p_gidot.c colibri.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)# bench_gemv_stream: microbenchmark (decode-regime GEMV bandwidth vs the read ceiling;# frozen-baseline + deinterleaved-x candidate A/B), NOT a test gate.# Build on demand: make tests/bench_gemv_stream ARCH=native-tests/bench_gemv_stream$(EXE): tests/bench_gemv_stream.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h+tests/bench_gemv_stream$(EXE): tests/bench_gemv_stream.c colibri.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)# bench_mla_simd: microbenchmark (scalar vs AVX2/NEON MLA-absorb reductions, #442),# NOT a test gate. Build on demand: make tests/bench_mla_simd-tests/bench_mla_simd$(EXE): tests/bench_mla_simd.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h+tests/bench_mla_simd$(EXE): tests/bench_mla_simd.c colibri.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)tests/test_uring$(EXE): tests/test_uring.c colibri.c $(VK_OBJ)@@ -2472,8 +2453,9 @@ tests/test_kimi_cuda_expert$(EXE): tests/test_kimi_cuda_expert.c kimi_k3.c $(VK_## Every rule in AUTODEP_BINS compiles ONE translation unit with $(CFLAGS), so# -MMD (added to CFLAGS near the top) leaves a complete <target>.d and the rule-# does not list headers by hand. The engines and tests are derived the same way-# TEST_RULES is, so a new rule joins automatically.+# does not list headers by hand. The engines and everything under tests/ are+# derived the same way TEST_RULES is, so a new rule joins automatically; the+# objects and the ctypes library are named here.## A command that compiles several .c files writes a .d for the LAST one only,# silently dropping the headers of the others. So a helper source a test links@@ -2483,9 +2465,11 @@ tests/test_kimi_cuda_expert$(EXE): tests/test_kimi_cuda_expert.c kimi_k3.c $(VK_## AUTODEP_ELSEWHERE keeps its header lists: test_deepseek_v4 and# test_v4_serve_framing are built by Makefile.deepseek-v4, and the four-# segment/edge adapter tests still compile three units in one command. The-# nvcc, MSVC-hosted CUDA and Metal objects keep theirs too and are not listed-# here at all.+# segment/edge adapter tests still compile three units in one command.+# AUTODEP_OWN_FLAGS does not compile through $(CFLAGS) at all (a GPU compiler,+# or a benchmark pinned to its own flags), so it writes no .d. The nvcc,+# MSVC-hosted CUDA, Metal and fuzz targets, and the gcc-built CUDA loader+# harnesses (dsv4-cuda-*), keep their lists and are not listed here at all.## Most rules do not depend on .build-config, so a change to CFLAGS alone would# not rebuild a binary made before this change, and it would have no .d. Each@@ -2495,13 +2479,18 @@ tests/test_kimi_cuda_expert$(EXE): tests/test_kimi_cuda_expert.c kimi_k3.c $(VK_AUTODEP_ELSEWHERE = test_segment_adapters_registration test_segment_adapters_real \test_edge_adapters_registration test_edge_adapters_real test_deepseek_v4 \test_v4_serve_framing+AUTODEP_OWN_FLAGS = bench_cuda_resident_batch bench_load_tq_peak_rss \+ bench_indexer_allocationsENGINE_RULES := $(foreach n,$(RULE_NAMES),$(if $(findstring /,$(n)),,$(if $(wildcard $(n).c),$(n))))+AUTODEP_TESTS := $(patsubst tests/%,%,$(filter tests/%,$(RULE_NAMES)))AUTODEP_OBJS = backend_vulkan.o backend_loader.o backend_xdna.o qwen36_tier.o \tests/backend_xdna_lane.o tests/edge_runtime.o tests/edge_runtime_nocuda.o \tests/segment_runtime.o tests/segment_runtime_nocuda.o \- tests/segment_conformance_fixtures.o tests/deepseek_v4_native_quant.o+ tests/segment_conformance_fixtures.o tests/deepseek_v4_native_quant.o \+ $(SEGMENT_ALL_OBJS) $(SEGMENT_RUNTIME_OBJS) $(V4_OWNERSHIP_OBJS) $(RANSLIB)AUTODEP_BINS = $(addsuffix $(EXE),$(ENGINE_RULES)) \- $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(AUTODEP_ELSEWHERE),$(TEST_RULES)))) \+ $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(AUTODEP_ELSEWHERE) \+ $(AUTODEP_OWN_FLAGS),$(AUTODEP_TESTS)))) \$(AUTODEP_OBJS)AUTODEP_DEPS = $(addsuffix .d,$(basename $(AUTODEP_BINS)))$(foreach b,$(AUTODEP_BINS),$(eval $(b): $(basename $(b)).d))diff --git a/c/tests/test_makefile_deps.py b/c/tests/test_makefile_deps.pyindex 741f3a45..835b4301 100644--- a/c/tests/test_makefile_deps.py+++ b/c/tests/test_makefile_deps.py@@ -110,18 +110,58 @@ def _make(*args):capture_output=True, timeout=600)-def _make_var(name):+def _make_vars(*names):+ """Makefile variables as make expands them, {name: words}, in one call."""# `make --eval` needs GNU Make 3.82; macOS ships 3.81, which does read an# extra makefile from stdin.with _build_config_kept():proc = subprocess.run([MAKE, "-s", "--no-print-directory", "-f", "Makefile",- "-f", "-", f"print-{name}"], cwd=C_DIR, text=True,- input="print-%: ; @echo $($*)\n", capture_output=True,- timeout=300)+ "-f", "-", *(f"print-{n}" for n in names)], cwd=C_DIR,+ text=True, input="print-%: ; @echo $*=$($*)\n",+ capture_output=True, timeout=300)if proc.returncode != 0:- raise AssertionError(f"could not read {name} from the Makefile:\n"+ raise AssertionError(f"could not read {' '.join(names)} from the Makefile:\n"f"{proc.stderr}")- return proc.stdout.split()+ values = {}+ for line in proc.stdout.splitlines():+ name, sep, value = line.partition("=")+ if sep and name in names:+ values[name] = value.split()+ return values+++def _make_var(name):+ return _make_vars(name).get(name, [])+++RULE_LINE_RE = re.compile(r"(?m)^([^\s#:=][^:=\n]*?):(?![=:])(.*)$")+VAR_RE = re.compile(r"\$\(([A-Za-z0-9_]+)\)")+++def _rules_by_target():+ """Each rule's prerequisites as written, keyed by the file make builds.++ Rules are spelled `colibri$(EXE):`, `$(SEGMENT_BUILD_DIR)/glm.o:`,+ `$(RANSLIB):` or `$(SEGMENT_V4_OBJS): $(SEGMENT_BUILD_DIR)/%.o: ...`, so+ the target side is expanded by asking make for those variables, and a+ static pattern rule gives its prerequisites to each object it names. The+ prerequisites keep their spelling: a hand-listed header is a plain word.+ """+ # `$(foreach b,...,$(eval ...))` lines generate rules; they are not rules.+ lines = [(t, r) for t, r in RULE_LINE_RE.findall(_joined_makefile())+ if not re.search(r"\$\([a-z-]+ ", t)]+ names = sorted({v for target, _ in lines for v in VAR_RE.findall(target)})+ values = _make_vars(*names)+ rules = {}+ for target, rest in lines:+ prereqs = rest.split("#", 1)[0]+ pattern, sep, after = prereqs.partition(":")+ if sep and "%" in pattern:+ prereqs = after+ expanded = VAR_RE.sub(lambda m: " ".join(values.get(m.group(1), [])), target)+ for name in expanded.split():+ rules.setdefault(name, []).extend(prereqs.split())+ return rulesdef _commands(targets, *variables):@@ -208,21 +248,11 @@ def _unconditional_includes(source):return found-def _rule_form(target, exe):- """How a built target is spelled in the Makefile: colibri.exe -> colibri$(EXE)."""- if target.endswith(".o"):- return target- if exe and target.endswith(exe):- return target[:-len(exe)] + "$(EXE)"- return target + "$(EXE)"---def _source_of(form, rules_text):+def _source_of(target, rules):"""The .c a rule compiles: the first source among its prerequisites."""- for m in re.finditer(r"(?m)^" + re.escape(form) + r":[ \t]*(.*)$", rules_text):- for word in m.group(1).split("#", 1)[0].split():- if SOURCE_RE.search(word):- return C_DIR / word+ for word in rules.get(target, []):+ if SOURCE_RE.search(word):+ return C_DIR / wordreturn None@@ -266,16 +296,19 @@ class GeneratedDepsWiringTest(unittest.TestCase):def test_no_generated_rule_lists_headers_by_hand(self):"""A hand-listed header on these rules is redundant and brings back the- merge conflicts #1741 removed. Bite: add `st.h` to `colibri$(EXE):`."""- text = _joined_makefile()+ merge conflicts #1741 removed. Bite: add `st.h` to `colibri$(EXE):`,+ or to `$(SEGMENT_BUILD_DIR)/glm.o:`."""+ rules = _rules_by_target()problems = []for target in self.bins:- form = _rule_form(target, self.exe)- for m in re.finditer(r"(?m)^" + re.escape(form) + r":[ \t]*(.*)$", text):- listed = [w for w in m.group(1).split("#", 1)[0].split()- if re.search(r"\.(h|inc)$", w) and not w.startswith("$")]- if listed:- problems.append(f"{form}: {' '.join(listed)}")+ if target not in rules:+ problems.append(f"{target}: no rule found for it, so its prerequisites "+ f"cannot be checked")+ continue+ listed = [w for w in rules[target]+ if re.search(r"\.(h|inc)$", w) and not w.startswith("$")]+ if listed:+ problems.append(f"{target}: {' '.join(listed)}")self.assertEqual(problems, [],"these rules get their headers from -MMD and should ""not list them by hand:\n " + "\n ".join(problems))@@ -345,7 +378,7 @@ class GeneratedDepsCoverageTest(unittest.TestCase):"this after `make check`")def test_every_built_target_has_a_dep_file_covering_its_includes(self):- text = _joined_makefile()+ rules = _rules_by_target()problems = []for target in self.built:dep = _dep_file(target)@@ -353,7 +386,7 @@ class GeneratedDepsCoverageTest(unittest.TestCase):problems.append(f"{target}: built, but has no {dep.name}; its "f"headers are untracked until it is rebuilt")continue- source = _source_of(_rule_form(target, self.exe), text)+ source = _source_of(target, rules)if source is None:problems.append(f"{target}: no source found in its rule")continuediff --git a/c/tools/clean.py b/c/tools/clean.pyindex 6805529c..33c005a4 100644--- a/c/tools/clean.py+++ b/c/tools/clean.py@@ -67,17 +67,19 @@ FILES = [# (#1741). A stale one only adds prerequisites, but clean should leave nothing# the build made, and removing it forces the rebuild that writes a fresh one.# They land wherever an output does: c/ and tests/ for the engines and tests,-# tools/ for the ctypes libraries, build/segment/ for the V4 unit objects-# (build/ownership/ goes as a whole directory below).+# tools/ for the ctypes library; build/segment/ and build/ownership/ go as+# whole directories below.ARTIFACT_GLOBS = ["tests/test_*", "tests/bench_*", "tests/fuzz_*","tests/*_probe*", "COLI_V4_UNIT_*.o", "*.d", "tests/*.d",- "tools/*.d", "build/segment/*.d",+ "tools/*.d",# helper objects the tests link (#1741), one unit per command"tests/*.o"]KEEP_EXT = (".c", ".h", ".cc", ".cpp", ".cu", ".mm", ".py", ".txt", ".json",".md", ".bin", ".sh", ".toml", ".yml", ".yaml")-# Directories to remove.-DIRS = ["tests/__pycache__", "build/ownership"]+# Directories to remove. build/segment/ holds only build output (the+# segment-library objects, their .d and the archive); an object left there+# without its .d would be a built target whose headers are untracked (#1741).+DIRS = ["tests/__pycache__", "build/ownership", "build/segment"]def clean():"""Remove everything above, relative to the current directory."""