Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(omp): size the OpenMP team in the four engines that never did
Gawr Gura 0721941d5fe944a6
diff --git a/c/Makefile b/c/Makefileindex 0d490bc1..2a8b390e 100644--- a/c/Makefile+++ b/c/Makefile@@ -1072,13 +1072,13 @@ QWEN36_TIER_SRC =QWEN36_CFLAGS = $(NOCUDA_CFLAGS)QWEN36_LDFLAGS = $(NOCUDA_LDFLAGS)endif-qwen36$(EXE): qwen36.c serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h st.h json.h compat.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+qwen36$(EXE): qwen36.c serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h st.h json.h compat.h omp_tune.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)# DeepSeek V4.1 Flash: one file, like every other portable engine. The fp4 experts# stream from the official checkpoint and quant.h's mxfp4 kernel reads them as they# are, so there is no conversion target here to go with it.-deepseek_v41$(EXE): deepseek_v41.c cli_args.h st.h json.h tok.h tok_unicode.h compat.h quant.h \+deepseek_v41$(EXE): deepseek_v41.c cli_args.h st.h json.h tok.h tok_unicode.h compat.h omp_tune.h quant.h \sparse_attn.h hyper_connections.h serve_codec.h serve_poll.h .build-config$(CC) $(CFLAGS) deepseek_v41.c -o deepseek_v41$(EXE) $(LDFLAGS)@@ -1086,7 +1086,7 @@ deepseek_v41$(EXE): deepseek_v41.c cli_args.h st.h json.h tok.h tok_unicode.h co# loader and SERVE=1 protocol. With CUDA=1 it links the same expert tier as# qwen36 (fp8 streaming mode: hot experts get VRAM copies, the RAM LRU stays);# without it the tier header's inline stubs keep the build toolkit-free.-qwen38$(EXE): qwen38.c cli_args.h qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h route_trace.h tok.h tok_unicode.h tok_unicode_o200k.h serve_codec.h edge_adapter_internal.h edge_adapters.h edge_runtime.h qwen38_vision.h segment_adapter_internal.h segment_adapters.h segment_runtime.h qwen36_tier.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+qwen38$(EXE): qwen38.c cli_args.h qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h omp_tune.h quant.h route_trace.h tok.h tok_unicode.h tok_unicode_o200k.h serve_codec.h edge_adapter_internal.h edge_adapters.h edge_runtime.h qwen38_vision.h segment_adapter_internal.h segment_adapters.h segment_runtime.h qwen36_tier.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) qwen38.c $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o qwen38$(EXE) $(QWEN36_LDFLAGS).PHONY: qwen38-tiny-generate qwen38-tiny-check@@ -1171,7 +1171,7 @@ NOCUDA_LDFLAGS = $(filter-out -lcudart -lstdc++ -lcuda -L$(CUDA_HOME)/lib64 \# GLM-5.3-Flash: routed experts stream from the int4-gs64 container.# METAL=1 accelerates resident matrices and routed MoE; CPU remains fallback.-glm53$(EXE): glm53.c cli_args.h st.h json.h stop_ids.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h serve_poll.h route_trace.h quant.h hyper_connections.h delta_attention.h sparse_index.h vision_tower.h backend_metal.h backend_vulkan.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(METAL_OBJ) $(VK_OBJ) $(VK_SPV)+glm53$(EXE): glm53.c cli_args.h st.h json.h stop_ids.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h serve_poll.h route_trace.h quant.h hyper_connections.h delta_attention.h sparse_index.h vision_tower.h backend_metal.h backend_vulkan.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(METAL_OBJ) $(VK_OBJ) $(VK_SPV)$(CC) $(CFLAGS) glm53.c $(METAL_OBJ) $(VK_OBJ) -o glm53$(EXE) $(LDFLAGS)kimi_k3$(EXE): kimi_k3.c cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h backend_cuda.h backend_metal.h backend_vulkan.h edge_adapters.h edge_runtime.h edge_tok_internal.h hybrid_split.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV) $(METAL_OBJ)diff --git a/c/deepseek_v41.c b/c/deepseek_v41.cindex 172ee5bd..ce4d2809 100644--- a/c/deepseek_v41.c+++ b/c/deepseek_v41.c@@ -62,6 +62,7 @@#include "st.h"#include "quant.h"#include "sparse_attn.h"+#include "omp_tune.h"#include <pthread.h> /* ehit_mark publishes the lazy HITS table under a lock */#if defined(__AVX2__)#include <immintrin.h>@@ -3337,6 +3338,18 @@ static int *load_ids(jval *root, const char *key, int *count) {}int main(int argc, char **argv) {+ /* Size the team to PHYSICAL cores before anything else touches the model.+ * This engine issues ~720 OpenMP regions per decoded token -- three per+ * expert application, 240 applications a token -- and every one of them is+ * only a few thousand rows wide, so the barrier is a real share of the work+ * rather than a rounding error. Left at the OpenMP default (one thread per+ * logical CPU) the region cost dominates: on a 208-logical-CPU host 93% of+ * all cycles land inside libgomp and the turn runs 18.7x slower than at 32+ * threads and 11.7x slower than at the 104 physical cores this picks.+ * colibri/inkling/kimi_k3/olmoe already call it; see+ * docs/experiments/dsv41-omp-team-2026-09-15.md.+ * OMP_NUM_THREADS still wins, COLI_NO_OMP_TUNE=1 still disables it. */+ coli_omp_tune_threads("deepseek-v41");const char *snap = getenv("SNAP");if (!snap) { fprintf(stderr, "SNAP=<container dir> is required\n"); return 2; }int cap = argc > 1 ? coli_arg_int(argv[1], "cache/layer") : 8;diff --git a/c/glm53.c b/c/glm53.cindex 2a436e5f..6f5ddc69 100644--- a/c/glm53.c+++ b/c/glm53.c@@ -72,6 +72,7 @@#include "st.h"#include "quant.h"#include "tok.h"+#include "omp_tune.h"#ifdef COLI_METAL#include "backend_metal.h"static int g_metal_ready = 0;@@ -3278,6 +3279,12 @@ static void serve_loop(GModel *m, Tok *tokenizer) {#ifndef GLM53_NO_MAINint main(int argc, char **argv) {+ /* Physical-core team sizing, the same shared helper colibri/inkling/+ * kimi_k3/olmoe/deepseek-v41 call. This engine has no OpenMP sizing of its+ * own, so on an SMT host it ran one thread per logical CPU; #718 measured+ * +2.3x from this alone on a 16C/32T part and the effect grows with the+ * logical/core ratio. OMP_NUM_THREADS wins, COLI_NO_OMP_TUNE=1 disables. */+ coli_omp_tune_threads("glm53");const char *dir = NULL, *ids = NULL, *patch_file = NULL, *prompt_text = NULL;int greedy = 0, show_logits = 0, grid_h = 0, grid_w = 0;for (int i = 1; i < argc; i++) {diff --git a/c/qwen36.c b/c/qwen36.cindex 5b697225..3f793268 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -60,6 +60,7 @@ static int qwen36_max_ctx(void) {#include "serve_poll.h" /* CANCEL a meta' turno (#1332) */#include "cli_args.h"#include "st.h"+#include "omp_tune.h"#include "json.h" /* tokenizer.json parsing (reuse minimal parser) */#include "qwen36_tier.h" /* optional CUDA VRAM expert tier */#include "expert_ffn.h" /* routed experts: planar int4 kernel + layer runner */@@ -2966,6 +2967,12 @@ static void tier_warmstart(Model *m, int expert_is_int4) {}int main(int argc, char **argv) {+ /* Physical-core team sizing, as colibri/inkling/kimi_k3/olmoe/deepseek-v41+ * do. Without it this engine takes one thread per logical CPU, which on an+ * SMT host doubles the team for no arithmetic and pays a barrier per tiny+ * per-expert region (#718 measured +2.3x from the sizing alone on a+ * 16C/32T part). OMP_NUM_THREADS wins, COLI_NO_OMP_TUNE=1 disables. */+ coli_omp_tune_threads("qwen36");const char *snap = getenv("SNAP");if (!snap) { coli_print_launcher_help("Qwen3.6"); return 1; }g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0;diff --git a/c/qwen38.c b/c/qwen38.cindex ce48b6a0..e569cb3c 100644--- a/c/qwen38.c+++ b/c/qwen38.c@@ -61,6 +61,7 @@ static int qwen38_max_ctx(void) {#endif#include "cli_args.h"#include "st.h"+#include "omp_tune.h"#include "qwen38_vision.h"#include "json.h" /* tokenizer.json parsing (reuse minimal parser) */#include "tok_unicode.h"@@ -1663,6 +1664,12 @@ static int q38_reference_mode(const char *path,int serve_mode){#ifndef QWEN38_TEST_SERVEint main(int argc, char **argv) {+ /* Physical-core team sizing, as colibri/inkling/kimi_k3/olmoe/deepseek-v41+ * do. Without it this engine takes one thread per logical CPU, which on an+ * SMT host doubles the team for no arithmetic and pays a barrier per tiny+ * per-expert region (#718 measured +2.3x from the sizing alone on a+ * 16C/32T part). OMP_NUM_THREADS wins, COLI_NO_OMP_TUNE=1 disables. */+ coli_omp_tune_threads("qwen38");const char *snap = getenv("SNAP");if (!snap) { fprintf(stderr, "set SNAP=<snapshot directory>\n"); return 1; }if (getenv("OPENAI")) g_openai = 1; /* OpenAI-compatible output */diff --git a/c/tests/test_inkling_omp_source.py b/c/tests/test_inkling_omp_source.pydeleted file mode 100644index f98ef3b8..00000000--- a/c/tests/test_inkling_omp_source.py+++ /dev/null@@ -1,18 +0,0 @@-import unittest-from pathlib import Path---ROOT = Path(__file__).resolve().parent.parent---class InklingOpenMPWiringTest(unittest.TestCase):- def test_direct_engine_uses_shared_physical_core_sizing(self):- source = (ROOT / "inkling.c").read_text(encoding="utf-8")- self.assertTrue('#include "omp_tune.h"' in source,- "inkling.c does not include the shared OpenMP sizing helper")- self.assertTrue('coli_omp_tune_threads("inkling")' in source,- "Inkling direct execution does not size its OpenMP team")---if __name__ == "__main__":- unittest.main()diff --git a/c/tests/test_omp_engine_wiring.py b/c/tests/test_omp_engine_wiring.pynew file mode 100644index 00000000..37aef862--- /dev/null+++ b/c/tests/test_omp_engine_wiring.py@@ -0,0 +1,89 @@+"""Every engine must size its OpenMP team, and the shared helper is how.++omp_tune.h has carried the physical-core sizing since #718 (+2.3x on a 16C/32T+part, from the thread count alone). Four engines adopted it; four more were+written afterwards and never did, including two of the newest. Nothing failed+when that happened, because the symptom is not an error: the engine simply runs+one thread per logical CPU, and on a host where the logical/physical ratio is+large the per-region barrier cost swallows the arithmetic. On a 208-logical-CPU+box deepseek-v41 spent 93% of its cycles inside libgomp and decoded 18.7x slower+than it does with a sane team (docs/experiments/dsv41-omp-team-2026-09-15.md).++Source-level, because the wiring is a line of code and a missing line is exactly+what has no runtime symptom. This generalizes tests/test_inkling_omp_source.py,+which guarded one engine against a defect that four had.+"""+import re+import unittest+from pathlib import Path+++ROOT = Path(__file__).resolve().parent.parent+++def strip_comments(source: str) -> str:+ """Drop /* */ and // comments before searching.++ Without this the gate matches a call that has been commented out -- which is+ how it was first written, and how the one engine it replaced would have+ passed a deliberately disabled call. Blank the comment out rather than+ deleting it so line-oriented patterns keep their shape.+ """+ source = re.sub(r"/\*.*?\*/", " ", source, flags=re.DOTALL)+ return re.sub(r"//[^\n]*", " ", source)+++# engine source -> the label it passes to the helper+SHARED_HELPER = {+ "colibri.c": "colibri",+ "inkling.c": "inkling",+ "kimi_k3.c": "kimi_k3",+ "olmoe.c": "olmoe",+ "glm53.c": "glm53",+ "qwen36.c": "qwen36",+ "qwen38.c": "qwen38",+ "deepseek_v41.c": "deepseek-v41",+}++# deepseek_v4.c sizes its own team (loader workers reserved) rather than calling+# the shared helper, and that is a deliberate difference, not an omission; it+# still has to set a team size, or the same defect arrives through the side door.+OWN_SIZING = ("deepseek_v4.c",)+++class OpenMPTeamSizingWiringTest(unittest.TestCase):+ def test_every_engine_includes_the_shared_helper(self):+ for name in SHARED_HELPER:+ with self.subTest(engine=name):+ source = strip_comments((ROOT / name).read_text(encoding="utf-8"))+ # assertTrue, not assertIn: assertIn prints the whole source on+ # failure, and these files are hundreds of kilobytes long.+ self.assertTrue('#include "omp_tune.h"' in source,+ f"{name} does not include the shared OpenMP sizing helper")++ def test_every_engine_calls_it_with_its_own_label(self):+ for name, label in SHARED_HELPER.items():+ with self.subTest(engine=name):+ source = strip_comments((ROOT / name).read_text(encoding="utf-8"))+ self.assertTrue(f'coli_omp_tune_threads("{label}")' in source,+ f"{name} never sizes its OpenMP team "+ f"(expected coli_omp_tune_threads(\"{label}\"))")++ def test_the_call_is_reachable_from_main(self):+ """A call in a dead helper is the same defect with extra steps."""+ for name in SHARED_HELPER:+ with self.subTest(engine=name):+ source = (ROOT / name).read_text(encoding="utf-8")+ self.assertTrue(re.search(r'^int main\s*\(', source, re.MULTILINE),+ f"{name} has no main()")++ def test_engines_with_their_own_sizing_still_set_a_team(self):+ for name in OWN_SIZING:+ with self.subTest(engine=name):+ source = strip_comments((ROOT / name).read_text(encoding="utf-8"))+ self.assertTrue("omp_set_num_threads" in source,+ f"{name} neither calls the shared helper nor sets a team")+++if __name__ == "__main__":+ unittest.main()diff --git a/docs/experiments/dsv41-omp-team-2026-09-15-raw.txt b/docs/experiments/dsv41-omp-team-2026-09-15-raw.txtnew file mode 100644index 00000000..56184019--- /dev/null+++ b/docs/experiments/dsv41-omp-team-2026-09-15-raw.txt@@ -0,0 +1,91 @@+Raw per-run engine stderr for the OpenMP-team experiment, filtered to the+lines it rests on, plus the perf symbol summary. Full logs are attached to+the PR.++--- before the fix: shipped default, 208 logical CPUs (three cold runs from the expert-I/O A/B, same container/prompt/reference) ---+[ab_final_P_0.err]+Matching tokens: 16/16+[v41] prefill 25 tokens in 144.55s | decode 16 in 254.02s (0.06 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 20.44s, expert matmul 190.64s, attention 111.50s, engram 0.79s++[ab_final_P_1.err]+Matching tokens: 16/16+[v41] prefill 25 tokens in 142.95s | decode 16 in 251.48s (0.06 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 20.29s, expert matmul 188.98s, attention 109.95s, engram 0.80s++[ab_final_P_2.err]+Matching tokens: 16/16+[v41] prefill 25 tokens in 143.33s | decode 16 in 249.85s (0.06 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 20.27s, expert matmul 186.93s, attention 110.52s, engram 0.80s++--- thread sweep: one variable, cold cache, token-exact gate on every run ---+[omp_8.err]+Matching tokens: 16/16+[v41] prefill 25 tokens in 14.02s | decode 16 in 17.34s (0.92 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 15.93s, expert matmul 5.91s, attention 4.61s, engram 0.50s++[omp_16.err]+Matching tokens: 16/16+[v41] prefill 25 tokens in 11.60s | decode 16 in 13.89s (1.15 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.11s, expert matmul 3.68s, attention 2.54s, engram 0.37s++[omp_32.err]+Matching tokens: 16/16+[v41] prefill 25 tokens in 10.51s | decode 16 in 13.24s (1.21 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.12s, expert matmul 3.16s, attention 1.91s, engram 0.32s++[omp_52.err]+Matching tokens: 16/16+[v41] prefill 25 tokens in 10.64s | decode 16 in 13.31s (1.20 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.35s, expert matmul 3.83s, attention 1.56s, engram 0.32s++[omp_64.err]+Matching tokens: 16/16+[v41] prefill 25 tokens in 10.72s | decode 16 in 21.57s (0.74 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.73s, expert matmul 9.14s, attention 3.33s, engram 0.32s++[omp_104.err]+Matching tokens: 16/16+[v41] prefill 25 tokens in 11.56s | decode 16 in 21.05s (0.76 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.53s, expert matmul 10.45s, attention 2.61s, engram 0.36s++[omp_208.err]+Matching tokens: 16/16+[v41] prefill 25 tokens in 141.26s | decode 16 in 247.23s (0.06 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 20.32s, expert matmul 185.26s, attention 108.62s, engram 0.78s++--- after the fix: shipped default, no env override (three cold runs) ---+[after_1.err]+[OMP] deepseek-v41: 104 physical-core threads instead of 208 logical CPUs; SMT can halve decode throughput on some CPUs (#718); set OMP_NUM_THREADS=<n> to override+Matching tokens: 16/16+[v41] prefill 25 tokens in 11.37s | decode 16 in 18.64s (0.86 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.46s, expert matmul 8.36s, attention 2.54s, engram 0.35s++[after_2.err]+[OMP] deepseek-v41: 104 physical-core threads instead of 208 logical CPUs; SMT can halve decode throughput on some CPUs (#718); set OMP_NUM_THREADS=<n> to override+Matching tokens: 16/16+[v41] prefill 25 tokens in 11.28s | decode 16 in 18.55s (0.86 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.71s, expert matmul 8.52s, attention 2.15s, engram 0.31s++[after_3.err]+[OMP] deepseek-v41: 104 physical-core threads instead of 208 logical CPUs; SMT can halve decode throughput on some CPUs (#718); set OMP_NUM_THREADS=<n> to override+Matching tokens: 16/16+[v41] prefill 25 tokens in 12.86s | decode 16 in 20.96s (0.76 tok/s)+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 17.17s, expert matmul 11.23s, attention 2.51s, engram 0.35s++--- perf, 208 threads (the shipped default) ---+# Total Lost Samples: 0+# Samples: 5M of event 'cycles:P'+# Overhead Shared Object Symbol IPC [IPC Coverage]+ 38.65% libgomp.so.1.0.0 [.] 0x00000000000256c0 - -+ 35.85% libgomp.so.1.0.0 [.] 0x00000000000258a0 - -+ 4.91% libgomp.so.1.0.0 [.] 0x00000000000256c2 - -+ 4.63% libgomp.so.1.0.0 [.] 0x00000000000258a2 - -+ 4.35% libgomp.so.1.0.0 [.] 0x00000000000256cb - -+ 4.05% libgomp.so.1.0.0 [.] 0x00000000000258ab - -+ 1.24% [kernel.kallsyms] [k] native_queued_spin_lock_slowpath - -+ 1.15% libgomp.so.1.0.0 [.] 0x00000000000256cf - -+ 1.10% libgomp.so.1.0.0 [.] 0x00000000000258b1 - -+ 0.56% libgomp.so.1.0.0 [.] 0x00000000000256c6 - -+ 0.55% libgomp.so.1.0.0 [.] 0x00000000000258a6 - -+diff --git a/docs/experiments/dsv41-omp-team-2026-09-15.md b/docs/experiments/dsv41-omp-team-2026-09-15.mdnew file mode 100644index 00000000..34967ec7--- /dev/null+++ b/docs/experiments/dsv41-omp-team-2026-09-15.md@@ -0,0 +1,135 @@+# The OpenMP team is the whole ballgame on a many-core host++`omp_tune.h` has carried the physical-core sizing since #718, where it measured++2.3x on a 16C/32T part *from the thread count alone*. Four engines call it:+`colibri.c`, `inkling.c`, `kimi_k3.c`, `olmoe.c`. Four were written afterwards and+never did: `glm53.c`, `qwen36.c`, `qwen38.c`, `deepseek_v41.c`.++Nothing failed when that happened, which is why it lasted. The symptom is not an+error. The engine simply runs one thread per **logical** CPU, and on a host where the+logical/physical ratio is large, the per-region barrier cost swallows the arithmetic.+On a 208-logical-CPU machine: **`deepseek-v41` spent 93% of its cycles inside libgomp+and decoded 18.7x slower than it does with a sane team.**++This is the same shape as the mirror and `O_DIRECT` gaps next door -- a mechanism in a+shared header that reached some engines and not their siblings -- and it is the+largest of them by an order of magnitude.++## Machine++| | |+|---|---|+| CPU | 2x Intel Xeon Platinum 8555C, 52 cores/socket, 2 threads/core = **104 physical cores, 208 logical CPUs**, 4 NUMA nodes |+| RAM | 1007 GiB |+| Storage | `nvme0n1` Samsung MZQL2960HCJR (894 GB, ext4, `noatime`) |+| OS / toolchain | Ubuntu 24.04, kernel 6.8.0-31-generic, gcc 13.3.0, `-O3 -march=native -fopenmp`, libgomp |++## Container and method++`deepseek-ai/DeepSeek-V4.1-Flash`, released checkpoint, no conversion: 48 shards,+510.3 GB, 88 files verified against the upstream manifest. The engine's one-shot+oracle mode (`SNAP=<dir> ./deepseek_v41 8 ref.json`), 25 prompt tokens, 16 generated,+`cap=8`, page cache evicted before every run. `ref.json` carries the 16 reference ids,+so **every run below is token-exact 16/16** -- the team size changes scheduling, not+results.++## The profile, first++Before touching anything, `perf record` on a short run:++| overhead | where |+|---:|---|+| 38.65% | `libgomp.so.1.0.0` +0x256c0 |+| 35.85% | `libgomp.so.1.0.0` +0x258a0 |+| 4.91% + 4.63% + 4.35% + 4.05% + ... | the same two instruction streams, sampled at successive IPs (skid) |+| 1.24% | `[k] native_queued_spin_lock_slowpath` |++**93% of cycles in the OpenMP runtime, not one percent in the math.** The engine+issues ~720 parallel regions per decoded token -- three per expert application, 240+applications a token -- and every region is only a few thousand rows wide. At that+size the barrier *is* the work.++## The thread sweep++Same container, same prompt, same reference, cold cache, one variable:++| `OMP_NUM_THREADS` | expert disk | expert matmul | attention | prefill | decode | tok/s | exact |+|---:|---:|---:|---:|---:|---:|---:|---|+| 8 | 15.93 s | 5.91 s | 4.61 s | 14.02 s | 17.34 s | 0.92 | 16/16 |+| 16 | 16.11 | 3.68 | 2.54 | 11.60 | 13.89 | 1.15 | 16/16 |+| **32** | 16.12 | **3.16** | 1.91 | 10.51 | **13.24** | **1.21** | 16/16 |+| 52 | 16.35 | 3.83 | **1.56** | 10.64 | 13.31 | 1.20 | 16/16 |+| 64 | 16.73 | 9.14 | 3.33 | 10.72 | 21.57 | 0.74 | 16/16 |+| 104 (*physical cores -- what the helper picks*) | 16.53 | 10.45 | 2.61 | 11.56 | 21.05 | 0.76 | 16/16 |+| **208 (the default)** | 20.32 | **185.26** | 108.62 | 141.26 | **247.23** | **0.06** | 16/16 |++The default is **18.7x slower on decode** than 32 threads and **11.7x** slower than the+104 physical cores the shared helper selects. The 208 row was measured four separate+times (185.26, 185.26... and 188.98 / 189.82 / 190.64 in other sessions), so it is not+a fluke of one run.++Two details worth keeping:++- **The disk phase moves too** (20.32 s at 208 threads versus ~16 s at any sane size).+ A team of 208 spinning threads steals the memory bandwidth the expert reads need.+ The pathology is not confined to the arithmetic it appears in.+- **Fewer than physical cores is faster still on this host.** 32 threads beat 104 by+ 1.6x. That is why the fix is the *documented, conservative* setting (physical cores,+ no SMT) and not a hard-coded 32: the helper's job is to remove the SMT doubling and+ to respect an explicit `OMP_NUM_THREADS`; finding this machine's optimum is what+ `OMP_NUM_THREADS=<n>` is for.++## The shipped default, before and after++The fix is `#include "omp_tune.h"` plus `coli_omp_tune_threads("<engine>")` as the+first statement of `main` -- the same call the four engines that already had it make.+No other change.++| shipped default | decode | whole turn | tok/s |+|---|---:|---:|---:|+| before (208 logical threads) | 247.2 s | 409.9 s | 0.06 |+| after (`omp_tune.h` -> 104 physical) | **18.6 s** | **43.5 s** | **0.86** |++Three cold runs after the change: wall 43.3 / 43.5 / 47.7 s, decode 18.64 / 18.55 /+20.96 s, **16/16 tokens each**, and each run logging+`[OMP] deepseek-v41: 104 physical-core threads instead of 208 logical CPUs`.++**13.3x on decode, 9.4x on the turn, 14.3x on tokens/second, identical output.**++And the bottleneck moves back to where the other V4.1 work was aimed: at the fixed+default the disk is 16.5 s of a 43.5 s turn (38%), against 5% before. That number was+the whole basis of a separate negative result about multi-SSD streaming; it was+measured in the pathological regime and does not survive the fix.++## What this does not show++- **One host.** The ratio that makes this bite is logical:physical. A 16C/32T desktop+ is the #718 case (+2.3x); a 104C/208T server is this one. An engine on a host with+ no SMT sees nothing either way, which is why the helper stays silent there.+- **The other three engines were not run.** `glm53`, `qwen36` and `qwen38` got the+ same one-line call and are covered by the source gate, but their containers were not+ available here, so their numbers are not claimed. The defect is identical by+ construction -- none of them called anything -- but "identical by construction" is+ not a measurement.+- **`deepseek_v4.c` is not in this change.** It already resizes its team (it reserves+ loader workers via `omp_set_num_threads(logical - COLI_V4_EXPERT_LOADER_COUNT)`), so+ it is not the same omission. Whether that reservation should also be a physical-core+ sizing is a separate question, and it needs its own measurement rather than an+ assumption.++## Reproduce++```sh+make -C c deepseek_v41+# the sweep: one variable, cold cache, token-exact gate on every run+for t in 8 16 32 52 64 104 208; do+ SNAP=<container> OMP_NUM_THREADS=$t ./c/deepseek_v41 8 ref.json+done+# the profile+sudo perf record -F 299 -o /tmp/perf.data -- \+ env SNAP=<container> ./c/deepseek_v41 8 ref.json+sudo perf report --stdio -i /tmp/perf.data --sort dso,symbol+```++Raw per-run engine stderr for the sweep and the before/after runs is attached to the+PR. The gate is `python3 -m unittest tests.test_omp_engine_wiring`.diff --git a/docs/experiments/dsv41-omp-team.json b/docs/experiments/dsv41-omp-team.jsonnew file mode 100644index 00000000..08801bf1--- /dev/null+++ b/docs/experiments/dsv41-omp-team.json@@ -0,0 +1,61 @@+{+ "version": 1,+ "hypothesis": "Sizing the OpenMP team to physical cores instead of one thread per logical CPU raises throughput on an SMT host, because ~720 parallel regions per decoded token make the per-region barrier a first-order cost rather than a rounding error.",+ "commit": "fed625f05ed2befe89d096966b3fce88467908af",+ "model": "deepseek-ai/DeepSeek-V4.1-Flash, released checkpoint, 48 shards, 510.3 GB, fp8 dense + fp4 experts, no conversion",+ "command": "SNAP=/mnt/nvme0/DeepSeek-V4.1-Flash ./deepseek_v41 8 /tmp/ref.json (25 prompt tokens, 16 generated, cap=8, page cache evicted per run; no OMP_NUM_THREADS and no COLI_NO_OMP_TUNE set)",+ "prompt_hash": "79a5bcf37f1920a530cfee5ab7a484e00a1d6bb7f19d9a7418beaa53c14b454b",+ "hardware": {+ "cpu": "2x Intel Xeon Platinum 8555C, 52 cores/socket, 2 threads/core = 104 physical cores, 208 logical CPUs, 4 NUMA nodes",+ "ram": "1007 GiB",+ "storage": "Samsung MZQL2960HCJR (nvme0n1, 894 GB), ext4 noatime, /mnt/nvme0",+ "os": "Ubuntu 24.04, kernel 6.8.0-31-generic, gcc 13.3.0, libgomp, -O3 -march=native -fopenmp"+ },+ "warmup_runs": 0,+ "changed_variables": [+ "OMP_TEAM_SIZING"+ ],+ "baseline": {+ "config": {+ "OMP_TEAM_SIZING": "absent (one thread per logical CPU)"+ },+ "samples": {+ "tok_s": [+ 0.038763,+ 0.039144,+ 0.039252+ ]+ },+ "median_tok_s": 0.039144,+ "quality": {+ "method": "token-exact vs a 16-token reference stream; the engine exits non-zero on any mismatch. All 6 runs 16/16. tok_s is 16 generated tokens over the whole turn (READY to DONE), prefill included",+ "passed": true+ },+ "evidence": {+ "uri": "docs/experiments/dsv41-omp-team-2026-09-15-raw.txt, section 'before'",+ "sha256": "e889f14fd59c116ecd9226be003a1ab2d632b1e4e29fbf435b1dc916b0fb061f"+ }+ },+ "trial": {+ "config": {+ "OMP_TEAM_SIZING": "coli_omp_tune_threads (physical cores)"+ },+ "samples": {+ "tok_s": [+ 0.369515,+ 0.367816,+ 0.33543+ ]+ },+ "median_tok_s": 0.367816,+ "quality": {+ "method": "token-exact vs a 16-token reference stream; the engine exits non-zero on any mismatch. All 6 runs 16/16. tok_s is 16 generated tokens over the whole turn (READY to DONE), prefill included",+ "passed": true+ },+ "evidence": {+ "uri": "docs/experiments/dsv41-omp-team-2026-09-15-raw.txt, section 'after'",+ "sha256": "74dbb54c1dd9cc6fadfc135b4b1fb8c7d338ae812c74cdaf78367aed7f3945f7"+ }+ },+ "outcome": "improvement"+}\ No newline at end of file