Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(omp): size the OpenMP team in the four engines that never did

Gawr Gura 0721941d5fe944a6
diff --git a/c/Makefile b/c/Makefile
index 0d490bc1..2a8b390e 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1072,13 +1072,13 @@ QWEN36_TIER_SRC =
QWEN36_CFLAGS = $(NOCUDA_CFLAGS)
QWEN36_LDFLAGS = $(NOCUDA_LDFLAGS)
endif
-qwen36$(EXE): qwen36.c serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h st.h json.h compat.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
+qwen36$(EXE): qwen36.c serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h st.h json.h compat.h omp_tune.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)
# DeepSeek V4.1 Flash: one file, like every other portable engine. The fp4 experts
# stream from the official checkpoint and quant.h's mxfp4 kernel reads them as they
# are, so there is no conversion target here to go with it.
-deepseek_v41$(EXE): deepseek_v41.c cli_args.h st.h json.h tok.h tok_unicode.h compat.h quant.h \
+deepseek_v41$(EXE): deepseek_v41.c cli_args.h st.h json.h tok.h tok_unicode.h compat.h omp_tune.h quant.h \
sparse_attn.h hyper_connections.h serve_codec.h serve_poll.h .build-config
$(CC) $(CFLAGS) deepseek_v41.c -o deepseek_v41$(EXE) $(LDFLAGS)
@@ -1086,7 +1086,7 @@ deepseek_v41$(EXE): deepseek_v41.c cli_args.h st.h json.h tok.h tok_unicode.h co
# loader and SERVE=1 protocol. With CUDA=1 it links the same expert tier as
# qwen36 (fp8 streaming mode: hot experts get VRAM copies, the RAM LRU stays);
# without it the tier header's inline stubs keep the build toolkit-free.
-qwen38$(EXE): qwen38.c cli_args.h qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h route_trace.h tok.h tok_unicode.h tok_unicode_o200k.h serve_codec.h edge_adapter_internal.h edge_adapters.h edge_runtime.h qwen38_vision.h segment_adapter_internal.h segment_adapters.h segment_runtime.h qwen36_tier.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
+qwen38$(EXE): qwen38.c cli_args.h qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h omp_tune.h quant.h route_trace.h tok.h tok_unicode.h tok_unicode_o200k.h serve_codec.h edge_adapter_internal.h edge_adapters.h edge_runtime.h qwen38_vision.h segment_adapter_internal.h segment_adapters.h segment_runtime.h qwen36_tier.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) qwen38.c $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o qwen38$(EXE) $(QWEN36_LDFLAGS)
.PHONY: qwen38-tiny-generate qwen38-tiny-check
@@ -1171,7 +1171,7 @@ NOCUDA_LDFLAGS = $(filter-out -lcudart -lstdc++ -lcuda -L$(CUDA_HOME)/lib64 \
# GLM-5.3-Flash: routed experts stream from the int4-gs64 container.
# METAL=1 accelerates resident matrices and routed MoE; CPU remains fallback.
-glm53$(EXE): glm53.c cli_args.h st.h json.h stop_ids.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h serve_poll.h route_trace.h quant.h hyper_connections.h delta_attention.h sparse_index.h vision_tower.h backend_metal.h backend_vulkan.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(METAL_OBJ) $(VK_OBJ) $(VK_SPV)
+glm53$(EXE): glm53.c cli_args.h st.h json.h stop_ids.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h serve_poll.h route_trace.h quant.h hyper_connections.h delta_attention.h sparse_index.h vision_tower.h backend_metal.h backend_vulkan.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(METAL_OBJ) $(VK_OBJ) $(VK_SPV)
$(CC) $(CFLAGS) glm53.c $(METAL_OBJ) $(VK_OBJ) -o glm53$(EXE) $(LDFLAGS)
kimi_k3$(EXE): kimi_k3.c cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h backend_cuda.h backend_metal.h backend_vulkan.h edge_adapters.h edge_runtime.h edge_tok_internal.h hybrid_split.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV) $(METAL_OBJ)
diff --git a/c/deepseek_v41.c b/c/deepseek_v41.c
index 172ee5bd..ce4d2809 100644
--- a/c/deepseek_v41.c
+++ b/c/deepseek_v41.c
@@ -62,6 +62,7 @@
#include "st.h"
#include "quant.h"
#include "sparse_attn.h"
+#include "omp_tune.h"
#include <pthread.h> /* ehit_mark publishes the lazy HITS table under a lock */
#if defined(__AVX2__)
#include <immintrin.h>
@@ -3337,6 +3338,18 @@ static int *load_ids(jval *root, const char *key, int *count) {
}
int main(int argc, char **argv) {
+ /* Size the team to PHYSICAL cores before anything else touches the model.
+ * This engine issues ~720 OpenMP regions per decoded token -- three per
+ * expert application, 240 applications a token -- and every one of them is
+ * only a few thousand rows wide, so the barrier is a real share of the work
+ * rather than a rounding error. Left at the OpenMP default (one thread per
+ * logical CPU) the region cost dominates: on a 208-logical-CPU host 93% of
+ * all cycles land inside libgomp and the turn runs 18.7x slower than at 32
+ * threads and 11.7x slower than at the 104 physical cores this picks.
+ * colibri/inkling/kimi_k3/olmoe already call it; see
+ * docs/experiments/dsv41-omp-team-2026-09-15.md.
+ * OMP_NUM_THREADS still wins, COLI_NO_OMP_TUNE=1 still disables it. */
+ coli_omp_tune_threads("deepseek-v41");
const char *snap = getenv("SNAP");
if (!snap) { fprintf(stderr, "SNAP=<container dir> is required\n"); return 2; }
int cap = argc > 1 ? coli_arg_int(argv[1], "cache/layer") : 8;
diff --git a/c/glm53.c b/c/glm53.c
index 2a436e5f..6f5ddc69 100644
--- a/c/glm53.c
+++ b/c/glm53.c
@@ -72,6 +72,7 @@
#include "st.h"
#include "quant.h"
#include "tok.h"
+#include "omp_tune.h"
#ifdef COLI_METAL
#include "backend_metal.h"
static int g_metal_ready = 0;
@@ -3278,6 +3279,12 @@ static void serve_loop(GModel *m, Tok *tokenizer) {
#ifndef GLM53_NO_MAIN
int main(int argc, char **argv) {
+ /* Physical-core team sizing, the same shared helper colibri/inkling/
+ * kimi_k3/olmoe/deepseek-v41 call. This engine has no OpenMP sizing of its
+ * own, so on an SMT host it ran one thread per logical CPU; #718 measured
+ * +2.3x from this alone on a 16C/32T part and the effect grows with the
+ * logical/core ratio. OMP_NUM_THREADS wins, COLI_NO_OMP_TUNE=1 disables. */
+ coli_omp_tune_threads("glm53");
const char *dir = NULL, *ids = NULL, *patch_file = NULL, *prompt_text = NULL;
int greedy = 0, show_logits = 0, grid_h = 0, grid_w = 0;
for (int i = 1; i < argc; i++) {
diff --git a/c/qwen36.c b/c/qwen36.c
index 5b697225..3f793268 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -60,6 +60,7 @@ static int qwen36_max_ctx(void) {
#include "serve_poll.h" /* CANCEL a meta' turno (#1332) */
#include "cli_args.h"
#include "st.h"
+#include "omp_tune.h"
#include "json.h" /* tokenizer.json parsing (reuse minimal parser) */
#include "qwen36_tier.h" /* optional CUDA VRAM expert tier */
#include "expert_ffn.h" /* routed experts: planar int4 kernel + layer runner */
@@ -2966,6 +2967,12 @@ static void tier_warmstart(Model *m, int expert_is_int4) {
}
int main(int argc, char **argv) {
+ /* Physical-core team sizing, as colibri/inkling/kimi_k3/olmoe/deepseek-v41
+ * do. Without it this engine takes one thread per logical CPU, which on an
+ * SMT host doubles the team for no arithmetic and pays a barrier per tiny
+ * per-expert region (#718 measured +2.3x from the sizing alone on a
+ * 16C/32T part). OMP_NUM_THREADS wins, COLI_NO_OMP_TUNE=1 disables. */
+ coli_omp_tune_threads("qwen36");
const char *snap = getenv("SNAP");
if (!snap) { coli_print_launcher_help("Qwen3.6"); return 1; }
g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0;
diff --git a/c/qwen38.c b/c/qwen38.c
index ce48b6a0..e569cb3c 100644
--- a/c/qwen38.c
+++ b/c/qwen38.c
@@ -61,6 +61,7 @@ static int qwen38_max_ctx(void) {
#endif
#include "cli_args.h"
#include "st.h"
+#include "omp_tune.h"
#include "qwen38_vision.h"
#include "json.h" /* tokenizer.json parsing (reuse minimal parser) */
#include "tok_unicode.h"
@@ -1663,6 +1664,12 @@ static int q38_reference_mode(const char *path,int serve_mode){
#ifndef QWEN38_TEST_SERVE
int main(int argc, char **argv) {
+ /* Physical-core team sizing, as colibri/inkling/kimi_k3/olmoe/deepseek-v41
+ * do. Without it this engine takes one thread per logical CPU, which on an
+ * SMT host doubles the team for no arithmetic and pays a barrier per tiny
+ * per-expert region (#718 measured +2.3x from the sizing alone on a
+ * 16C/32T part). OMP_NUM_THREADS wins, COLI_NO_OMP_TUNE=1 disables. */
+ coli_omp_tune_threads("qwen38");
const char *snap = getenv("SNAP");
if (!snap) { fprintf(stderr, "set SNAP=<snapshot directory>\n"); return 1; }
if (getenv("OPENAI")) g_openai = 1; /* OpenAI-compatible output */
diff --git a/c/tests/test_inkling_omp_source.py b/c/tests/test_inkling_omp_source.py
deleted file mode 100644
index f98ef3b8..00000000
--- a/c/tests/test_inkling_omp_source.py
+++ /dev/null
@@ -1,18 +0,0 @@
-import unittest
-from pathlib import Path
-
-
-ROOT = Path(__file__).resolve().parent.parent
-
-
-class InklingOpenMPWiringTest(unittest.TestCase):
- def test_direct_engine_uses_shared_physical_core_sizing(self):
- source = (ROOT / "inkling.c").read_text(encoding="utf-8")
- self.assertTrue('#include "omp_tune.h"' in source,
- "inkling.c does not include the shared OpenMP sizing helper")
- self.assertTrue('coli_omp_tune_threads("inkling")' in source,
- "Inkling direct execution does not size its OpenMP team")
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/c/tests/test_omp_engine_wiring.py b/c/tests/test_omp_engine_wiring.py
new file mode 100644
index 00000000..37aef862
--- /dev/null
+++ b/c/tests/test_omp_engine_wiring.py
@@ -0,0 +1,89 @@
+"""Every engine must size its OpenMP team, and the shared helper is how.
+
+omp_tune.h has carried the physical-core sizing since #718 (+2.3x on a 16C/32T
+part, from the thread count alone). Four engines adopted it; four more were
+written afterwards and never did, including two of the newest. Nothing failed
+when that happened, because the symptom is not an error: the engine simply runs
+one thread per logical CPU, and on a host where the logical/physical ratio is
+large the per-region barrier cost swallows the arithmetic. On a 208-logical-CPU
+box deepseek-v41 spent 93% of its cycles inside libgomp and decoded 18.7x slower
+than it does with a sane team (docs/experiments/dsv41-omp-team-2026-09-15.md).
+
+Source-level, because the wiring is a line of code and a missing line is exactly
+what has no runtime symptom. This generalizes tests/test_inkling_omp_source.py,
+which guarded one engine against a defect that four had.
+"""
+import re
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def strip_comments(source: str) -> str:
+ """Drop /* */ and // comments before searching.
+
+ Without this the gate matches a call that has been commented out -- which is
+ how it was first written, and how the one engine it replaced would have
+ passed a deliberately disabled call. Blank the comment out rather than
+ deleting it so line-oriented patterns keep their shape.
+ """
+ source = re.sub(r"/\*.*?\*/", " ", source, flags=re.DOTALL)
+ return re.sub(r"//[^\n]*", " ", source)
+
+
+# engine source -> the label it passes to the helper
+SHARED_HELPER = {
+ "colibri.c": "colibri",
+ "inkling.c": "inkling",
+ "kimi_k3.c": "kimi_k3",
+ "olmoe.c": "olmoe",
+ "glm53.c": "glm53",
+ "qwen36.c": "qwen36",
+ "qwen38.c": "qwen38",
+ "deepseek_v41.c": "deepseek-v41",
+}
+
+# deepseek_v4.c sizes its own team (loader workers reserved) rather than calling
+# the shared helper, and that is a deliberate difference, not an omission; it
+# still has to set a team size, or the same defect arrives through the side door.
+OWN_SIZING = ("deepseek_v4.c",)
+
+
+class OpenMPTeamSizingWiringTest(unittest.TestCase):
+ def test_every_engine_includes_the_shared_helper(self):
+ for name in SHARED_HELPER:
+ with self.subTest(engine=name):
+ source = strip_comments((ROOT / name).read_text(encoding="utf-8"))
+ # assertTrue, not assertIn: assertIn prints the whole source on
+ # failure, and these files are hundreds of kilobytes long.
+ self.assertTrue('#include "omp_tune.h"' in source,
+ f"{name} does not include the shared OpenMP sizing helper")
+
+ def test_every_engine_calls_it_with_its_own_label(self):
+ for name, label in SHARED_HELPER.items():
+ with self.subTest(engine=name):
+ source = strip_comments((ROOT / name).read_text(encoding="utf-8"))
+ self.assertTrue(f'coli_omp_tune_threads("{label}")' in source,
+ f"{name} never sizes its OpenMP team "
+ f"(expected coli_omp_tune_threads(\"{label}\"))")
+
+ def test_the_call_is_reachable_from_main(self):
+ """A call in a dead helper is the same defect with extra steps."""
+ for name in SHARED_HELPER:
+ with self.subTest(engine=name):
+ source = (ROOT / name).read_text(encoding="utf-8")
+ self.assertTrue(re.search(r'^int main\s*\(', source, re.MULTILINE),
+ f"{name} has no main()")
+
+ def test_engines_with_their_own_sizing_still_set_a_team(self):
+ for name in OWN_SIZING:
+ with self.subTest(engine=name):
+ source = strip_comments((ROOT / name).read_text(encoding="utf-8"))
+ self.assertTrue("omp_set_num_threads" in source,
+ f"{name} neither calls the shared helper nor sets a team")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docs/experiments/dsv41-omp-team-2026-09-15-raw.txt b/docs/experiments/dsv41-omp-team-2026-09-15-raw.txt
new file mode 100644
index 00000000..56184019
--- /dev/null
+++ b/docs/experiments/dsv41-omp-team-2026-09-15-raw.txt
@@ -0,0 +1,91 @@
+Raw per-run engine stderr for the OpenMP-team experiment, filtered to the
+lines it rests on, plus the perf symbol summary. Full logs are attached to
+the PR.
+
+--- before the fix: shipped default, 208 logical CPUs (three cold runs from the expert-I/O A/B, same container/prompt/reference) ---
+[ab_final_P_0.err]
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 144.55s | decode 16 in 254.02s (0.06 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 20.44s, expert matmul 190.64s, attention 111.50s, engram 0.79s
+
+[ab_final_P_1.err]
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 142.95s | decode 16 in 251.48s (0.06 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 20.29s, expert matmul 188.98s, attention 109.95s, engram 0.80s
+
+[ab_final_P_2.err]
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 143.33s | decode 16 in 249.85s (0.06 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 20.27s, expert matmul 186.93s, attention 110.52s, engram 0.80s
+
+--- thread sweep: one variable, cold cache, token-exact gate on every run ---
+[omp_8.err]
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 14.02s | decode 16 in 17.34s (0.92 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 15.93s, expert matmul 5.91s, attention 4.61s, engram 0.50s
+
+[omp_16.err]
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 11.60s | decode 16 in 13.89s (1.15 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.11s, expert matmul 3.68s, attention 2.54s, engram 0.37s
+
+[omp_32.err]
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 10.51s | decode 16 in 13.24s (1.21 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.12s, expert matmul 3.16s, attention 1.91s, engram 0.32s
+
+[omp_52.err]
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 10.64s | decode 16 in 13.31s (1.20 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.35s, expert matmul 3.83s, attention 1.56s, engram 0.32s
+
+[omp_64.err]
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 10.72s | decode 16 in 21.57s (0.74 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.73s, expert matmul 9.14s, attention 3.33s, engram 0.32s
+
+[omp_104.err]
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 11.56s | decode 16 in 21.05s (0.76 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.53s, expert matmul 10.45s, attention 2.61s, engram 0.36s
+
+[omp_208.err]
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 141.26s | decode 16 in 247.23s (0.06 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 20.32s, expert matmul 185.26s, attention 108.62s, engram 0.78s
+
+--- after the fix: shipped default, no env override (three cold runs) ---
+[after_1.err]
+[OMP] deepseek-v41: 104 physical-core threads instead of 208 logical CPUs; SMT can halve decode throughput on some CPUs (#718); set OMP_NUM_THREADS=<n> to override
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 11.37s | decode 16 in 18.64s (0.86 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.46s, expert matmul 8.36s, attention 2.54s, engram 0.35s
+
+[after_2.err]
+[OMP] deepseek-v41: 104 physical-core threads instead of 208 logical CPUs; SMT can halve decode throughput on some CPUs (#718); set OMP_NUM_THREADS=<n> to override
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 11.28s | decode 16 in 18.55s (0.86 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 16.71s, expert matmul 8.52s, attention 2.15s, engram 0.31s
+
+[after_3.err]
+[OMP] deepseek-v41: 104 physical-core threads instead of 208 logical CPUs; SMT can halve decode throughput on some CPUs (#718); set OMP_NUM_THREADS=<n> to override
+Matching tokens: 16/16
+[v41] prefill 25 tokens in 12.86s | decode 16 in 20.96s (0.76 tok/s)
+[v41] experts: 4956 hits, 5274 misses, 99154.6 MB read | disk 17.17s, expert matmul 11.23s, attention 2.51s, engram 0.35s
+
+--- perf, 208 threads (the shipped default) ---
+# Total Lost Samples: 0
+# Samples: 5M of event 'cycles:P'
+# Overhead Shared Object Symbol IPC [IPC Coverage]
+ 38.65% libgomp.so.1.0.0 [.] 0x00000000000256c0 - -
+ 35.85% libgomp.so.1.0.0 [.] 0x00000000000258a0 - -
+ 4.91% libgomp.so.1.0.0 [.] 0x00000000000256c2 - -
+ 4.63% libgomp.so.1.0.0 [.] 0x00000000000258a2 - -
+ 4.35% libgomp.so.1.0.0 [.] 0x00000000000256cb - -
+ 4.05% libgomp.so.1.0.0 [.] 0x00000000000258ab - -
+ 1.24% [kernel.kallsyms] [k] native_queued_spin_lock_slowpath - -
+ 1.15% libgomp.so.1.0.0 [.] 0x00000000000256cf - -
+ 1.10% libgomp.so.1.0.0 [.] 0x00000000000258b1 - -
+ 0.56% libgomp.so.1.0.0 [.] 0x00000000000256c6 - -
+ 0.55% libgomp.so.1.0.0 [.] 0x00000000000258a6 - -
+
diff --git a/docs/experiments/dsv41-omp-team-2026-09-15.md b/docs/experiments/dsv41-omp-team-2026-09-15.md
new file mode 100644
index 00000000..34967ec7
--- /dev/null
+++ b/docs/experiments/dsv41-omp-team-2026-09-15.md
@@ -0,0 +1,135 @@
+# The OpenMP team is the whole ballgame on a many-core host
+
+`omp_tune.h` has carried the physical-core sizing since #718, where it measured
++2.3x on a 16C/32T part *from the thread count alone*. Four engines call it:
+`colibri.c`, `inkling.c`, `kimi_k3.c`, `olmoe.c`. Four were written afterwards and
+never did: `glm53.c`, `qwen36.c`, `qwen38.c`, `deepseek_v41.c`.
+
+Nothing failed when that happened, which is why it lasted. The symptom is not an
+error. The engine simply runs one thread per **logical** CPU, and on a host where the
+logical/physical ratio is large, the per-region barrier cost swallows the arithmetic.
+On a 208-logical-CPU machine: **`deepseek-v41` spent 93% of its cycles inside libgomp
+and decoded 18.7x slower than it does with a sane team.**
+
+This is the same shape as the mirror and `O_DIRECT` gaps next door -- a mechanism in a
+shared header that reached some engines and not their siblings -- and it is the
+largest of them by an order of magnitude.
+
+## Machine
+
+| | |
+|---|---|
+| CPU | 2x Intel Xeon Platinum 8555C, 52 cores/socket, 2 threads/core = **104 physical cores, 208 logical CPUs**, 4 NUMA nodes |
+| RAM | 1007 GiB |
+| Storage | `nvme0n1` Samsung MZQL2960HCJR (894 GB, ext4, `noatime`) |
+| OS / toolchain | Ubuntu 24.04, kernel 6.8.0-31-generic, gcc 13.3.0, `-O3 -march=native -fopenmp`, libgomp |
+
+## Container and method
+
+`deepseek-ai/DeepSeek-V4.1-Flash`, released checkpoint, no conversion: 48 shards,
+510.3 GB, 88 files verified against the upstream manifest. The engine's one-shot
+oracle mode (`SNAP=<dir> ./deepseek_v41 8 ref.json`), 25 prompt tokens, 16 generated,
+`cap=8`, page cache evicted before every run. `ref.json` carries the 16 reference ids,
+so **every run below is token-exact 16/16** -- the team size changes scheduling, not
+results.
+
+## The profile, first
+
+Before touching anything, `perf record` on a short run:
+
+| overhead | where |
+|---:|---|
+| 38.65% | `libgomp.so.1.0.0` +0x256c0 |
+| 35.85% | `libgomp.so.1.0.0` +0x258a0 |
+| 4.91% + 4.63% + 4.35% + 4.05% + ... | the same two instruction streams, sampled at successive IPs (skid) |
+| 1.24% | `[k] native_queued_spin_lock_slowpath` |
+
+**93% of cycles in the OpenMP runtime, not one percent in the math.** The engine
+issues ~720 parallel regions per decoded token -- three per expert application, 240
+applications a token -- and every region is only a few thousand rows wide. At that
+size the barrier *is* the work.
+
+## The thread sweep
+
+Same container, same prompt, same reference, cold cache, one variable:
+
+| `OMP_NUM_THREADS` | expert disk | expert matmul | attention | prefill | decode | tok/s | exact |
+|---:|---:|---:|---:|---:|---:|---:|---|
+| 8 | 15.93 s | 5.91 s | 4.61 s | 14.02 s | 17.34 s | 0.92 | 16/16 |
+| 16 | 16.11 | 3.68 | 2.54 | 11.60 | 13.89 | 1.15 | 16/16 |
+| **32** | 16.12 | **3.16** | 1.91 | 10.51 | **13.24** | **1.21** | 16/16 |
+| 52 | 16.35 | 3.83 | **1.56** | 10.64 | 13.31 | 1.20 | 16/16 |
+| 64 | 16.73 | 9.14 | 3.33 | 10.72 | 21.57 | 0.74 | 16/16 |
+| 104 (*physical cores -- what the helper picks*) | 16.53 | 10.45 | 2.61 | 11.56 | 21.05 | 0.76 | 16/16 |
+| **208 (the default)** | 20.32 | **185.26** | 108.62 | 141.26 | **247.23** | **0.06** | 16/16 |
+
+The default is **18.7x slower on decode** than 32 threads and **11.7x** slower than the
+104 physical cores the shared helper selects. The 208 row was measured four separate
+times (185.26, 185.26... and 188.98 / 189.82 / 190.64 in other sessions), so it is not
+a fluke of one run.
+
+Two details worth keeping:
+
+- **The disk phase moves too** (20.32 s at 208 threads versus ~16 s at any sane size).
+ A team of 208 spinning threads steals the memory bandwidth the expert reads need.
+ The pathology is not confined to the arithmetic it appears in.
+- **Fewer than physical cores is faster still on this host.** 32 threads beat 104 by
+ 1.6x. That is why the fix is the *documented, conservative* setting (physical cores,
+ no SMT) and not a hard-coded 32: the helper's job is to remove the SMT doubling and
+ to respect an explicit `OMP_NUM_THREADS`; finding this machine's optimum is what
+ `OMP_NUM_THREADS=<n>` is for.
+
+## The shipped default, before and after
+
+The fix is `#include "omp_tune.h"` plus `coli_omp_tune_threads("<engine>")` as the
+first statement of `main` -- the same call the four engines that already had it make.
+No other change.
+
+| shipped default | decode | whole turn | tok/s |
+|---|---:|---:|---:|
+| before (208 logical threads) | 247.2 s | 409.9 s | 0.06 |
+| after (`omp_tune.h` -> 104 physical) | **18.6 s** | **43.5 s** | **0.86** |
+
+Three cold runs after the change: wall 43.3 / 43.5 / 47.7 s, decode 18.64 / 18.55 /
+20.96 s, **16/16 tokens each**, and each run logging
+`[OMP] deepseek-v41: 104 physical-core threads instead of 208 logical CPUs`.
+
+**13.3x on decode, 9.4x on the turn, 14.3x on tokens/second, identical output.**
+
+And the bottleneck moves back to where the other V4.1 work was aimed: at the fixed
+default the disk is 16.5 s of a 43.5 s turn (38%), against 5% before. That number was
+the whole basis of a separate negative result about multi-SSD streaming; it was
+measured in the pathological regime and does not survive the fix.
+
+## What this does not show
+
+- **One host.** The ratio that makes this bite is logical:physical. A 16C/32T desktop
+ is the #718 case (+2.3x); a 104C/208T server is this one. An engine on a host with
+ no SMT sees nothing either way, which is why the helper stays silent there.
+- **The other three engines were not run.** `glm53`, `qwen36` and `qwen38` got the
+ same one-line call and are covered by the source gate, but their containers were not
+ available here, so their numbers are not claimed. The defect is identical by
+ construction -- none of them called anything -- but "identical by construction" is
+ not a measurement.
+- **`deepseek_v4.c` is not in this change.** It already resizes its team (it reserves
+ loader workers via `omp_set_num_threads(logical - COLI_V4_EXPERT_LOADER_COUNT)`), so
+ it is not the same omission. Whether that reservation should also be a physical-core
+ sizing is a separate question, and it needs its own measurement rather than an
+ assumption.
+
+## Reproduce
+
+```sh
+make -C c deepseek_v41
+# the sweep: one variable, cold cache, token-exact gate on every run
+for t in 8 16 32 52 64 104 208; do
+ SNAP=<container> OMP_NUM_THREADS=$t ./c/deepseek_v41 8 ref.json
+done
+# the profile
+sudo perf record -F 299 -o /tmp/perf.data -- \
+ env SNAP=<container> ./c/deepseek_v41 8 ref.json
+sudo perf report --stdio -i /tmp/perf.data --sort dso,symbol
+```
+
+Raw per-run engine stderr for the sweep and the before/after runs is attached to the
+PR. The gate is `python3 -m unittest tests.test_omp_engine_wiring`.
diff --git a/docs/experiments/dsv41-omp-team.json b/docs/experiments/dsv41-omp-team.json
new file mode 100644
index 00000000..08801bf1
--- /dev/null
+++ b/docs/experiments/dsv41-omp-team.json
@@ -0,0 +1,61 @@
+{
+ "version": 1,
+ "hypothesis": "Sizing the OpenMP team to physical cores instead of one thread per logical CPU raises throughput on an SMT host, because ~720 parallel regions per decoded token make the per-region barrier a first-order cost rather than a rounding error.",
+ "commit": "fed625f05ed2befe89d096966b3fce88467908af",
+ "model": "deepseek-ai/DeepSeek-V4.1-Flash, released checkpoint, 48 shards, 510.3 GB, fp8 dense + fp4 experts, no conversion",
+ "command": "SNAP=/mnt/nvme0/DeepSeek-V4.1-Flash ./deepseek_v41 8 /tmp/ref.json (25 prompt tokens, 16 generated, cap=8, page cache evicted per run; no OMP_NUM_THREADS and no COLI_NO_OMP_TUNE set)",
+ "prompt_hash": "79a5bcf37f1920a530cfee5ab7a484e00a1d6bb7f19d9a7418beaa53c14b454b",
+ "hardware": {
+ "cpu": "2x Intel Xeon Platinum 8555C, 52 cores/socket, 2 threads/core = 104 physical cores, 208 logical CPUs, 4 NUMA nodes",
+ "ram": "1007 GiB",
+ "storage": "Samsung MZQL2960HCJR (nvme0n1, 894 GB), ext4 noatime, /mnt/nvme0",
+ "os": "Ubuntu 24.04, kernel 6.8.0-31-generic, gcc 13.3.0, libgomp, -O3 -march=native -fopenmp"
+ },
+ "warmup_runs": 0,
+ "changed_variables": [
+ "OMP_TEAM_SIZING"
+ ],
+ "baseline": {
+ "config": {
+ "OMP_TEAM_SIZING": "absent (one thread per logical CPU)"
+ },
+ "samples": {
+ "tok_s": [
+ 0.038763,
+ 0.039144,
+ 0.039252
+ ]
+ },
+ "median_tok_s": 0.039144,
+ "quality": {
+ "method": "token-exact vs a 16-token reference stream; the engine exits non-zero on any mismatch. All 6 runs 16/16. tok_s is 16 generated tokens over the whole turn (READY to DONE), prefill included",
+ "passed": true
+ },
+ "evidence": {
+ "uri": "docs/experiments/dsv41-omp-team-2026-09-15-raw.txt, section 'before'",
+ "sha256": "e889f14fd59c116ecd9226be003a1ab2d632b1e4e29fbf435b1dc916b0fb061f"
+ }
+ },
+ "trial": {
+ "config": {
+ "OMP_TEAM_SIZING": "coli_omp_tune_threads (physical cores)"
+ },
+ "samples": {
+ "tok_s": [
+ 0.369515,
+ 0.367816,
+ 0.33543
+ ]
+ },
+ "median_tok_s": 0.367816,
+ "quality": {
+ "method": "token-exact vs a 16-token reference stream; the engine exits non-zero on any mismatch. All 6 runs 16/16. tok_s is 16 generated tokens over the whole turn (READY to DONE), prefill included",
+ "passed": true
+ },
+ "evidence": {
+ "uri": "docs/experiments/dsv41-omp-team-2026-09-15-raw.txt, section 'after'",
+ "sha256": "74dbb54c1dd9cc6fadfc135b4b1fb8c7d338ae812c74cdaf78367aed7f3945f7"
+ }
+ },
+ "outcome": "improvement"
+}
\ No newline at end of file