Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(v4): delegate OpenMP sizing through launchers

Blake06dd05455475
diff --git a/c/coli b/c/coli
index ba6dc0bc..3eb4a9e8 100755
--- a/c/coli
+++ b/c/coli
@@ -362,7 +362,7 @@ def env_for_engine(a, arch):
if arch == "glm":
return env_for(a)
env = os.environ.copy()
- # OMP_NUM_THREADS for the sister engines too.
+ # OMP_NUM_THREADS for the sister engines that do not size their own team.
#
# #805 set this from physical cores "on every platform" -- but only inside
# env_for(), which this function calls only for glm. inkling, kimi_k3,
@@ -375,12 +375,18 @@ def env_for_engine(a, arch):
# Same defect shape as the one #805 fixed, one level out: a mechanism that
# reaches one engine and not its siblings.
#
- # setdefault, so an explicit OMP_NUM_THREADS still wins -- which is also
- # how to A/B it: OMP_NUM_THREADS=12 reproduces the old behaviour.
+ # DeepSeek V4 is now the exception: its C runtime sizes the team from
+ # logical CPUs minus its expert-loader reservation. Setting the physical
+ # count here makes that runtime mistake a launcher default for a user
+ # override and bypass its loader-aware policy. The other engines still
+ # need the physical-core default below.
+ #
+ # setdefault, so an explicit OMP_NUM_THREADS still wins.
if not env.get("COLI_NO_OMP_TUNE"):
- from resource_plan import physical_cpu_count
- env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))
- if arch == "deepseek_v4":
+ if arch != "deepseek_v4":
+ from resource_plan import physical_cpu_count
+ env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))
+ else:
env.setdefault("OMP_WAIT_POLICY", "active")
env.setdefault("GOMP_SPINCOUNT", "200000")
env.setdefault("OMP_DYNAMIC", "FALSE")
diff --git a/c/openai_server.py b/c/openai_server.py
index c4787e2e..eff780d2 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -1490,8 +1490,9 @@ def tune_child_env(env, arch):
if arch != "deepseek_v4":
return env
if not env.get("COLI_NO_OMP_TUNE"):
- from resource_plan import physical_cpu_count
- env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))
+ # The V4 runtime owns OMP_NUM_THREADS: it reserves logical CPUs for its
+ # expert-loader workers. Supplying a physical-core default here makes
+ # that runtime policy treat the launcher value as a user override.
env.setdefault("OMP_WAIT_POLICY", "active")
env.setdefault("GOMP_SPINCOUNT", "200000")
env.setdefault("OMP_DYNAMIC", "FALSE")
diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.py
index bd48d0d7..655713e4 100644
--- a/c/tests/test_cli_output.py
+++ b/c/tests/test_cli_output.py
@@ -265,12 +265,11 @@ class BannerModelLineTest(unittest.TestCase):
class OmpThreadsForEveryEngineTest(unittest.TestCase):
- """#805 set OMP_NUM_THREADS from physical cores -- for glm only.
+ """Launchers size shared engines; V4 delegates to its loader-aware runtime.
- env_for_engine() forwarded to env_for() when arch was "glm" and built its
- own environment otherwise, so inkling, kimi_k3, olmoe and deepseek_v4 kept
- libgomp's nproc default: logical cores, a 2x over-subscription of a
- memory-bound int4 GEMV on any SMT host.
+ #805's physical-core default still covers the memory-bound sister engines.
+ DeepSeek V4 instead reserves logical CPUs for its expert-loader workers in
+ v4_omp_reserve_loader_cpus(), unless the operator overrides or disables it.
"""
@classmethod
@@ -284,33 +283,43 @@ class OmpThreadsForEveryEngineTest(unittest.TestCase):
return types.SimpleNamespace(model="/x", ram=None, ctx=None, ngen=None,
temp=None, cap=None)
- def test_every_engine_gets_physical_cores(self):
+ def test_other_engines_get_physical_cores(self):
with mock.patch("resource_plan.physical_cpu_count", return_value=6):
- for arch in ("inkling", "kimi", "olmoe", "deepseek_v4"):
+ for arch in ("inkling", "kimi", "olmoe"):
with self.subTest(arch=arch):
env = self.coli.env_for_engine(self.args(), arch)
self.assertEqual(env.get("OMP_NUM_THREADS"), "6")
+ def test_v4_delegates_thread_team_to_runtime(self):
+ with mock.patch.dict(os.environ, {}, clear=True), \
+ mock.patch("resource_plan.physical_cpu_count",
+ side_effect=AssertionError("V4 launcher sized the team")):
+ env = self.coli.env_for_engine(self.args(), "deepseek_v4")
+ self.assertNotIn("OMP_NUM_THREADS", env)
+
def test_v4_gets_memory_bound_affinity_defaults(self):
- with mock.patch.object(self.coli.sys, "platform", "linux"), \
- mock.patch("resource_plan.physical_cpu_count", return_value=6):
+ with mock.patch.dict(os.environ, {}, clear=True), \
+ mock.patch.object(self.coli.sys, "platform", "linux"):
env = self.coli.env_for_engine(self.args(), "deepseek_v4")
self.assertEqual(env.get("OMP_PROC_BIND"), "close")
self.assertEqual(env.get("OMP_PLACES"), "cores")
self.assertEqual(env.get("OMP_WAIT_POLICY"), "active")
+ self.assertEqual(env.get("GOMP_SPINCOUNT"), "200000")
self.assertEqual(env.get("OMP_DYNAMIC"), "FALSE")
def test_explicit_setting_still_wins(self):
- with mock.patch.dict(os.environ, {"OMP_NUM_THREADS": "3"}), \
- mock.patch("resource_plan.physical_cpu_count", return_value=6):
+ with mock.patch.dict(os.environ, {"OMP_NUM_THREADS": "3"}, clear=True), \
+ mock.patch("resource_plan.physical_cpu_count",
+ side_effect=AssertionError("V4 launcher sized the team")):
env = self.coli.env_for_engine(self.args(), "deepseek_v4")
self.assertEqual(env["OMP_NUM_THREADS"], "3")
def test_kill_switch_is_honoured(self):
- with mock.patch.dict(os.environ, {"COLI_NO_OMP_TUNE": "1"}, clear=False):
- os.environ.pop("OMP_NUM_THREADS", None)
+ with mock.patch.dict(os.environ, {"COLI_NO_OMP_TUNE": "1"}, clear=True):
env = self.coli.env_for_engine(self.args(), "deepseek_v4")
- self.assertNotIn("OMP_NUM_THREADS", env)
+ for key in ("OMP_NUM_THREADS", "OMP_WAIT_POLICY", "GOMP_SPINCOUNT",
+ "OMP_DYNAMIC", "OMP_PROC_BIND", "OMP_PLACES"):
+ self.assertNotIn(key, env)
if __name__ == "__main__":
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 5e2836b2..27d7495f 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -708,10 +708,11 @@ class CapSentinelShimTest(unittest.TestCase):
def test_direct_v4_server_gets_bounded_dspark_defaults(self):
env = {"V4_MTP_CONF": "0.7"}
- with patch("resource_plan.physical_cpu_count", return_value=6), \
+ with patch("resource_plan.physical_cpu_count",
+ side_effect=AssertionError("V4 server sized the team")), \
patch("openai_server.sys.platform", "linux"):
tune_child_env(env, "deepseek_v4")
- self.assertEqual(env["OMP_NUM_THREADS"], "6")
+ self.assertNotIn("OMP_NUM_THREADS", env)
self.assertEqual(env["OMP_PROC_BIND"], "close")
self.assertEqual(env["V4_DRAFT"], "0")
self.assertEqual(env["V4_MTP"], "0")
@@ -719,6 +720,18 @@ class CapSentinelShimTest(unittest.TestCase):
self.assertEqual(env["V4_MTP_GB"], "0.45")
self.assertEqual(env["V4_MTP_CONF"], "0.7") # explicit override wins
+ def test_direct_v4_server_preserves_explicit_omp_threads(self):
+ env = {"OMP_NUM_THREADS": "3"}
+ tune_child_env(env, "deepseek_v4")
+ self.assertEqual(env["OMP_NUM_THREADS"], "3")
+
+ def test_direct_v4_server_honours_omp_kill_switch(self):
+ env = {"COLI_NO_OMP_TUNE": "1"}
+ tune_child_env(env, "deepseek_v4")
+ for key in ("OMP_NUM_THREADS", "OMP_WAIT_POLICY", "GOMP_SPINCOUNT",
+ "OMP_DYNAMIC", "OMP_PROC_BIND", "OMP_PLACES"):
+ self.assertNotIn(key, env)
+
class HTTPTest(unittest.TestCase):
@classmethod