Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(v4): delegate OpenMP sizing through launchers
Blake06dd05455475
diff --git a/c/coli b/c/coliindex ba6dc0bc..3eb4a9e8 100755--- a/c/coli+++ b/c/coli@@ -362,7 +362,7 @@ def env_for_engine(a, arch):if arch == "glm":return env_for(a)env = os.environ.copy()- # OMP_NUM_THREADS for the sister engines too.+ # OMP_NUM_THREADS for the sister engines that do not size their own team.## #805 set this from physical cores "on every platform" -- but only inside# env_for(), which this function calls only for glm. inkling, kimi_k3,@@ -375,12 +375,18 @@ def env_for_engine(a, arch):# Same defect shape as the one #805 fixed, one level out: a mechanism that# reaches one engine and not its siblings.#- # setdefault, so an explicit OMP_NUM_THREADS still wins -- which is also- # how to A/B it: OMP_NUM_THREADS=12 reproduces the old behaviour.+ # DeepSeek V4 is now the exception: its C runtime sizes the team from+ # logical CPUs minus its expert-loader reservation. Setting the physical+ # count here makes that runtime mistake a launcher default for a user+ # override and bypass its loader-aware policy. The other engines still+ # need the physical-core default below.+ #+ # setdefault, so an explicit OMP_NUM_THREADS still wins.if not env.get("COLI_NO_OMP_TUNE"):- from resource_plan import physical_cpu_count- env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))- if arch == "deepseek_v4":+ if arch != "deepseek_v4":+ from resource_plan import physical_cpu_count+ env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))+ else:env.setdefault("OMP_WAIT_POLICY", "active")env.setdefault("GOMP_SPINCOUNT", "200000")env.setdefault("OMP_DYNAMIC", "FALSE")diff --git a/c/openai_server.py b/c/openai_server.pyindex c4787e2e..eff780d2 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1490,8 +1490,9 @@ def tune_child_env(env, arch):if arch != "deepseek_v4":return envif not env.get("COLI_NO_OMP_TUNE"):- from resource_plan import physical_cpu_count- env.setdefault("OMP_NUM_THREADS", str(physical_cpu_count()))+ # The V4 runtime owns OMP_NUM_THREADS: it reserves logical CPUs for its+ # expert-loader workers. Supplying a physical-core default here makes+ # that runtime policy treat the launcher value as a user override.env.setdefault("OMP_WAIT_POLICY", "active")env.setdefault("GOMP_SPINCOUNT", "200000")env.setdefault("OMP_DYNAMIC", "FALSE")diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.pyindex bd48d0d7..655713e4 100644--- a/c/tests/test_cli_output.py+++ b/c/tests/test_cli_output.py@@ -265,12 +265,11 @@ class BannerModelLineTest(unittest.TestCase):class OmpThreadsForEveryEngineTest(unittest.TestCase):- """#805 set OMP_NUM_THREADS from physical cores -- for glm only.+ """Launchers size shared engines; V4 delegates to its loader-aware runtime.- env_for_engine() forwarded to env_for() when arch was "glm" and built its- own environment otherwise, so inkling, kimi_k3, olmoe and deepseek_v4 kept- libgomp's nproc default: logical cores, a 2x over-subscription of a- memory-bound int4 GEMV on any SMT host.+ #805's physical-core default still covers the memory-bound sister engines.+ DeepSeek V4 instead reserves logical CPUs for its expert-loader workers in+ v4_omp_reserve_loader_cpus(), unless the operator overrides or disables it."""@classmethod@@ -284,33 +283,43 @@ class OmpThreadsForEveryEngineTest(unittest.TestCase):return types.SimpleNamespace(model="/x", ram=None, ctx=None, ngen=None,temp=None, cap=None)- def test_every_engine_gets_physical_cores(self):+ def test_other_engines_get_physical_cores(self):with mock.patch("resource_plan.physical_cpu_count", return_value=6):- for arch in ("inkling", "kimi", "olmoe", "deepseek_v4"):+ for arch in ("inkling", "kimi", "olmoe"):with self.subTest(arch=arch):env = self.coli.env_for_engine(self.args(), arch)self.assertEqual(env.get("OMP_NUM_THREADS"), "6")+ def test_v4_delegates_thread_team_to_runtime(self):+ with mock.patch.dict(os.environ, {}, clear=True), \+ mock.patch("resource_plan.physical_cpu_count",+ side_effect=AssertionError("V4 launcher sized the team")):+ env = self.coli.env_for_engine(self.args(), "deepseek_v4")+ self.assertNotIn("OMP_NUM_THREADS", env)+def test_v4_gets_memory_bound_affinity_defaults(self):- with mock.patch.object(self.coli.sys, "platform", "linux"), \- mock.patch("resource_plan.physical_cpu_count", return_value=6):+ with mock.patch.dict(os.environ, {}, clear=True), \+ mock.patch.object(self.coli.sys, "platform", "linux"):env = self.coli.env_for_engine(self.args(), "deepseek_v4")self.assertEqual(env.get("OMP_PROC_BIND"), "close")self.assertEqual(env.get("OMP_PLACES"), "cores")self.assertEqual(env.get("OMP_WAIT_POLICY"), "active")+ self.assertEqual(env.get("GOMP_SPINCOUNT"), "200000")self.assertEqual(env.get("OMP_DYNAMIC"), "FALSE")def test_explicit_setting_still_wins(self):- with mock.patch.dict(os.environ, {"OMP_NUM_THREADS": "3"}), \- mock.patch("resource_plan.physical_cpu_count", return_value=6):+ with mock.patch.dict(os.environ, {"OMP_NUM_THREADS": "3"}, clear=True), \+ mock.patch("resource_plan.physical_cpu_count",+ side_effect=AssertionError("V4 launcher sized the team")):env = self.coli.env_for_engine(self.args(), "deepseek_v4")self.assertEqual(env["OMP_NUM_THREADS"], "3")def test_kill_switch_is_honoured(self):- with mock.patch.dict(os.environ, {"COLI_NO_OMP_TUNE": "1"}, clear=False):- os.environ.pop("OMP_NUM_THREADS", None)+ with mock.patch.dict(os.environ, {"COLI_NO_OMP_TUNE": "1"}, clear=True):env = self.coli.env_for_engine(self.args(), "deepseek_v4")- self.assertNotIn("OMP_NUM_THREADS", env)+ for key in ("OMP_NUM_THREADS", "OMP_WAIT_POLICY", "GOMP_SPINCOUNT",+ "OMP_DYNAMIC", "OMP_PROC_BIND", "OMP_PLACES"):+ self.assertNotIn(key, env)if __name__ == "__main__":diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 5e2836b2..27d7495f 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -708,10 +708,11 @@ class CapSentinelShimTest(unittest.TestCase):def test_direct_v4_server_gets_bounded_dspark_defaults(self):env = {"V4_MTP_CONF": "0.7"}- with patch("resource_plan.physical_cpu_count", return_value=6), \+ with patch("resource_plan.physical_cpu_count",+ side_effect=AssertionError("V4 server sized the team")), \patch("openai_server.sys.platform", "linux"):tune_child_env(env, "deepseek_v4")- self.assertEqual(env["OMP_NUM_THREADS"], "6")+ self.assertNotIn("OMP_NUM_THREADS", env)self.assertEqual(env["OMP_PROC_BIND"], "close")self.assertEqual(env["V4_DRAFT"], "0")self.assertEqual(env["V4_MTP"], "0")@@ -719,6 +720,18 @@ class CapSentinelShimTest(unittest.TestCase):self.assertEqual(env["V4_MTP_GB"], "0.45")self.assertEqual(env["V4_MTP_CONF"], "0.7") # explicit override wins+ def test_direct_v4_server_preserves_explicit_omp_threads(self):+ env = {"OMP_NUM_THREADS": "3"}+ tune_child_env(env, "deepseek_v4")+ self.assertEqual(env["OMP_NUM_THREADS"], "3")++ def test_direct_v4_server_honours_omp_kill_switch(self):+ env = {"COLI_NO_OMP_TUNE": "1"}+ tune_child_env(env, "deepseek_v4")+ for key in ("OMP_NUM_THREADS", "OMP_WAIT_POLICY", "GOMP_SPINCOUNT",+ "OMP_DYNAMIC", "OMP_PROC_BIND", "OMP_PLACES"):+ self.assertNotIn(key, env)+class HTTPTest(unittest.TestCase):@classmethod