Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
chore: apply the v1.6.2 warning-cleanup patches from #1032
ZacharyZcR863bf1ea367a
diff --git a/c/Makefile b/c/Makefileindex fb8c6722..461940e9 100644--- a/c/Makefile+++ b/c/Makefile@@ -650,8 +650,15 @@ deepseek-v4-tiny-generate:@echo "SKIP deepseek-v4-tiny-generate: V4 runtime requires x86-64/aarch64 Linux or Windows/MSYS2"endif-# phony targets β 'glm' kept for backward compatibility+# Convenience aliases β 'glm' kept for backward compatibility.+# On POSIX EXE is empty, so `colibri: colibri$(EXE)` becomes a self-dependency+# and marking it phony also forces the real binary to rebuild on every invocation.+# Only create the alias when the platform executable suffix makes it distinct.+ifneq ($(EXE),)+.PHONY: colibricolibri: colibri$(EXE)+endif+.PHONY: glmglm: colibri$(EXE)# Config stamp: make only tracks file timestamps, not flag changes. Without this,@@ -1417,7 +1424,7 @@ clean:bench: iobench$(EXE)@if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) β run: ./iobench$(EXE) <file> <MB> <iters> <threads> <direct 0|1>"; fi-.PHONY: all colibri glm deepseek-v4 deepseek-v4-clean deepseek-v4-oracle \+.PHONY: all deepseek-v4 deepseek-v4-clean deepseek-v4-oracle \deepseek-v4-tiny-generate deepseek-v4-tiny-check \iq3 rans fuzz-rans cuda-test hip-test gpu-compile cuda-bench fp8-bench cuda-dll hip-dll portable \test-c test-python test check clean install uninstall benchdiff --git a/c/colibri.c b/c/colibri.cindex 9668be55..fb44283f 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -492,6 +492,37 @@ typedef struct {} Model;#include "quant.h"++/* Runtime policy knobs used only by the GLM engine. Keep them here rather+ * than in quant.h: that header is shared by standalone kernel tests and+ * sibling engines, where translation-unit-local copies are unused and trip+ * -Wunused-variable. */+static int g_idot=1;+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)+static int g_i4s=1;+#elif defined(__VSX__)+static int g_i4s=1;+#elif defined(__AVX512VNNI__) && defined(__AVX512BW__)+static int g_i4s=1; /* AVX-512 VNNI: come SDOT, l'IDOT int4 conviene anche a S=1. Misurato su+ * 2x Xeon 8370C (48 core, GLM-5.2 int4 tutto residente, TEMP=0 DRAFT=0,+ * 256 token): 3.65 -> 3.85 tok/s (+5.5%), expert-matmul 67.8 -> 89.5 GB/s.+ * EN: with AVX-512 VNNI, like SDOT, int4 IDOT pays at S=1 too. Measured on+ * a 2-socket Ice Lake (config above): +5.5% end-to-end greedy decode. */+#else+static int g_i4s=2;+#endif+static int g_xexp=0; /* XEXP=1 (opt-in): S==1 decode, all-resident int4 block -> ONE OpenMP+ * region across all experts of the batch-union block instead of ~2+ * fork/joins per expert. Engages only with the int4-IDOT S=1 family+ * (g_i4s<=1) and off the speculation window (spec_pinned): output is+ * byte-identical to that family (same dot_i4i8 per row, same silu,+ * same requant, same accumulation order into out). Measured on a+ * 2-socket Ice Lake 48C (GLM-5.2 int4 fully resident, TEMP=0 DRAFT=0,+ * 256 tok greedy, ABAB 3 prompts x 2 reps): 4.20 -> 4.68 tok/s+ * (+11.6% mean, worst prompt +11.3%), expert-matmul effective+ * 89.5 -> 131.9 GB/s. A similar restructuring was NEUTRAL/negative on+ * a 24-core box (docs/experiments/glm52-6x5090-2026-07-12.md) - hence+ * opt-in; measure on your host. */static int g_no_fused_pair=0;static int g_spec_pin=1;static int g_spec_live=0;diff --git a/c/quant.h b/c/quant.hindex 85eb7dc3..31ac13a1 100644--- a/c/quant.h+++ b/c/quant.h@@ -558,32 +558,6 @@ static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float#else#define IDOT_KERNEL "scalar"#endif-static int g_idot=1;-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)-static int g_i4s=1;-#elif defined(__VSX__)-static int g_i4s=1;-#elif defined(__AVX512VNNI__) && defined(__AVX512BW__)-static int g_i4s=1; /* AVX-512 VNNI: come SDOT, l'IDOT int4 conviene anche a S=1. Misurato su- * 2x Xeon 8370C (48 core, GLM-5.2 int4 tutto residente, TEMP=0 DRAFT=0,- * 256 token): 3.65 -> 3.85 tok/s (+5.5%), expert-matmul 67.8 -> 89.5 GB/s.- * EN: with AVX-512 VNNI, like SDOT, int4 IDOT pays at S=1 too. Measured on- * a 2-socket Ice Lake (config above): +5.5% end-to-end greedy decode. */-#else-static int g_i4s=2;-#endif-static int g_xexp=0; /* XEXP=1 (opt-in): S==1 decode, all-resident int4 block -> ONE OpenMP- * region across all experts of the batch-union block instead of ~2- * fork/joins per expert. Engages only with the int4-IDOT S=1 family- * (g_i4s<=1) and off the speculation window (spec_pinned): output is- * byte-identical to that family (same dot_i4i8 per row, same silu,- * same requant, same accumulation order into out). Measured on a- * 2-socket Ice Lake 48C (GLM-5.2 int4 fully resident, TEMP=0 DRAFT=0,- * 256 tok greedy, ABAB 3 prompts x 2 reps): 4.20 -> 4.68 tok/s- * (+11.6% mean, worst prompt +11.3%), expert-matmul effective- * 89.5 -> 131.9 GB/s. A similar restructuring was NEUTRAL/negative on- * a 24-core box (docs/experiments/glm52-6x5090-2026-07-12.md) - hence- * opt-in; measure on your host. */static inline float qrow_i8(const float *x, int8_t *q, int I){float amax=0; for(int i=0;i<I;i++){ float a=fabsf(x[i]); if(a>amax)amax=a; }diff --git a/c/tests/test_anthropic_messages.py b/c/tests/test_anthropic_messages.pyindex 18f5a607..46320c8d 100644--- a/c/tests/test_anthropic_messages.py+++ b/c/tests/test_anthropic_messages.py@@ -174,6 +174,7 @@ class MessagesHTTPTest(unittest.TestCase):before = len(self.engine.prompts)with self.assertRaises(HTTPError) as caught:self.post(body)+ self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 400)self.assertIn("tool", json.load(caught.exception)["error"]["message"].lower())self.assertEqual(len(self.engine.prompts), before)@@ -197,11 +198,13 @@ class MessagesHTTPTest(unittest.TestCase):self.assertEqual(response.status, 200)with self.assertRaises(HTTPError) as caught:self.post(self.base_body(), {"x-api-key": "wrong"})+ self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 401)def test_error_envelope_is_anthropic_shaped(self):with self.assertRaises(HTTPError) as caught:self.post({"model": "test-model", "messages": [{"role": "user", "content": "x"}]})+ self.addCleanup(caught.exception.close)payload = json.load(caught.exception)self.assertEqual(payload["type"], "error")self.assertEqual(payload["error"]["type"], "invalid_request_error")@@ -403,6 +406,7 @@ class MessagesHTTPTest(unittest.TestCase):for field, value in (("stop_sequences", ["STOP"]), ("top_k", 40)):with self.assertRaises(HTTPError) as caught:self.post(self.base_body(**{field: value}))+ self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 400)self.assertIn(field, json.load(caught.exception)["error"]["message"])diff --git a/c/tests/test_env_defaults.py b/c/tests/test_env_defaults.pyindex 2a8cfb37..1562ea51 100644--- a/c/tests/test_env_defaults.py+++ b/c/tests/test_env_defaults.py@@ -48,7 +48,8 @@ class EnvDefaultsTest(unittest.TestCase):False keeps these tests independent of the host's GPU."""with mock.patch.dict(os.environ, environ, clear=True), \mock.patch.object(sys, "platform", platform), \- mock.patch.object(coli, "cuda_binary", return_value=cuda):+ mock.patch.object(coli, "cuda_binary", return_value=cuda), \+ mock.patch("resource_plan.physical_cpu_count", return_value=8):return coli.env_for(args())def test_win32_sets_measured_defaults(self):@@ -109,6 +110,7 @@ class CudaAutoEnableTest(unittest.TestCase):mock.patch.object(sys, "platform", platform), \mock.patch.object(coli, "cuda_binary", return_value=cuda), \mock.patch.object(resource_plan, "discover_gpus", return_value=gpus), \+ mock.patch.object(resource_plan, "physical_cpu_count", return_value=8), \mock.patch.object(resource_plan, "build_plan", return_value=plan), \mock.patch.object(resource_plan, "environment_for_plan",side_effect=fake_environment_for_plan):diff --git a/c/tests/test_k3_ram_budget.c b/c/tests/test_k3_ram_budget.cindex 1af34c3b..8fcfb0de 100644--- a/c/tests/test_k3_ram_budget.c+++ b/c/tests/test_k3_ram_budget.c@@ -26,7 +26,7 @@ enum { NMOE = 93, N_EXPERTS = 384 };static const double SLOT_GB = 17.5e-3; /* 17.5 MB per slot */static const double RESIDENT = 35.2; /* RSS at the init line */static const double KV_GB = 3.0; /* projected, K3_MAXT default */-static const double RESERVE = 2.5 + 1.2 + 3.0;+static const double RESERVE = 2.5 + 1.2 + KV_GB;/* What v1.5.0 did: K3_EXPERT_GB straight into a cap, nothing subtracted. */static int cap_v150(double egb){diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex c3caa8d1..1922b3bc 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -918,6 +918,7 @@ class HTTPTest(unittest.TestCase):self.assertEqual(json.load(response)["data"][0]["id"], "test-model")with self.assertRaises(HTTPError) as caught:self.request("/v1/models", key="wrong")+ self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 401)def test_health_reports_scheduler_and_kv_slots(self):@@ -1028,6 +1029,7 @@ class HTTPTest(unittest.TestCase):"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],"stop": "H", "x_colibri_ignore_leading_stop": "yes",})+ self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 400)def test_rejects_invalid_cache_slot(self):@@ -1036,6 +1038,7 @@ class HTTPTest(unittest.TestCase):"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],"cache_slot": 2,})+ self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 400)def test_olmoe_never_files_the_answer_as_reasoning(self):@@ -1107,6 +1110,7 @@ class HTTPTest(unittest.TestCase):def test_rejects_empty_legacy_completion(self):with self.assertRaises(HTTPError) as caught:self.request("/v1/completions", {"model": "test-model", "prompt": ""})+ self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 400)self.assertEqual(json.load(caught.exception)["error"]["param"], "prompt")@@ -1116,6 +1120,7 @@ class HTTPTest(unittest.TestCase):"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],"stream": True, "stream_options": "usage",})+ self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 400)@@ -1192,7 +1197,8 @@ class ClientHangupTest(unittest.TestCase):"the old except clause would have covered this; the test proves nothing")with patch.object(APIHandler, "end_headers", self._abort):try:- urlopen(f"http://127.0.0.1:{self.server.server_port}/health", timeout=2)+ with urlopen(f"http://127.0.0.1:{self.server.server_port}/health", timeout=2):+ passexcept Exception:pass # the client sees a broken response; that is finetime.sleep(0.3)@@ -1203,7 +1209,8 @@ class ClientHangupTest(unittest.TestCase):"""Same as the hangup case: the damage is a lost handler, not the log."""with patch.object(APIHandler, "end_headers", self._abort):try:- urlopen(f"http://127.0.0.1:{self.server.server_port}/health", timeout=2)+ with urlopen(f"http://127.0.0.1:{self.server.server_port}/health", timeout=2):+ passexcept Exception:passtime.sleep(0.3)@@ -1242,6 +1249,7 @@ class StaticServingTest(unittest.TestCase):self.assertEqual(response.read(), b"dashboard")with self.assertRaises(HTTPError) as caught:urlopen(self.base + "/%2e%2e/dist-private/secret.txt", timeout=2)+ self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 404)@@ -1277,6 +1285,7 @@ class SchedulerHTTPTest(unittest.TestCase):self.assertTrue(self.engine.entered.wait(1))with self.assertRaises(HTTPError) as caught:self.request()+ self.addCleanup(caught.exception.close)error = json.loads(caught.exception.read())["error"]self.assertEqual(caught.exception.code, 429)self.assertEqual(caught.exception.headers["Retry-After"], "1")@@ -1848,6 +1857,7 @@ class StreamingContextRejectTest(unittest.TestCase):headers={"Content-Type": "application/json"})with self.assertRaises(HTTPError) as caught:urlopen(req, timeout=3)+ self.addCleanup(caught.exception.close)self.assertEqual(caught.exception.code, 400) # a real 400, not a 200 streambody = json.load(caught.exception)self.assertEqual(body["error"]["code"], "context_length_exceeded")diff --git a/c/tests/test_openai_tools_e2e.py b/c/tests/test_openai_tools_e2e.pyindex fada942a..54e730bc 100644--- a/c/tests/test_openai_tools_e2e.py+++ b/c/tests/test_openai_tools_e2e.py@@ -98,7 +98,8 @@ class ToolCallingE2E(unittest.TestCase):cls.base = f"http://127.0.0.1:{cls.port}/v1"for _ in range(100):try:- urllib.request.urlopen(cls.base + "/models", timeout=2)+ with urllib.request.urlopen(cls.base + "/models", timeout=2):+ passreturnexcept OSError:if cls.server.poll() is not None:@@ -117,15 +118,15 @@ class ToolCallingE2E(unittest.TestCase):req = urllib.request.Request(self.base + "/chat/completions", json.dumps(body).encode(),{"Content-Type": "application/json"})- resp = urllib.request.urlopen(req, timeout=30)- if not stream:- return json.loads(resp.read())- events = []- for raw in resp:- line = raw.decode().strip()- if line.startswith("data: ") and line != "data: [DONE]":- events.append(json.loads(line[6:]))- return events+ with urllib.request.urlopen(req, timeout=30) as resp:+ if not stream:+ return json.loads(resp.read())+ events = []+ for raw in resp:+ line = raw.decode().strip()+ if line.startswith("data: ") and line != "data: [DONE]":+ events.append(json.loads(line[6:]))+ return eventsdef test_tool_call_non_stream(self):r = self.post({"model": MODEL_ID, "tools": TOOLS,diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.pyindex 4645a14b..ac0133fd 100644--- a/c/tests/test_v4_cli.py+++ b/c/tests/test_v4_cli.py@@ -183,6 +183,7 @@ class V4CliTest(unittest.TestCase):return_value="deepseek_v4.exe"), \mock.patch.object(self.cli, "need_model"), \mock.patch.object(self.cli, "banner"), \+ mock.patch("resource_plan.physical_cpu_count", return_value=8), \mock.patch.object(self.cli.subprocess, "call",side_effect=fake_call):with self.assertRaises(SystemExit) as stopped: