Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

chore: apply the v1.6.2 warning-cleanup patches from #1032

ZacharyZcR863bf1ea367a
diff --git a/c/Makefile b/c/Makefile
index fb8c6722..461940e9 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -650,8 +650,15 @@ deepseek-v4-tiny-generate:
@echo "SKIP deepseek-v4-tiny-generate: V4 runtime requires x86-64/aarch64 Linux or Windows/MSYS2"
endif
-# phony targets β€” 'glm' kept for backward compatibility
+# Convenience aliases β€” 'glm' kept for backward compatibility.
+# On POSIX EXE is empty, so `colibri: colibri$(EXE)` becomes a self-dependency
+# and marking it phony also forces the real binary to rebuild on every invocation.
+# Only create the alias when the platform executable suffix makes it distinct.
+ifneq ($(EXE),)
+.PHONY: colibri
colibri: colibri$(EXE)
+endif
+.PHONY: glm
glm: colibri$(EXE)
# Config stamp: make only tracks file timestamps, not flag changes. Without this,
@@ -1417,7 +1424,7 @@ clean:
bench: iobench$(EXE)
@if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) β€” run: ./iobench$(EXE) <file> <MB> <iters> <threads> <direct 0|1>"; fi
-.PHONY: all colibri glm deepseek-v4 deepseek-v4-clean deepseek-v4-oracle \
+.PHONY: all deepseek-v4 deepseek-v4-clean deepseek-v4-oracle \
deepseek-v4-tiny-generate deepseek-v4-tiny-check \
iq3 rans fuzz-rans cuda-test hip-test gpu-compile cuda-bench fp8-bench cuda-dll hip-dll portable \
test-c test-python test check clean install uninstall bench
diff --git a/c/colibri.c b/c/colibri.c
index 9668be55..fb44283f 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -492,6 +492,37 @@ typedef struct {
} Model;
#include "quant.h"
+
+/* Runtime policy knobs used only by the GLM engine. Keep them here rather
+ * than in quant.h: that header is shared by standalone kernel tests and
+ * sibling engines, where translation-unit-local copies are unused and trip
+ * -Wunused-variable. */
+static int g_idot=1;
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
+static int g_i4s=1;
+#elif defined(__VSX__)
+static int g_i4s=1;
+#elif defined(__AVX512VNNI__) && defined(__AVX512BW__)
+static int g_i4s=1; /* AVX-512 VNNI: come SDOT, l'IDOT int4 conviene anche a S=1. Misurato su
+ * 2x Xeon 8370C (48 core, GLM-5.2 int4 tutto residente, TEMP=0 DRAFT=0,
+ * 256 token): 3.65 -> 3.85 tok/s (+5.5%), expert-matmul 67.8 -> 89.5 GB/s.
+ * EN: with AVX-512 VNNI, like SDOT, int4 IDOT pays at S=1 too. Measured on
+ * a 2-socket Ice Lake (config above): +5.5% end-to-end greedy decode. */
+#else
+static int g_i4s=2;
+#endif
+static int g_xexp=0; /* XEXP=1 (opt-in): S==1 decode, all-resident int4 block -> ONE OpenMP
+ * region across all experts of the batch-union block instead of ~2
+ * fork/joins per expert. Engages only with the int4-IDOT S=1 family
+ * (g_i4s<=1) and off the speculation window (spec_pinned): output is
+ * byte-identical to that family (same dot_i4i8 per row, same silu,
+ * same requant, same accumulation order into out). Measured on a
+ * 2-socket Ice Lake 48C (GLM-5.2 int4 fully resident, TEMP=0 DRAFT=0,
+ * 256 tok greedy, ABAB 3 prompts x 2 reps): 4.20 -> 4.68 tok/s
+ * (+11.6% mean, worst prompt +11.3%), expert-matmul effective
+ * 89.5 -> 131.9 GB/s. A similar restructuring was NEUTRAL/negative on
+ * a 24-core box (docs/experiments/glm52-6x5090-2026-07-12.md) - hence
+ * opt-in; measure on your host. */
static int g_no_fused_pair=0;
static int g_spec_pin=1;
static int g_spec_live=0;
diff --git a/c/quant.h b/c/quant.h
index 85eb7dc3..31ac13a1 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -558,32 +558,6 @@ static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float
#else
#define IDOT_KERNEL "scalar"
#endif
-static int g_idot=1;
-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
-static int g_i4s=1;
-#elif defined(__VSX__)
-static int g_i4s=1;
-#elif defined(__AVX512VNNI__) && defined(__AVX512BW__)
-static int g_i4s=1; /* AVX-512 VNNI: come SDOT, l'IDOT int4 conviene anche a S=1. Misurato su
- * 2x Xeon 8370C (48 core, GLM-5.2 int4 tutto residente, TEMP=0 DRAFT=0,
- * 256 token): 3.65 -> 3.85 tok/s (+5.5%), expert-matmul 67.8 -> 89.5 GB/s.
- * EN: with AVX-512 VNNI, like SDOT, int4 IDOT pays at S=1 too. Measured on
- * a 2-socket Ice Lake (config above): +5.5% end-to-end greedy decode. */
-#else
-static int g_i4s=2;
-#endif
-static int g_xexp=0; /* XEXP=1 (opt-in): S==1 decode, all-resident int4 block -> ONE OpenMP
- * region across all experts of the batch-union block instead of ~2
- * fork/joins per expert. Engages only with the int4-IDOT S=1 family
- * (g_i4s<=1) and off the speculation window (spec_pinned): output is
- * byte-identical to that family (same dot_i4i8 per row, same silu,
- * same requant, same accumulation order into out). Measured on a
- * 2-socket Ice Lake 48C (GLM-5.2 int4 fully resident, TEMP=0 DRAFT=0,
- * 256 tok greedy, ABAB 3 prompts x 2 reps): 4.20 -> 4.68 tok/s
- * (+11.6% mean, worst prompt +11.3%), expert-matmul effective
- * 89.5 -> 131.9 GB/s. A similar restructuring was NEUTRAL/negative on
- * a 24-core box (docs/experiments/glm52-6x5090-2026-07-12.md) - hence
- * opt-in; measure on your host. */
static inline float qrow_i8(const float *x, int8_t *q, int I){
float amax=0; for(int i=0;i<I;i++){ float a=fabsf(x[i]); if(a>amax)amax=a; }
diff --git a/c/tests/test_anthropic_messages.py b/c/tests/test_anthropic_messages.py
index 18f5a607..46320c8d 100644
--- a/c/tests/test_anthropic_messages.py
+++ b/c/tests/test_anthropic_messages.py
@@ -174,6 +174,7 @@ class MessagesHTTPTest(unittest.TestCase):
before = len(self.engine.prompts)
with self.assertRaises(HTTPError) as caught:
self.post(body)
+ self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
self.assertIn("tool", json.load(caught.exception)["error"]["message"].lower())
self.assertEqual(len(self.engine.prompts), before)
@@ -197,11 +198,13 @@ class MessagesHTTPTest(unittest.TestCase):
self.assertEqual(response.status, 200)
with self.assertRaises(HTTPError) as caught:
self.post(self.base_body(), {"x-api-key": "wrong"})
+ self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 401)
def test_error_envelope_is_anthropic_shaped(self):
with self.assertRaises(HTTPError) as caught:
self.post({"model": "test-model", "messages": [{"role": "user", "content": "x"}]})
+ self.addCleanup(caught.exception.close)
payload = json.load(caught.exception)
self.assertEqual(payload["type"], "error")
self.assertEqual(payload["error"]["type"], "invalid_request_error")
@@ -403,6 +406,7 @@ class MessagesHTTPTest(unittest.TestCase):
for field, value in (("stop_sequences", ["STOP"]), ("top_k", 40)):
with self.assertRaises(HTTPError) as caught:
self.post(self.base_body(**{field: value}))
+ self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
self.assertIn(field, json.load(caught.exception)["error"]["message"])
diff --git a/c/tests/test_env_defaults.py b/c/tests/test_env_defaults.py
index 2a8cfb37..1562ea51 100644
--- a/c/tests/test_env_defaults.py
+++ b/c/tests/test_env_defaults.py
@@ -48,7 +48,8 @@ class EnvDefaultsTest(unittest.TestCase):
False keeps these tests independent of the host's GPU."""
with mock.patch.dict(os.environ, environ, clear=True), \
mock.patch.object(sys, "platform", platform), \
- mock.patch.object(coli, "cuda_binary", return_value=cuda):
+ mock.patch.object(coli, "cuda_binary", return_value=cuda), \
+ mock.patch("resource_plan.physical_cpu_count", return_value=8):
return coli.env_for(args())
def test_win32_sets_measured_defaults(self):
@@ -109,6 +110,7 @@ class CudaAutoEnableTest(unittest.TestCase):
mock.patch.object(sys, "platform", platform), \
mock.patch.object(coli, "cuda_binary", return_value=cuda), \
mock.patch.object(resource_plan, "discover_gpus", return_value=gpus), \
+ mock.patch.object(resource_plan, "physical_cpu_count", return_value=8), \
mock.patch.object(resource_plan, "build_plan", return_value=plan), \
mock.patch.object(resource_plan, "environment_for_plan",
side_effect=fake_environment_for_plan):
diff --git a/c/tests/test_k3_ram_budget.c b/c/tests/test_k3_ram_budget.c
index 1af34c3b..8fcfb0de 100644
--- a/c/tests/test_k3_ram_budget.c
+++ b/c/tests/test_k3_ram_budget.c
@@ -26,7 +26,7 @@ enum { NMOE = 93, N_EXPERTS = 384 };
static const double SLOT_GB = 17.5e-3; /* 17.5 MB per slot */
static const double RESIDENT = 35.2; /* RSS at the init line */
static const double KV_GB = 3.0; /* projected, K3_MAXT default */
-static const double RESERVE = 2.5 + 1.2 + 3.0;
+static const double RESERVE = 2.5 + 1.2 + KV_GB;
/* What v1.5.0 did: K3_EXPERT_GB straight into a cap, nothing subtracted. */
static int cap_v150(double egb){
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index c3caa8d1..1922b3bc 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -918,6 +918,7 @@ class HTTPTest(unittest.TestCase):
self.assertEqual(json.load(response)["data"][0]["id"], "test-model")
with self.assertRaises(HTTPError) as caught:
self.request("/v1/models", key="wrong")
+ self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 401)
def test_health_reports_scheduler_and_kv_slots(self):
@@ -1028,6 +1029,7 @@ class HTTPTest(unittest.TestCase):
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"stop": "H", "x_colibri_ignore_leading_stop": "yes",
})
+ self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
def test_rejects_invalid_cache_slot(self):
@@ -1036,6 +1038,7 @@ class HTTPTest(unittest.TestCase):
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"cache_slot": 2,
})
+ self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
def test_olmoe_never_files_the_answer_as_reasoning(self):
@@ -1107,6 +1110,7 @@ class HTTPTest(unittest.TestCase):
def test_rejects_empty_legacy_completion(self):
with self.assertRaises(HTTPError) as caught:
self.request("/v1/completions", {"model": "test-model", "prompt": ""})
+ self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
self.assertEqual(json.load(caught.exception)["error"]["param"], "prompt")
@@ -1116,6 +1120,7 @@ class HTTPTest(unittest.TestCase):
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
"stream": True, "stream_options": "usage",
})
+ self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400)
@@ -1192,7 +1197,8 @@ class ClientHangupTest(unittest.TestCase):
"the old except clause would have covered this; the test proves nothing")
with patch.object(APIHandler, "end_headers", self._abort):
try:
- urlopen(f"http://127.0.0.1:{self.server.server_port}/health", timeout=2)
+ with urlopen(f"http://127.0.0.1:{self.server.server_port}/health", timeout=2):
+ pass
except Exception:
pass # the client sees a broken response; that is fine
time.sleep(0.3)
@@ -1203,7 +1209,8 @@ class ClientHangupTest(unittest.TestCase):
"""Same as the hangup case: the damage is a lost handler, not the log."""
with patch.object(APIHandler, "end_headers", self._abort):
try:
- urlopen(f"http://127.0.0.1:{self.server.server_port}/health", timeout=2)
+ with urlopen(f"http://127.0.0.1:{self.server.server_port}/health", timeout=2):
+ pass
except Exception:
pass
time.sleep(0.3)
@@ -1242,6 +1249,7 @@ class StaticServingTest(unittest.TestCase):
self.assertEqual(response.read(), b"dashboard")
with self.assertRaises(HTTPError) as caught:
urlopen(self.base + "/%2e%2e/dist-private/secret.txt", timeout=2)
+ self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 404)
@@ -1277,6 +1285,7 @@ class SchedulerHTTPTest(unittest.TestCase):
self.assertTrue(self.engine.entered.wait(1))
with self.assertRaises(HTTPError) as caught:
self.request()
+ self.addCleanup(caught.exception.close)
error = json.loads(caught.exception.read())["error"]
self.assertEqual(caught.exception.code, 429)
self.assertEqual(caught.exception.headers["Retry-After"], "1")
@@ -1848,6 +1857,7 @@ class StreamingContextRejectTest(unittest.TestCase):
headers={"Content-Type": "application/json"})
with self.assertRaises(HTTPError) as caught:
urlopen(req, timeout=3)
+ self.addCleanup(caught.exception.close)
self.assertEqual(caught.exception.code, 400) # a real 400, not a 200 stream
body = json.load(caught.exception)
self.assertEqual(body["error"]["code"], "context_length_exceeded")
diff --git a/c/tests/test_openai_tools_e2e.py b/c/tests/test_openai_tools_e2e.py
index fada942a..54e730bc 100644
--- a/c/tests/test_openai_tools_e2e.py
+++ b/c/tests/test_openai_tools_e2e.py
@@ -98,7 +98,8 @@ class ToolCallingE2E(unittest.TestCase):
cls.base = f"http://127.0.0.1:{cls.port}/v1"
for _ in range(100):
try:
- urllib.request.urlopen(cls.base + "/models", timeout=2)
+ with urllib.request.urlopen(cls.base + "/models", timeout=2):
+ pass
return
except OSError:
if cls.server.poll() is not None:
@@ -117,15 +118,15 @@ class ToolCallingE2E(unittest.TestCase):
req = urllib.request.Request(
self.base + "/chat/completions", json.dumps(body).encode(),
{"Content-Type": "application/json"})
- resp = urllib.request.urlopen(req, timeout=30)
- if not stream:
- return json.loads(resp.read())
- events = []
- for raw in resp:
- line = raw.decode().strip()
- if line.startswith("data: ") and line != "data: [DONE]":
- events.append(json.loads(line[6:]))
- return events
+ with urllib.request.urlopen(req, timeout=30) as resp:
+ if not stream:
+ return json.loads(resp.read())
+ events = []
+ for raw in resp:
+ line = raw.decode().strip()
+ if line.startswith("data: ") and line != "data: [DONE]":
+ events.append(json.loads(line[6:]))
+ return events
def test_tool_call_non_stream(self):
r = self.post({"model": MODEL_ID, "tools": TOOLS,
diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.py
index 4645a14b..ac0133fd 100644
--- a/c/tests/test_v4_cli.py
+++ b/c/tests/test_v4_cli.py
@@ -183,6 +183,7 @@ class V4CliTest(unittest.TestCase):
return_value="deepseek_v4.exe"), \
mock.patch.object(self.cli, "need_model"), \
mock.patch.object(self.cli, "banner"), \
+ mock.patch("resource_plan.physical_cpu_count", return_value=8), \
mock.patch.object(self.cli.subprocess, "call",
side_effect=fake_call):
with self.assertRaises(SystemExit) as stopped: