Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
openai_server.py/coli: keep the cap-0 sentinel away from non-glm engines (#379)
monotophicb9114f4ca75c
diff --git a/c/coli b/c/coliindex c9e297dc..ddcf292e 100755--- a/c/coli+++ b/c/coli@@ -740,8 +740,12 @@ def cmd_chat(a):model_id="kimi-k3-colibri" if arch=="kimi" else "inkling-colibri"banner(f"chat · {model_id} · local server")import openai_server+ # --cap rides along so an explicit `coli chat --cap N` reaches the+ # engine; the 0-default sentinel is translated per engine inside+ # openai_server (cap_for_engine, #379: non-glm engines get legacy 8).cmd=[sys.executable,openai_server.__file__,"--model",a.model,"--engine",engine,"--arch",arch,"--model-id",model_id,+ "--cap",str(a.cap),"--host","127.0.0.1","--port","8000","--max-tokens",str(a.ngen)]if a.api_key: cmd+=["--api-key",a.api_key]p=subprocess.Popen(cmd,env=env_for_engine(a,arch))@@ -1050,8 +1054,10 @@ def main():default=os.environ.get("COLI_POLICY","quality"),help="resource policy (explicit --topk/--topp overrides warn and proceed)")common.add_argument("--repin", type=int, default=0, help="adapt RAM/VRAM experts every N tokens")- # 0 = not explicitly set: the engine picks (8 historically; 1 on Metal+darwin+fast- # SSD, #379 -- an honest F_NOCACHE probe measures "fast", cached in <model>/.coli_ssd).+ # 0 = not explicitly set: the glm engine picks (8 historically; 1 on Metal+darwin++ # fast SSD, #379 -- an honest F_NOCACHE probe measures "fast", cached in+ # <model>/.coli_ssd), while non-glm engines get the legacy 8 (the 0 sentinel means+ # "RAM-auto" to inkling, not "engine default" -- openai_server.cap_for_engine).# An explicit --cap always wins; see colibri.c's coli_resolve_cap().common.add_argument("--cap", type=int, default=0, help="cache slots/layer (0=auto)")common.add_argument("--ngen", type=int, default=1024) # rete di sicurezza: la fine vera la decidono gli stop tokendiff --git a/c/openai_server.py b/c/openai_server.pyindex 2e1ac420..b717d5d9 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1151,17 +1151,47 @@ def read_engine_turn(stream, sentinel, on_bytes):}+# Engines whose cap argv understands the "0 = platform-auto" sentinel+# (colibri.c coli_resolve_cap resolves it Metal/darwin/SSD-aware, #379).+_SENTINEL_CAP_ENGINES = ("colibri", "glm")+++def cap_for_engine(executable, cap):+ """Cap-sentinel shim (#379): CURRENT-STATE CALIBRATION, not durable core.++ cap 0 means different things across today's engines -- platform-auto in+ colibri.c (coli_resolve_cap), RAM-auto in inkling.c (cap <= 0 fits the+ expert LRU to available RAM), while the coli wrapper historically forced 8+ on every engine. This shim INTERNALIZES that external inconsistency at the+ one funnel every engine launch passes through: a non-glm engine receives+ the legacy 8 when the user gave no explicit cap, an explicit --cap N+ passes through to any engine, and the glm engine receives the 0 sentinel+ to resolve platform-aware. Keyed on the engine binary's identity, not the+ chat-template arch: COLI_ENGINE can route an inkling-arch model to the glm+ binary, and it is the binary that interprets the argv.++ MOOTING TRIGGER: upstream unifies cap-sentinel semantics across engines+ -> this shim must be removed and re-derived."""+ if cap != 0:+ return cap+ name = Path(executable).name.lower()+ if name.endswith(".exe"):+ name = name[:-4]+ return 0 if name in _SENTINEL_CAP_ENGINES else 8++class Engine:- # cap=0 = "not explicitly set": the engine resolves it (8 historically, 1 on- # Metal+darwin+fast SSD -- colibri.c coli_resolve_cap, #379). Same sentinel as- # the --cap flags in coli and main() below, so programmatic callers that+ # cap=0 = "not explicitly set": the glm engine resolves it (8 historically,+ # 1 on Metal+darwin+fast SSD -- colibri.c coli_resolve_cap, #379), while+ # non-glm engines get the legacy 8 via cap_for_engine above. Same sentinel+ # as the --cap flags in coli and main() below, so programmatic callers that# never pass cap get the same auto behavior as the CLI.def __init__(self, executable, model, cap=0, max_tokens=1024, env=None, kv_slots=1):child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))self.process = subprocess.Popen(- [str(executable), str(cap)], env=child_env, stdin=subprocess.PIPE,- stdout=subprocess.PIPE, bufsize=0,+ [str(executable), str(cap_for_engine(executable, cap))], env=child_env,+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0,)self.write_lock = threading.Lock()self.pending_lock = threading.Lock()diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 68f883ff..2fd7862a 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -14,8 +14,8 @@ from pathlib import Pathfrom openai_server import (APIError, APIHandler, APIServer, ClientCancelled,DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,READY, Engine, InklingStreamSplit, StopFilter, ThinkingStreamSplit,- _engine_error, conversation_cache_slot, generation_options,- parse_tool_calls, read_engine_turn,+ _engine_error, cap_for_engine, conversation_cache_slot,+ generation_options, parse_tool_calls, read_engine_turn,render_chat, render_chat_kimi, serve,split_thinking_reply, stop_policy)@@ -592,6 +592,45 @@ class DispatcherTest(unittest.TestCase):self.assertEqual(process.writes[-1].split(), [b"STOP", request_id])+class CapSentinelShimTest(unittest.TestCase):+ # #379 cap-sentinel shim: cap 0 is "platform-auto" only to the glm engine+ # (colibri.c coli_resolve_cap); inkling reads cap <= 0 as "fit the expert+ # LRU to all available RAM" (inkling.c), so leaking the sentinel to a+ # non-glm engine silently changes its memory behavior. Engine() is the one+ # funnel every launch passes through -- coli serve, coli chat's local+ # server, openai_server.py --model, and programmatic callers -- so the+ # translation is pinned HERE, at the argv it actually emits.+ def _spawn_argv(self, executable, **kwargs):+ process = FakeProcess(lambda _process, _frame: None)+ with patch("openai_server.subprocess.Popen", return_value=process) as popen:+ engine = Engine(executable, "model", **kwargs)+ engine.close()+ return popen.call_args[0][0]++ def test_non_glm_engine_gets_legacy_8_for_the_sentinel(self):+ for executable in ("inkling", "/opt/libexec/colibri/inkling",+ "kimi_k3", "kimi_k3.exe", "olmoe"):+ self.assertEqual(self._spawn_argv(executable), [executable, "8"],+ f"sentinel leaked to {executable}")++ def test_glm_engine_keeps_the_platform_auto_sentinel(self):+ for executable in ("glm", "glm.exe", "colibri", "/repo/c/colibri"):+ self.assertEqual(self._spawn_argv(executable), [executable, "0"],+ f"sentinel lost for {executable}")++ def test_explicit_cap_passes_through_to_any_engine(self):+ self.assertEqual(self._spawn_argv("inkling", cap=5), ["inkling", "5"])+ self.assertEqual(self._spawn_argv("colibri", cap=5), ["colibri", "5"])++ def test_cap_for_engine_is_the_single_translation_point(self):+ self.assertEqual(cap_for_engine("inkling", 0), 8)+ self.assertEqual(cap_for_engine("kimi_k3", 0), 8)+ self.assertEqual(cap_for_engine("colibri", 0), 0)+ self.assertEqual(cap_for_engine("glm", 0), 0)+ self.assertEqual(cap_for_engine("inkling", 3), 3)+ self.assertEqual(cap_for_engine("glm", 3), 3)++class HTTPTest(unittest.TestCase):@classmethoddef setUpClass(cls):