Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
openai_server.py/coli: key the cap shim on model arch; --cap absent means absent (#379)
monotophic1d3862fb44a5
diff --git a/c/coli b/c/coliindex ddcf292e..43a0ef48 100755--- a/c/coli+++ b/c/coli@@ -606,7 +606,7 @@ def cmd_run(a):# <think> open so the engine emits its reasoning block; the default stays nothink.tk="<think>" if os.environ.get("THINK","0")=="1" else "<think></think>"e=env_for(a); e["PROMPT"]=f"[gMASK]<sop><|user|>{prompt}<|assistant|>{tk}"- sys.exit(subprocess.call([GLM, str(a.cap)], env=e))+ sys.exit(subprocess.call([GLM, str(a.cap if a.cap is not None else 0)], env=e))def server_probe(base, api_key=None, timeout=1.5):"""Is a coli serve alive at `base`? Returns its model_id, or None.@@ -741,12 +741,14 @@ def cmd_chat(a):banner(f"chat · {model_id} · local server")import openai_server# --cap rides along so an explicit `coli chat --cap N` reaches the- # engine; the 0-default sentinel is translated per engine inside- # openai_server (cap_for_engine, #379: non-glm engines get legacy 8).+ # engine (it was silently eaten for years -- disclosed behavior+ # change); absent stays absent, and openai_server translates the+ # missing flag per model arch (cap_for_arch, #379: non-glm gets the+ # legacy 8, an explicit value -- 0 included -- passes verbatim).cmd=[sys.executable,openai_server.__file__,"--model",a.model,"--engine",engine,"--arch",arch,"--model-id",model_id,- "--cap",str(a.cap),"--host","127.0.0.1","--port","8000","--max-tokens",str(a.ngen)]+ if a.cap is not None: cmd+=["--cap",str(a.cap)]if a.api_key: cmd+=["--api-key",a.api_key]p=subprocess.Popen(cmd,env=env_for_engine(a,arch))try:@@ -779,7 +781,7 @@ def cmd_chat(a):# once the READY sentinel arrives, so the status-line display below works# exactly as before. (Do NOT add a concurrent stderr drain thread: on# Windows, reading two child pipes simultaneously deadlocks CPython's IO.)- p=subprocess.Popen([GLM,str(a.cap)], env=e, stdin=subprocess.PIPE,+ p=subprocess.Popen([GLM,str(a.cap if a.cap is not None else 0)], env=e, stdin=subprocess.PIPE,stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0)sp=Spinner("waking the giant (744B)…"); sp.start()st=stream_turn(p, READY, lambda b: None)@@ -1054,12 +1056,12 @@ def main():default=os.environ.get("COLI_POLICY","quality"),help="resource policy (explicit --topk/--topp overrides warn and proceed)")common.add_argument("--repin", type=int, default=0, help="adapt RAM/VRAM experts every N tokens")- # 0 = not explicitly set: the glm engine picks (8 historically; 1 on Metal+darwin+- # fast SSD, #379 -- an honest F_NOCACHE probe measures "fast", cached in- # <model>/.coli_ssd), while non-glm engines get the legacy 8 (the 0 sentinel means- # "RAM-auto" to inkling, not "engine default" -- openai_server.cap_for_engine).- # An explicit --cap always wins; see colibri.c's coli_resolve_cap().- common.add_argument("--cap", type=int, default=0, help="cache slots/layer (0=auto)")+ # Absent (None) = not explicitly set: the glm engine gets the 0 sentinel and picks+ # (8 historically; 1 on Metal+darwin+fast SSD, #379 -- an honest F_NOCACHE probe+ # measures "fast", cached in <model>/.coli_ssd), while non-glm arches get the+ # legacy 8 (openai_server.cap_for_arch). An explicit --cap N -- 0 included --+ # reaches the engine verbatim; see colibri.c's coli_resolve_cap().+ common.add_argument("--cap", type=int, default=None, help="cache slots/layer (default: auto)")common.add_argument("--ngen", type=int, default=1024) # rete di sicurezza: la fine vera la decidono gli stop tokencommon.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0)common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95)diff --git a/c/openai_server.py b/c/openai_server.pyindex b717d5d9..e0493d40 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1151,46 +1151,59 @@ def read_engine_turn(stream, sentinel, on_bytes):}-# Engines whose cap argv understands the "0 = platform-auto" sentinel-# (colibri.c coli_resolve_cap resolves it Metal/darwin/SSD-aware, #379).-_SENTINEL_CAP_ENGINES = ("colibri", "glm")---def cap_for_engine(executable, cap):+def model_arch(model):+ """The model's engine family from its config.json model_type -- the same+ rule as coli's model_arch(): "inkling"/"kimi" substring, everything else+ (including an unreadable config) is glm."""+ try:+ with open(Path(model) / "config.json", encoding="utf-8") as fh:+ model_type = (json.load(fh).get("model_type") or "").lower()+ except (OSError, ValueError, TypeError):+ return "glm"+ if "inkling" in model_type:+ return "inkling"+ if "kimi" in model_type:+ return "kimi"+ return "glm"+++def cap_for_arch(arch, cap):"""Cap-sentinel shim (#379): CURRENT-STATE CALIBRATION, not durable core.- cap 0 means different things across today's engines -- platform-auto in- colibri.c (coli_resolve_cap), RAM-auto in inkling.c (cap <= 0 fits the- expert LRU to available RAM), while the coli wrapper historically forced 8- on every engine. This shim INTERNALIZES that external inconsistency at the- one funnel every engine launch passes through: a non-glm engine receives- the legacy 8 when the user gave no explicit cap, an explicit --cap N- passes through to any engine, and the glm engine receives the 0 sentinel- to resolve platform-aware. Keyed on the engine binary's identity, not the- chat-template arch: COLI_ENGINE can route an inkling-arch model to the glm- binary, and it is the binary that interprets the argv.+ An absent cap (None) means different things across today's engines --+ platform-auto in colibri.c (coli_resolve_cap resolves the 0 sentinel+ Metal/darwin/SSD-aware), RAM-auto in inkling.c (cap <= 0 fits the expert+ LRU to available RAM), while the coli wrapper historically forced 8 on+ every engine. This shim INTERNALIZES that external inconsistency at the+ one funnel every engine launch passes through: with no explicit cap, a+ glm-arch model's engine receives the 0 sentinel to resolve platform-aware+ and a non-glm arch receives the legacy 8. An EXPLICIT cap passes through+ verbatim to any engine -- including an explicit 0, which for inkling means+ upstream's RAM-auto (people who ask for upstream semantics get them).+ Keyed on the MODEL's arch (config.json model_type), not the engine+ binary's file name: COLI_ENGINE users package the glm engine under+ arbitrary names (glm52, colibri-1.2, ...), and basename keying silently+ disabled the platform default for exactly them.MOOTING TRIGGER: upstream unifies cap-sentinel semantics across engines-> this shim must be removed and re-derived."""- if cap != 0:+ if cap is not None:return cap- name = Path(executable).name.lower()- if name.endswith(".exe"):- name = name[:-4]- return 0 if name in _SENTINEL_CAP_ENGINES else 8+ return 0 if arch == "glm" else 8class Engine:- # cap=0 = "not explicitly set": the glm engine resolves it (8 historically,- # 1 on Metal+darwin+fast SSD -- colibri.c coli_resolve_cap, #379), while- # non-glm engines get the legacy 8 via cap_for_engine above. Same sentinel- # as the --cap flags in coli and main() below, so programmatic callers that- # never pass cap get the same auto behavior as the CLI.- def __init__(self, executable, model, cap=0, max_tokens=1024, env=None, kv_slots=1):+ # cap=None = "not explicitly set": a glm-arch model's engine resolves the+ # 0 sentinel (8 historically, 1 on Metal+darwin+fast SSD -- colibri.c+ # coli_resolve_cap, #379), non-glm arches get the legacy 8, via+ # cap_for_arch above. Same convention as the --cap flags in coli and+ # main() below, so programmatic callers that never pass cap get the same+ # auto behavior as the CLI; an explicit int (0 included) is verbatim.+ def __init__(self, executable, model, cap=None, max_tokens=1024, env=None, kv_slots=1):child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))self.process = subprocess.Popen(- [str(executable), str(cap_for_engine(executable, cap))], env=child_env,+ [str(executable), str(cap_for_arch(model_arch(model), cap))], env=child_env,stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0,)self.write_lock = threading.Lock()@@ -2324,7 +2337,7 @@ class APIHandler(BaseHTTPRequestHandler):def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_key=None,- cap=0, max_tokens=1024, engine=None, env=None, cors_origins=None,+ cap=None, max_tokens=1024, engine=None, env=None, cors_origins=None,max_queue=8, queue_timeout=300, kv_slots=1, allowed_hosts=()):if engine is None:engine = default_engine()@@ -2383,9 +2396,10 @@ def main():parser.add_argument("--api-key", default=os.environ.get("COLI_API_KEY"))parser.add_argument("--cors-origin", action="append", default=None,help="allowed browser origin; repeat as needed (use '*' for any origin)")- # 0 = not explicitly set: mirrors coli's --cap (0=auto; see colibri.c coli_resolve_cap- # and issue #379 -- Metal+darwin+fast SSD defaults to 1, everything else to 8).- parser.add_argument("--cap", type=int, default=0, help="cache slots/layer (0=auto)")+ # Absent = not explicitly set: mirrors coli's --cap (see cap_for_arch and issue+ # #379 -- glm arch resolves platform-aware, non-glm gets the legacy 8). An+ # explicit value, 0 included, reaches the engine verbatim.+ parser.add_argument("--cap", type=int, default=None, help="cache slots/layer (default: auto)")parser.add_argument("--max-tokens", type=int, default=1024)parser.add_argument("--max-queue", type=int, default=int(os.environ.get("COLI_MAX_QUEUE", "8")))parser.add_argument("--queue-timeout", type=float,@@ -2400,13 +2414,7 @@ def main():global ARCHARCH = args.archif ARCH == "auto":- try:- with open(Path(args.model) / "config.json") as fh:- model_type = (json.load(fh).get("model_type") or "").lower()- ARCH = ("inkling" if "inkling" in model_type else- "kimi" if "kimi" in model_type else "glm")- except OSError:- ARCH = "glm"+ ARCH = model_arch(args.model)if args.model_id is None:args.model_id = ("inkling-colibri" if ARCH == "inkling" else"kimi-k3-colibri" if ARCH == "kimi" else "glm-5.2-colibri")diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.pyindex efefadf5..bc6dbec2 100644--- a/c/tests/test_cli_output.py+++ b/c/tests/test_cli_output.py@@ -1,8 +1,13 @@+import importlib.util+import jsonimport subprocessimport sysimport tempfile+import typesimport unittest+from importlib.machinery import SourceFileLoaderfrom pathlib import Path+from unittest import mockHERE = Path(__file__).resolve().parent.parent@@ -50,5 +55,77 @@ class CliOutputLanguageTest(unittest.TestCase):self.assertIn("set COLI_MODEL or use --model", result.stderr)+class ChatCapForwardingTest(unittest.TestCase):+ """#379/#386 r2 (F9): `coli chat` on a non-glm model spawns openai_server+ as its local server. An explicit --cap must ride along on that command+ line (it was silently eaten for years -- keeping it is a DISCLOSED+ behavior change: a long-ignored `coli chat --cap 32` now takes effect),+ and an absent --cap must stay absent so openai_server's arch-keyed+ default (cap_for_arch) applies. This drives the real cmd_chat with the+ process boundary faked, and pins the argv it builds."""++ @classmethod+ def setUpClass(cls):+ loader = SourceFileLoader("coli_cli_under_test", str(CLI))+ spec = importlib.util.spec_from_loader(loader.name, loader)+ cls.coli = importlib.util.module_from_spec(spec)+ loader.exec_module(cls.coli)++ def _chat_server_cmd(self, cap):+ coli = self.coli+ captured = {}++ class FakeProc:+ def __init__(self, cmd, **_kw):+ captured["cmd"] = cmd+ def poll(self):+ return None+ def terminate(self):+ pass+ def wait(self, timeout=None):+ return 0+ def kill(self):+ pass++ class FakeSpinner:+ def __init__(self, *_a, **_k):+ pass+ def start(self):+ pass+ def stop(self):+ pass++ model = tempfile.mkdtemp()+ self.addCleanup(lambda: subprocess.run(["rm", "-rf", model], check=False))+ (Path(model) / "config.json").write_text(json.dumps({"model_type": "inkling"}))+ args = types.SimpleNamespace(model=model, cap=cap, ngen=256, api_key=None,+ no_attach=True, attach=None)+ with mock.patch.object(coli, "need_model"), \+ mock.patch.object(coli, "banner"), \+ mock.patch.object(coli, "engine_for", return_value="/stub/inkling"), \+ mock.patch.object(coli, "env_for_engine", return_value={}), \+ mock.patch.object(coli, "server_probe", return_value="inkling-colibri"), \+ mock.patch.object(coli, "chat_attached"), \+ mock.patch.object(coli, "Spinner", FakeSpinner), \+ mock.patch("subprocess.Popen", FakeProc):+ coli.cmd_chat(args)+ return captured["cmd"]++ def test_explicit_cap_rides_along(self):+ cmd = self._chat_server_cmd(cap=32)+ self.assertIn("--cap", cmd)+ self.assertEqual(cmd[cmd.index("--cap") + 1], "32")+ self.assertEqual(cmd[cmd.index("--arch") + 1], "inkling")++ def test_explicit_cap_zero_rides_along(self):+ # explicit 0 = upstream RAM-auto for inkling, by request (#386 r2, F8)+ cmd = self._chat_server_cmd(cap=0)+ self.assertEqual(cmd[cmd.index("--cap") + 1], "0")++ def test_absent_cap_stays_absent(self):+ cmd = self._chat_server_cmd(cap=None)+ self.assertNotIn("--cap", cmd)++if __name__ == "__main__":unittest.main()diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 2fd7862a..77225a6e 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -14,7 +14,7 @@ from pathlib import Pathfrom openai_server import (APIError, APIHandler, APIServer, ClientCancelled,DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,READY, Engine, InklingStreamSplit, StopFilter, ThinkingStreamSplit,- _engine_error, cap_for_engine, conversation_cache_slot,+ _engine_error, cap_for_arch, conversation_cache_slot, model_arch,generation_options, parse_tool_calls, read_engine_turn,render_chat, render_chat_kimi, serve,split_thinking_reply, stop_policy)@@ -593,42 +593,81 @@ class DispatcherTest(unittest.TestCase):class CapSentinelShimTest(unittest.TestCase):- # #379 cap-sentinel shim: cap 0 is "platform-auto" only to the glm engine- # (colibri.c coli_resolve_cap); inkling reads cap <= 0 as "fit the expert- # LRU to all available RAM" (inkling.c), so leaking the sentinel to a- # non-glm engine silently changes its memory behavior. Engine() is the one- # funnel every launch passes through -- coli serve, coli chat's local- # server, openai_server.py --model, and programmatic callers -- so the- # translation is pinned HERE, at the argv it actually emits.- def _spawn_argv(self, executable, **kwargs):+ # #379 cap-sentinel shim, arch-keyed (#386 r2, F3): an absent cap is+ # "platform-auto" only for the glm engine (colibri.c coli_resolve_cap);+ # inkling reads cap <= 0 as "fit the expert LRU to all available RAM"+ # (inkling.c), so leaking the sentinel to a non-glm arch silently changes+ # its memory behavior. The key is the MODEL's config.json model_type, not+ # the engine binary's file name -- COLI_ENGINE users package the glm+ # engine as glm52/colibri-1.2/glm-metal, and basename keying disabled the+ # platform default for exactly them (and an inkling binary someone names+ # `glm` would get the leak back). Engine() is the one funnel every launch+ # passes through, so the translation is pinned at the argv it emits, over+ # the full matrix: arch x arbitrary executable name x cap absent/explicit.+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.addCleanup(self.tmp.cleanup)++ def _model(self, model_type):+ model = Path(self.tmp.name) / f"model-{model_type}"+ model.mkdir(exist_ok=True)+ (model / "config.json").write_text(json.dumps({"model_type": model_type}))+ return str(model)++ def _spawn_argv(self, executable, model, **kwargs):process = FakeProcess(lambda _process, _frame: None)with patch("openai_server.subprocess.Popen", return_value=process) as popen:- engine = Engine(executable, "model", **kwargs)+ engine = Engine(executable, model, **kwargs)engine.close()return popen.call_args[0][0]- def test_non_glm_engine_gets_legacy_8_for_the_sentinel(self):- for executable in ("inkling", "/opt/libexec/colibri/inkling",- "kimi_k3", "kimi_k3.exe", "olmoe"):- self.assertEqual(self._spawn_argv(executable), [executable, "8"],- f"sentinel leaked to {executable}")-- def test_glm_engine_keeps_the_platform_auto_sentinel(self):- for executable in ("glm", "glm.exe", "colibri", "/repo/c/colibri"):- self.assertEqual(self._spawn_argv(executable), [executable, "0"],- f"sentinel lost for {executable}")-- def test_explicit_cap_passes_through_to_any_engine(self):- self.assertEqual(self._spawn_argv("inkling", cap=5), ["inkling", "5"])- self.assertEqual(self._spawn_argv("colibri", cap=5), ["colibri", "5"])-- def test_cap_for_engine_is_the_single_translation_point(self):- self.assertEqual(cap_for_engine("inkling", 0), 8)- self.assertEqual(cap_for_engine("kimi_k3", 0), 8)- self.assertEqual(cap_for_engine("colibri", 0), 0)- self.assertEqual(cap_for_engine("glm", 0), 0)- self.assertEqual(cap_for_engine("inkling", 3), 3)- self.assertEqual(cap_for_engine("glm", 3), 3)+ def test_matrix_arch_times_engine_name_times_cap(self):+ # COLI_ENGINE axis: the executable name carries no information; only+ # the model arch and the explicitness of cap may matter.+ executables = ("colibri", "glm", "glm52", "colibri-1.2", "glm-metal",+ "/opt/custom/inkling", "kimi_k3.exe")+ cases = ( # (model_type, cap kwargs, expected argv cap)+ ("glm_moe_dsa", {}, "0"), # absent -> glm sentinel+ ("inkling", {}, "8"), # absent -> legacy 8+ ("kimi_k3", {}, "8"), # absent -> legacy 8+ ("glm_moe_dsa", {"cap": 5}, "5"), # explicit -> verbatim+ ("inkling", {"cap": 5}, "5"),+ ("kimi_k3", {"cap": 5}, "5"),+ ("glm_moe_dsa", {"cap": 0}, "0"), # explicit 0 -> verbatim+ ("inkling", {"cap": 0}, "0"), # upstream RAM-auto, by request+ ("kimi_k3", {"cap": 0}, "0"),+ )+ for model_type, kwargs, want in cases:+ model = self._model(model_type)+ for executable in executables:+ self.assertEqual(+ self._spawn_argv(executable, model, **kwargs),+ [executable, want],+ f"arch={model_type} exe={executable} kwargs={kwargs}")++ def test_missing_or_unreadable_config_is_glm(self):+ # historic default: anything that cannot be classified is glm+ self.assertEqual(self._spawn_argv("engine", "/nonexistent/model"),+ ["engine", "0"])+ model = Path(self.tmp.name) / "model-broken"+ model.mkdir()+ (model / "config.json").write_text("{not json")+ self.assertEqual(self._spawn_argv("engine", str(model)), ["engine", "0"])++ def test_cap_for_arch_is_the_single_translation_point(self):+ self.assertEqual(cap_for_arch("glm", None), 0)+ self.assertEqual(cap_for_arch("inkling", None), 8)+ self.assertEqual(cap_for_arch("kimi", None), 8)+ self.assertEqual(cap_for_arch("glm", 3), 3)+ self.assertEqual(cap_for_arch("inkling", 3), 3)+ self.assertEqual(cap_for_arch("inkling", 0), 0) # explicit 0 is explicit+ self.assertEqual(cap_for_arch("glm", 0), 0)++ def test_model_arch_reads_model_type(self):+ self.assertEqual(model_arch(self._model("glm_moe_dsa")), "glm")+ self.assertEqual(model_arch(self._model("inkling")), "inkling")+ self.assertEqual(model_arch(self._model("kimi_k3")), "kimi")+ self.assertEqual(model_arch("/nonexistent"), "glm")class HTTPTest(unittest.TestCase):