Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

openai_server.py/coli: keep the cap-0 sentinel away from non-glm engines (#379)

monotophicb9114f4ca75c
diff --git a/c/coli b/c/coli
index c9e297dc..ddcf292e 100755
--- a/c/coli
+++ b/c/coli
@@ -740,8 +740,12 @@ def cmd_chat(a):
model_id="kimi-k3-colibri" if arch=="kimi" else "inkling-colibri"
banner(f"chat · {model_id} · local server")
import openai_server
+ # --cap rides along so an explicit `coli chat --cap N` reaches the
+ # engine; the 0-default sentinel is translated per engine inside
+ # openai_server (cap_for_engine, #379: non-glm engines get legacy 8).
cmd=[sys.executable,openai_server.__file__,"--model",a.model,
"--engine",engine,"--arch",arch,"--model-id",model_id,
+ "--cap",str(a.cap),
"--host","127.0.0.1","--port","8000","--max-tokens",str(a.ngen)]
if a.api_key: cmd+=["--api-key",a.api_key]
p=subprocess.Popen(cmd,env=env_for_engine(a,arch))
@@ -1050,8 +1054,10 @@ def main():
default=os.environ.get("COLI_POLICY","quality"),
help="resource policy (explicit --topk/--topp overrides warn and proceed)")
common.add_argument("--repin", type=int, default=0, help="adapt RAM/VRAM experts every N tokens")
- # 0 = not explicitly set: the engine picks (8 historically; 1 on Metal+darwin+fast
- # SSD, #379 -- an honest F_NOCACHE probe measures "fast", cached in <model>/.coli_ssd).
+ # 0 = not explicitly set: the glm engine picks (8 historically; 1 on Metal+darwin+
+ # fast SSD, #379 -- an honest F_NOCACHE probe measures "fast", cached in
+ # <model>/.coli_ssd), while non-glm engines get the legacy 8 (the 0 sentinel means
+ # "RAM-auto" to inkling, not "engine default" -- openai_server.cap_for_engine).
# An explicit --cap always wins; see colibri.c's coli_resolve_cap().
common.add_argument("--cap", type=int, default=0, help="cache slots/layer (0=auto)")
common.add_argument("--ngen", type=int, default=1024) # rete di sicurezza: la fine vera la decidono gli stop token
diff --git a/c/openai_server.py b/c/openai_server.py
index 2e1ac420..b717d5d9 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -1151,17 +1151,47 @@ def read_engine_turn(stream, sentinel, on_bytes):
}
+# Engines whose cap argv understands the "0 = platform-auto" sentinel
+# (colibri.c coli_resolve_cap resolves it Metal/darwin/SSD-aware, #379).
+_SENTINEL_CAP_ENGINES = ("colibri", "glm")
+
+
+def cap_for_engine(executable, cap):
+ """Cap-sentinel shim (#379): CURRENT-STATE CALIBRATION, not durable core.
+
+ cap 0 means different things across today's engines -- platform-auto in
+ colibri.c (coli_resolve_cap), RAM-auto in inkling.c (cap <= 0 fits the
+ expert LRU to available RAM), while the coli wrapper historically forced 8
+ on every engine. This shim INTERNALIZES that external inconsistency at the
+ one funnel every engine launch passes through: a non-glm engine receives
+ the legacy 8 when the user gave no explicit cap, an explicit --cap N
+ passes through to any engine, and the glm engine receives the 0 sentinel
+ to resolve platform-aware. Keyed on the engine binary's identity, not the
+ chat-template arch: COLI_ENGINE can route an inkling-arch model to the glm
+ binary, and it is the binary that interprets the argv.
+
+ MOOTING TRIGGER: upstream unifies cap-sentinel semantics across engines
+ -> this shim must be removed and re-derived."""
+ if cap != 0:
+ return cap
+ name = Path(executable).name.lower()
+ if name.endswith(".exe"):
+ name = name[:-4]
+ return 0 if name in _SENTINEL_CAP_ENGINES else 8
+
+
class Engine:
- # cap=0 = "not explicitly set": the engine resolves it (8 historically, 1 on
- # Metal+darwin+fast SSD -- colibri.c coli_resolve_cap, #379). Same sentinel as
- # the --cap flags in coli and main() below, so programmatic callers that
+ # cap=0 = "not explicitly set": the glm engine resolves it (8 historically,
+ # 1 on Metal+darwin+fast SSD -- colibri.c coli_resolve_cap, #379), while
+ # non-glm engines get the legacy 8 via cap_for_engine above. Same sentinel
+ # as the --cap flags in coli and main() below, so programmatic callers that
# never pass cap get the same auto behavior as the CLI.
def __init__(self, executable, model, cap=0, max_tokens=1024, env=None, kv_slots=1):
child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",
NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))
self.process = subprocess.Popen(
- [str(executable), str(cap)], env=child_env, stdin=subprocess.PIPE,
- stdout=subprocess.PIPE, bufsize=0,
+ [str(executable), str(cap_for_engine(executable, cap))], env=child_env,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0,
)
self.write_lock = threading.Lock()
self.pending_lock = threading.Lock()
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 68f883ff..2fd7862a 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -14,8 +14,8 @@ from pathlib import Path
from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,
DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,
READY, Engine, InklingStreamSplit, StopFilter, ThinkingStreamSplit,
- _engine_error, conversation_cache_slot, generation_options,
- parse_tool_calls, read_engine_turn,
+ _engine_error, cap_for_engine, conversation_cache_slot,
+ generation_options, parse_tool_calls, read_engine_turn,
render_chat, render_chat_kimi, serve,
split_thinking_reply, stop_policy)
@@ -592,6 +592,45 @@ class DispatcherTest(unittest.TestCase):
self.assertEqual(process.writes[-1].split(), [b"STOP", request_id])
+class CapSentinelShimTest(unittest.TestCase):
+ # #379 cap-sentinel shim: cap 0 is "platform-auto" only to the glm engine
+ # (colibri.c coli_resolve_cap); inkling reads cap <= 0 as "fit the expert
+ # LRU to all available RAM" (inkling.c), so leaking the sentinel to a
+ # non-glm engine silently changes its memory behavior. Engine() is the one
+ # funnel every launch passes through -- coli serve, coli chat's local
+ # server, openai_server.py --model, and programmatic callers -- so the
+ # translation is pinned HERE, at the argv it actually emits.
+ def _spawn_argv(self, executable, **kwargs):
+ process = FakeProcess(lambda _process, _frame: None)
+ with patch("openai_server.subprocess.Popen", return_value=process) as popen:
+ engine = Engine(executable, "model", **kwargs)
+ engine.close()
+ return popen.call_args[0][0]
+
+ def test_non_glm_engine_gets_legacy_8_for_the_sentinel(self):
+ for executable in ("inkling", "/opt/libexec/colibri/inkling",
+ "kimi_k3", "kimi_k3.exe", "olmoe"):
+ self.assertEqual(self._spawn_argv(executable), [executable, "8"],
+ f"sentinel leaked to {executable}")
+
+ def test_glm_engine_keeps_the_platform_auto_sentinel(self):
+ for executable in ("glm", "glm.exe", "colibri", "/repo/c/colibri"):
+ self.assertEqual(self._spawn_argv(executable), [executable, "0"],
+ f"sentinel lost for {executable}")
+
+ def test_explicit_cap_passes_through_to_any_engine(self):
+ self.assertEqual(self._spawn_argv("inkling", cap=5), ["inkling", "5"])
+ self.assertEqual(self._spawn_argv("colibri", cap=5), ["colibri", "5"])
+
+ def test_cap_for_engine_is_the_single_translation_point(self):
+ self.assertEqual(cap_for_engine("inkling", 0), 8)
+ self.assertEqual(cap_for_engine("kimi_k3", 0), 8)
+ self.assertEqual(cap_for_engine("colibri", 0), 0)
+ self.assertEqual(cap_for_engine("glm", 0), 0)
+ self.assertEqual(cap_for_engine("inkling", 3), 3)
+ self.assertEqual(cap_for_engine("glm", 3), 3)
+
+
class HTTPTest(unittest.TestCase):
@classmethod
def setUpClass(cls):