Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

openai_server.py/coli: key the cap shim on model arch; --cap absent means absent (#379)

monotophic1d3862fb44a5
diff --git a/c/coli b/c/coli
index ddcf292e..43a0ef48 100755
--- a/c/coli
+++ b/c/coli
@@ -606,7 +606,7 @@ def cmd_run(a):
# <think> open so the engine emits its reasoning block; the default stays nothink.
tk="<think>" if os.environ.get("THINK","0")=="1" else "<think></think>"
e=env_for(a); e["PROMPT"]=f"[gMASK]<sop><|user|>{prompt}<|assistant|>{tk}"
- sys.exit(subprocess.call([GLM, str(a.cap)], env=e))
+ sys.exit(subprocess.call([GLM, str(a.cap if a.cap is not None else 0)], env=e))
def server_probe(base, api_key=None, timeout=1.5):
"""Is a coli serve alive at `base`? Returns its model_id, or None.
@@ -741,12 +741,14 @@ def cmd_chat(a):
banner(f"chat · {model_id} · local server")
import openai_server
# --cap rides along so an explicit `coli chat --cap N` reaches the
- # engine; the 0-default sentinel is translated per engine inside
- # openai_server (cap_for_engine, #379: non-glm engines get legacy 8).
+ # engine (it was silently eaten for years -- disclosed behavior
+ # change); absent stays absent, and openai_server translates the
+ # missing flag per model arch (cap_for_arch, #379: non-glm gets the
+ # legacy 8, an explicit value -- 0 included -- passes verbatim).
cmd=[sys.executable,openai_server.__file__,"--model",a.model,
"--engine",engine,"--arch",arch,"--model-id",model_id,
- "--cap",str(a.cap),
"--host","127.0.0.1","--port","8000","--max-tokens",str(a.ngen)]
+ if a.cap is not None: cmd+=["--cap",str(a.cap)]
if a.api_key: cmd+=["--api-key",a.api_key]
p=subprocess.Popen(cmd,env=env_for_engine(a,arch))
try:
@@ -779,7 +781,7 @@ def cmd_chat(a):
# once the READY sentinel arrives, so the status-line display below works
# exactly as before. (Do NOT add a concurrent stderr drain thread: on
# Windows, reading two child pipes simultaneously deadlocks CPython's IO.)
- p=subprocess.Popen([GLM,str(a.cap)], env=e, stdin=subprocess.PIPE,
+ p=subprocess.Popen([GLM,str(a.cap if a.cap is not None else 0)], env=e, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0)
sp=Spinner("waking the giant (744B)…"); sp.start()
st=stream_turn(p, READY, lambda b: None)
@@ -1054,12 +1056,12 @@ def main():
default=os.environ.get("COLI_POLICY","quality"),
help="resource policy (explicit --topk/--topp overrides warn and proceed)")
common.add_argument("--repin", type=int, default=0, help="adapt RAM/VRAM experts every N tokens")
- # 0 = not explicitly set: the glm engine picks (8 historically; 1 on Metal+darwin+
- # fast SSD, #379 -- an honest F_NOCACHE probe measures "fast", cached in
- # <model>/.coli_ssd), while non-glm engines get the legacy 8 (the 0 sentinel means
- # "RAM-auto" to inkling, not "engine default" -- openai_server.cap_for_engine).
- # An explicit --cap always wins; see colibri.c's coli_resolve_cap().
- common.add_argument("--cap", type=int, default=0, help="cache slots/layer (0=auto)")
+ # Absent (None) = not explicitly set: the glm engine gets the 0 sentinel and picks
+ # (8 historically; 1 on Metal+darwin+fast SSD, #379 -- an honest F_NOCACHE probe
+ # measures "fast", cached in <model>/.coli_ssd), while non-glm arches get the
+ # legacy 8 (openai_server.cap_for_arch). An explicit --cap N -- 0 included --
+ # reaches the engine verbatim; see colibri.c's coli_resolve_cap().
+ common.add_argument("--cap", type=int, default=None, help="cache slots/layer (default: auto)")
common.add_argument("--ngen", type=int, default=1024) # rete di sicurezza: la fine vera la decidono gli stop token
common.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0)
common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95)
diff --git a/c/openai_server.py b/c/openai_server.py
index b717d5d9..e0493d40 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -1151,46 +1151,59 @@ def read_engine_turn(stream, sentinel, on_bytes):
}
-# Engines whose cap argv understands the "0 = platform-auto" sentinel
-# (colibri.c coli_resolve_cap resolves it Metal/darwin/SSD-aware, #379).
-_SENTINEL_CAP_ENGINES = ("colibri", "glm")
-
-
-def cap_for_engine(executable, cap):
+def model_arch(model):
+ """The model's engine family from its config.json model_type -- the same
+ rule as coli's model_arch(): "inkling"/"kimi" substring, everything else
+ (including an unreadable config) is glm."""
+ try:
+ with open(Path(model) / "config.json", encoding="utf-8") as fh:
+ model_type = (json.load(fh).get("model_type") or "").lower()
+ except (OSError, ValueError, TypeError):
+ return "glm"
+ if "inkling" in model_type:
+ return "inkling"
+ if "kimi" in model_type:
+ return "kimi"
+ return "glm"
+
+
+def cap_for_arch(arch, cap):
"""Cap-sentinel shim (#379): CURRENT-STATE CALIBRATION, not durable core.
- cap 0 means different things across today's engines -- platform-auto in
- colibri.c (coli_resolve_cap), RAM-auto in inkling.c (cap <= 0 fits the
- expert LRU to available RAM), while the coli wrapper historically forced 8
- on every engine. This shim INTERNALIZES that external inconsistency at the
- one funnel every engine launch passes through: a non-glm engine receives
- the legacy 8 when the user gave no explicit cap, an explicit --cap N
- passes through to any engine, and the glm engine receives the 0 sentinel
- to resolve platform-aware. Keyed on the engine binary's identity, not the
- chat-template arch: COLI_ENGINE can route an inkling-arch model to the glm
- binary, and it is the binary that interprets the argv.
+ An absent cap (None) means different things across today's engines --
+ platform-auto in colibri.c (coli_resolve_cap resolves the 0 sentinel
+ Metal/darwin/SSD-aware), RAM-auto in inkling.c (cap <= 0 fits the expert
+ LRU to available RAM), while the coli wrapper historically forced 8 on
+ every engine. This shim INTERNALIZES that external inconsistency at the
+ one funnel every engine launch passes through: with no explicit cap, a
+ glm-arch model's engine receives the 0 sentinel to resolve platform-aware
+ and a non-glm arch receives the legacy 8. An EXPLICIT cap passes through
+ verbatim to any engine -- including an explicit 0, which for inkling means
+ upstream's RAM-auto (people who ask for upstream semantics get them).
+ Keyed on the MODEL's arch (config.json model_type), not the engine
+ binary's file name: COLI_ENGINE users package the glm engine under
+ arbitrary names (glm52, colibri-1.2, ...), and basename keying silently
+ disabled the platform default for exactly them.
MOOTING TRIGGER: upstream unifies cap-sentinel semantics across engines
-> this shim must be removed and re-derived."""
- if cap != 0:
+ if cap is not None:
return cap
- name = Path(executable).name.lower()
- if name.endswith(".exe"):
- name = name[:-4]
- return 0 if name in _SENTINEL_CAP_ENGINES else 8
+ return 0 if arch == "glm" else 8
class Engine:
- # cap=0 = "not explicitly set": the glm engine resolves it (8 historically,
- # 1 on Metal+darwin+fast SSD -- colibri.c coli_resolve_cap, #379), while
- # non-glm engines get the legacy 8 via cap_for_engine above. Same sentinel
- # as the --cap flags in coli and main() below, so programmatic callers that
- # never pass cap get the same auto behavior as the CLI.
- def __init__(self, executable, model, cap=0, max_tokens=1024, env=None, kv_slots=1):
+ # cap=None = "not explicitly set": a glm-arch model's engine resolves the
+ # 0 sentinel (8 historically, 1 on Metal+darwin+fast SSD -- colibri.c
+ # coli_resolve_cap, #379), non-glm arches get the legacy 8, via
+ # cap_for_arch above. Same convention as the --cap flags in coli and
+ # main() below, so programmatic callers that never pass cap get the same
+ # auto behavior as the CLI; an explicit int (0 included) is verbatim.
+ def __init__(self, executable, model, cap=None, max_tokens=1024, env=None, kv_slots=1):
child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1",
NGEN=str(max_tokens), KV_SLOTS=str(kv_slots))
self.process = subprocess.Popen(
- [str(executable), str(cap_for_engine(executable, cap))], env=child_env,
+ [str(executable), str(cap_for_arch(model_arch(model), cap))], env=child_env,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0,
)
self.write_lock = threading.Lock()
@@ -2324,7 +2337,7 @@ class APIHandler(BaseHTTPRequestHandler):
def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_key=None,
- cap=0, max_tokens=1024, engine=None, env=None, cors_origins=None,
+ cap=None, max_tokens=1024, engine=None, env=None, cors_origins=None,
max_queue=8, queue_timeout=300, kv_slots=1, allowed_hosts=()):
if engine is None:
engine = default_engine()
@@ -2383,9 +2396,10 @@ def main():
parser.add_argument("--api-key", default=os.environ.get("COLI_API_KEY"))
parser.add_argument("--cors-origin", action="append", default=None,
help="allowed browser origin; repeat as needed (use '*' for any origin)")
- # 0 = not explicitly set: mirrors coli's --cap (0=auto; see colibri.c coli_resolve_cap
- # and issue #379 -- Metal+darwin+fast SSD defaults to 1, everything else to 8).
- parser.add_argument("--cap", type=int, default=0, help="cache slots/layer (0=auto)")
+ # Absent = not explicitly set: mirrors coli's --cap (see cap_for_arch and issue
+ # #379 -- glm arch resolves platform-aware, non-glm gets the legacy 8). An
+ # explicit value, 0 included, reaches the engine verbatim.
+ parser.add_argument("--cap", type=int, default=None, help="cache slots/layer (default: auto)")
parser.add_argument("--max-tokens", type=int, default=1024)
parser.add_argument("--max-queue", type=int, default=int(os.environ.get("COLI_MAX_QUEUE", "8")))
parser.add_argument("--queue-timeout", type=float,
@@ -2400,13 +2414,7 @@ def main():
global ARCH
ARCH = args.arch
if ARCH == "auto":
- try:
- with open(Path(args.model) / "config.json") as fh:
- model_type = (json.load(fh).get("model_type") or "").lower()
- ARCH = ("inkling" if "inkling" in model_type else
- "kimi" if "kimi" in model_type else "glm")
- except OSError:
- ARCH = "glm"
+ ARCH = model_arch(args.model)
if args.model_id is None:
args.model_id = ("inkling-colibri" if ARCH == "inkling" else
"kimi-k3-colibri" if ARCH == "kimi" else "glm-5.2-colibri")
diff --git a/c/tests/test_cli_output.py b/c/tests/test_cli_output.py
index efefadf5..bc6dbec2 100644
--- a/c/tests/test_cli_output.py
+++ b/c/tests/test_cli_output.py
@@ -1,8 +1,13 @@
+import importlib.util
+import json
import subprocess
import sys
import tempfile
+import types
import unittest
+from importlib.machinery import SourceFileLoader
from pathlib import Path
+from unittest import mock
HERE = Path(__file__).resolve().parent.parent
@@ -50,5 +55,77 @@ class CliOutputLanguageTest(unittest.TestCase):
self.assertIn("set COLI_MODEL or use --model", result.stderr)
+class ChatCapForwardingTest(unittest.TestCase):
+ """#379/#386 r2 (F9): `coli chat` on a non-glm model spawns openai_server
+ as its local server. An explicit --cap must ride along on that command
+ line (it was silently eaten for years -- keeping it is a DISCLOSED
+ behavior change: a long-ignored `coli chat --cap 32` now takes effect),
+ and an absent --cap must stay absent so openai_server's arch-keyed
+ default (cap_for_arch) applies. This drives the real cmd_chat with the
+ process boundary faked, and pins the argv it builds."""
+
+ @classmethod
+ def setUpClass(cls):
+ loader = SourceFileLoader("coli_cli_under_test", str(CLI))
+ spec = importlib.util.spec_from_loader(loader.name, loader)
+ cls.coli = importlib.util.module_from_spec(spec)
+ loader.exec_module(cls.coli)
+
+ def _chat_server_cmd(self, cap):
+ coli = self.coli
+ captured = {}
+
+ class FakeProc:
+ def __init__(self, cmd, **_kw):
+ captured["cmd"] = cmd
+ def poll(self):
+ return None
+ def terminate(self):
+ pass
+ def wait(self, timeout=None):
+ return 0
+ def kill(self):
+ pass
+
+ class FakeSpinner:
+ def __init__(self, *_a, **_k):
+ pass
+ def start(self):
+ pass
+ def stop(self):
+ pass
+
+ model = tempfile.mkdtemp()
+ self.addCleanup(lambda: subprocess.run(["rm", "-rf", model], check=False))
+ (Path(model) / "config.json").write_text(json.dumps({"model_type": "inkling"}))
+ args = types.SimpleNamespace(model=model, cap=cap, ngen=256, api_key=None,
+ no_attach=True, attach=None)
+ with mock.patch.object(coli, "need_model"), \
+ mock.patch.object(coli, "banner"), \
+ mock.patch.object(coli, "engine_for", return_value="/stub/inkling"), \
+ mock.patch.object(coli, "env_for_engine", return_value={}), \
+ mock.patch.object(coli, "server_probe", return_value="inkling-colibri"), \
+ mock.patch.object(coli, "chat_attached"), \
+ mock.patch.object(coli, "Spinner", FakeSpinner), \
+ mock.patch("subprocess.Popen", FakeProc):
+ coli.cmd_chat(args)
+ return captured["cmd"]
+
+ def test_explicit_cap_rides_along(self):
+ cmd = self._chat_server_cmd(cap=32)
+ self.assertIn("--cap", cmd)
+ self.assertEqual(cmd[cmd.index("--cap") + 1], "32")
+ self.assertEqual(cmd[cmd.index("--arch") + 1], "inkling")
+
+ def test_explicit_cap_zero_rides_along(self):
+ # explicit 0 = upstream RAM-auto for inkling, by request (#386 r2, F8)
+ cmd = self._chat_server_cmd(cap=0)
+ self.assertEqual(cmd[cmd.index("--cap") + 1], "0")
+
+ def test_absent_cap_stays_absent(self):
+ cmd = self._chat_server_cmd(cap=None)
+ self.assertNotIn("--cap", cmd)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 2fd7862a..77225a6e 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -14,7 +14,7 @@ from pathlib import Path
from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,
DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,
READY, Engine, InklingStreamSplit, StopFilter, ThinkingStreamSplit,
- _engine_error, cap_for_engine, conversation_cache_slot,
+ _engine_error, cap_for_arch, conversation_cache_slot, model_arch,
generation_options, parse_tool_calls, read_engine_turn,
render_chat, render_chat_kimi, serve,
split_thinking_reply, stop_policy)
@@ -593,42 +593,81 @@ class DispatcherTest(unittest.TestCase):
class CapSentinelShimTest(unittest.TestCase):
- # #379 cap-sentinel shim: cap 0 is "platform-auto" only to the glm engine
- # (colibri.c coli_resolve_cap); inkling reads cap <= 0 as "fit the expert
- # LRU to all available RAM" (inkling.c), so leaking the sentinel to a
- # non-glm engine silently changes its memory behavior. Engine() is the one
- # funnel every launch passes through -- coli serve, coli chat's local
- # server, openai_server.py --model, and programmatic callers -- so the
- # translation is pinned HERE, at the argv it actually emits.
- def _spawn_argv(self, executable, **kwargs):
+ # #379 cap-sentinel shim, arch-keyed (#386 r2, F3): an absent cap is
+ # "platform-auto" only for the glm engine (colibri.c coli_resolve_cap);
+ # inkling reads cap <= 0 as "fit the expert LRU to all available RAM"
+ # (inkling.c), so leaking the sentinel to a non-glm arch silently changes
+ # its memory behavior. The key is the MODEL's config.json model_type, not
+ # the engine binary's file name -- COLI_ENGINE users package the glm
+ # engine as glm52/colibri-1.2/glm-metal, and basename keying disabled the
+ # platform default for exactly them (and an inkling binary someone names
+ # `glm` would get the leak back). Engine() is the one funnel every launch
+ # passes through, so the translation is pinned at the argv it emits, over
+ # the full matrix: arch x arbitrary executable name x cap absent/explicit.
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+
+ def _model(self, model_type):
+ model = Path(self.tmp.name) / f"model-{model_type}"
+ model.mkdir(exist_ok=True)
+ (model / "config.json").write_text(json.dumps({"model_type": model_type}))
+ return str(model)
+
+ def _spawn_argv(self, executable, model, **kwargs):
process = FakeProcess(lambda _process, _frame: None)
with patch("openai_server.subprocess.Popen", return_value=process) as popen:
- engine = Engine(executable, "model", **kwargs)
+ engine = Engine(executable, model, **kwargs)
engine.close()
return popen.call_args[0][0]
- def test_non_glm_engine_gets_legacy_8_for_the_sentinel(self):
- for executable in ("inkling", "/opt/libexec/colibri/inkling",
- "kimi_k3", "kimi_k3.exe", "olmoe"):
- self.assertEqual(self._spawn_argv(executable), [executable, "8"],
- f"sentinel leaked to {executable}")
-
- def test_glm_engine_keeps_the_platform_auto_sentinel(self):
- for executable in ("glm", "glm.exe", "colibri", "/repo/c/colibri"):
- self.assertEqual(self._spawn_argv(executable), [executable, "0"],
- f"sentinel lost for {executable}")
-
- def test_explicit_cap_passes_through_to_any_engine(self):
- self.assertEqual(self._spawn_argv("inkling", cap=5), ["inkling", "5"])
- self.assertEqual(self._spawn_argv("colibri", cap=5), ["colibri", "5"])
-
- def test_cap_for_engine_is_the_single_translation_point(self):
- self.assertEqual(cap_for_engine("inkling", 0), 8)
- self.assertEqual(cap_for_engine("kimi_k3", 0), 8)
- self.assertEqual(cap_for_engine("colibri", 0), 0)
- self.assertEqual(cap_for_engine("glm", 0), 0)
- self.assertEqual(cap_for_engine("inkling", 3), 3)
- self.assertEqual(cap_for_engine("glm", 3), 3)
+ def test_matrix_arch_times_engine_name_times_cap(self):
+ # COLI_ENGINE axis: the executable name carries no information; only
+ # the model arch and the explicitness of cap may matter.
+ executables = ("colibri", "glm", "glm52", "colibri-1.2", "glm-metal",
+ "/opt/custom/inkling", "kimi_k3.exe")
+ cases = ( # (model_type, cap kwargs, expected argv cap)
+ ("glm_moe_dsa", {}, "0"), # absent -> glm sentinel
+ ("inkling", {}, "8"), # absent -> legacy 8
+ ("kimi_k3", {}, "8"), # absent -> legacy 8
+ ("glm_moe_dsa", {"cap": 5}, "5"), # explicit -> verbatim
+ ("inkling", {"cap": 5}, "5"),
+ ("kimi_k3", {"cap": 5}, "5"),
+ ("glm_moe_dsa", {"cap": 0}, "0"), # explicit 0 -> verbatim
+ ("inkling", {"cap": 0}, "0"), # upstream RAM-auto, by request
+ ("kimi_k3", {"cap": 0}, "0"),
+ )
+ for model_type, kwargs, want in cases:
+ model = self._model(model_type)
+ for executable in executables:
+ self.assertEqual(
+ self._spawn_argv(executable, model, **kwargs),
+ [executable, want],
+ f"arch={model_type} exe={executable} kwargs={kwargs}")
+
+ def test_missing_or_unreadable_config_is_glm(self):
+ # historic default: anything that cannot be classified is glm
+ self.assertEqual(self._spawn_argv("engine", "/nonexistent/model"),
+ ["engine", "0"])
+ model = Path(self.tmp.name) / "model-broken"
+ model.mkdir()
+ (model / "config.json").write_text("{not json")
+ self.assertEqual(self._spawn_argv("engine", str(model)), ["engine", "0"])
+
+ def test_cap_for_arch_is_the_single_translation_point(self):
+ self.assertEqual(cap_for_arch("glm", None), 0)
+ self.assertEqual(cap_for_arch("inkling", None), 8)
+ self.assertEqual(cap_for_arch("kimi", None), 8)
+ self.assertEqual(cap_for_arch("glm", 3), 3)
+ self.assertEqual(cap_for_arch("inkling", 3), 3)
+ self.assertEqual(cap_for_arch("inkling", 0), 0) # explicit 0 is explicit
+ self.assertEqual(cap_for_arch("glm", 0), 0)
+
+ def test_model_arch_reads_model_type(self):
+ self.assertEqual(model_arch(self._model("glm_moe_dsa")), "glm")
+ self.assertEqual(model_arch(self._model("inkling")), "inkling")
+ self.assertEqual(model_arch(self._model("kimi_k3")), "kimi")
+ self.assertEqual(model_arch("/nonexistent"), "glm")
class HTTPTest(unittest.TestCase):