Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(cli): route DeepSeek V4 through coli serving
DrewZ43c5db89e86a
diff --git a/c/coli b/c/coliindex 44a09179..31a5fd69 100755--- a/c/coli+++ b/c/coli@@ -167,13 +167,16 @@ def model_arch(model):return "inkling"if "kimi" in model_type:return "kimi"+ if "deepseek_v4" in model_type or ("deepseek" in model_type and "v4" in model_type):+ return "deepseek_v4"return "glm"def engine_for(model):arch = model_arch(model)if arch == "glm" or os.environ.get("COLI_ENGINE"):return GLM- name = "inkling" if arch == "inkling" else "kimi_k3"+ name = {"inkling": "inkling", "kimi": "kimi_k3",+ "deepseek_v4": "deepseek_v4"}[arch]local = os.path.join(HERE, name + _EXE)return local if os.path.exists(local) else os.path.join(_LIBEXEC, name + _EXE)@@ -187,7 +190,8 @@ def need_model(model, engine=None):engine = engine or engine_for(model)if not os.path.exists(engine):arch = model_arch(model)- target = "kimi_k3" if arch == "kimi" else "inkling" if arch == "inkling" else "colibri"+ target = ("kimi_k3" if arch == "kimi" else "inkling" if arch == "inkling" else+ "deepseek-v4" if arch == "deepseek_v4" else "colibri")sys.exit(f"{C.yel}{target} engine is not built.{C.r} Run: make -C c {target}")def env_for_engine(a, arch):@@ -196,6 +200,9 @@ def env_for_engine(a, arch):env = os.environ.copy()if a.ngen: env["NGEN"] = str(a.ngen)if a.temp is not None: env["COLI_TEMP"] = str(a.temp)+ if arch == "deepseek_v4":+ if a.ram: env["RAM_GB"] = str(a.ram)+ if a.ctx: env["CTX"] = str(a.ctx)return envdef cuda_binary():@@ -541,7 +548,10 @@ def cmd_build(a):if not os.path.exists(os.path.join(HERE, "Makefile")):sys.exit(f"{C.yel}coli build{C.r} only works from a source checkout (this is an installed copy).\n"f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c colibri.")- sys.exit(subprocess.call(["make","-C",HERE,"colibri"]))+ arch=model_arch(a.model) if a.model else "glm"+ target={"glm":"colibri","inkling":"inkling","kimi":"kimi_k3",+ "deepseek_v4":"deepseek-v4"}[arch]+ sys.exit(subprocess.call(["make","-C",HERE,target]))def cmd_info(a):banner("info")@@ -665,9 +675,16 @@ def cmd_tune(a):print(f" profile: {path}\n")def cmd_run(a):- need_model(a.model)+ arch=model_arch(a.model)+ engine=engine_for(a.model)+ need_model(a.model,engine)prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"')banner("run")+ if arch=="deepseek_v4":+ cmd=[engine,os.path.abspath(a.model),prompt,"--max-tokens",str(a.ngen)]+ if a.ram: cmd += ["--memory-gb",str(a.ram)]+ if os.environ.get("COLI_THINK","0")=="1": cmd.append("--thinking")+ sys.exit(subprocess.call(cmd,env=env_for_engine(a,arch)))# template ufficiale GLM-5.2: niente \n dopo i ruoli; <think></think> = risposta diretta (nothink).# THINK=1 lascia <think> aperto, stessa convenzione del serve mode (glm.c). EN: THINK=1 leaves# <think> open so the engine emits its reasoning block; the default stays nothink.@@ -804,7 +821,8 @@ def cmd_chat(a):if arch!="glm":engine=engine_for(a.model)need_model(a.model,engine)- model_id="kimi-k3-colibri" if arch=="kimi" else "inkling-colibri"+ model_id=("kimi-k3-colibri" if arch=="kimi" else+ "inkling-colibri" if arch=="inkling" else "deepseek-v4-colibri")banner(f"chat · {model_id} · local server")import openai_server# --cap rides along so an explicit `coli chat --cap N` reaches the@@ -985,7 +1003,9 @@ def cmd_serve(a):import openai_serveropenai_server.ARCH=archmodel_id=a.model_id or ("kimi-k3-colibri" if arch=="kimi" else- "inkling-colibri" if arch=="inkling" else "glm-5.2-colibri")+ "inkling-colibri" if arch=="inkling" else+ "deepseek-v4-colibri" if arch=="deepseek_v4" else+ "glm-5.2-colibri")try:openai_server.serve(a.model,a.host,a.port,model_id,a.api_key,a.cap,a.ngen,engine,env_for_engine(a,arch),a.cors_origin,@@ -1017,7 +1037,7 @@ def cmd_stop(a):if "coli" in cmd and " serve" in cmd and pid!=os.getpid():if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)"))comm=open(f"/proc/{pd}/comm").read().strip()- if comm in ("colibri","glm","exe","olmoe","inkling","kimi_k3"):+ if comm in ("colibri","glm","exe","olmoe","inkling","kimi_k3","deepseek_v4"):env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace")if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)"))except (OSError,PermissionError): continuediff --git a/c/openai_server.py b/c/openai_server.pyindex c129714b..ec946b2a 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -374,7 +374,7 @@ def parse_tool_calls(reply, tools=None):return text.strip(), calls-ARCH = "glm" # set in main(): "glm" | "inkling" | "kimi" (auto-detected)+ARCH = "glm" # set in main(): glm | inkling | kimi | deepseek_v4INK_THINK, INK_TEXT = "<|content_thinking|>", "<|content_text|>"@@ -657,6 +657,51 @@ def render_chat_kimi(messages, enable_thinking=False, reasoning_effort=None, tooreturn "".join(parts)+def render_chat_v4(messages, enable_thinking=False, reasoning_effort=None, tools=None,+ tool_choice=None):+ """DeepSeek V4's native multi-turn chat template.++ The target engine receives this as a raw prompt. Prior assistant turns end+ with the checkpoint's EOS marker; the final assistant marker selects the+ thinking or direct-answer prefix for the new turn.+ """+ if not isinstance(messages, list) or not messages:+ raise APIError(400, "`messages` must be a non-empty array.", "messages")+ if tools or tool_choice not in (None, "none"):+ raise APIError(400, "Tool use is not wired up for DeepSeek V4 yet.",+ "tools", "unsupported_parameter")+ bos = "<\uff5cbegin\u2581of\u2581sentence\uff5c>"+ user = "<\uff5cUser\uff5c>"+ assistant = "<\uff5cAssistant\uff5c>"+ eos = "<\uff5cend\u2581of\u2581sentence\uff5c>"+ parts = [bos]+ for index, message in enumerate(messages):+ if not isinstance(message, dict):+ raise APIError(400, "Each message must be an object.", f"messages.{index}")+ role = message.get("role")+ if role not in ("system", "developer", "user", "assistant"):+ raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")+ raw = message.get("content")+ text = content_text(raw, f"messages.{index}.content") if raw is not None else ""+ if role in ("system", "developer"):+ parts.append(text)+ elif role == "user":+ parts.extend((user, text))+ else:+ reasoning = message.get("reasoning_content")+ if reasoning is not None and not isinstance(reasoning, str):+ raise APIError(400, "`reasoning_content` must be a string.",+ f"messages.{index}.reasoning_content")+ parts.append(assistant)+ if reasoning:+ parts.extend(("<think>", reasoning, "</think>"))+ else:+ parts.append("</think>")+ parts.extend((text, eos))+ parts.extend((assistant, "<think>" if enable_thinking else "</think>"))+ return "".join(parts)++def render_chat_inkling(messages, enable_thinking=False, reasoning_effort=None, tools=None,tool_choice=None, audio_out=None):"""Text-only subset of Inkling's chat_template.jinja: role tokens with@@ -1340,6 +1385,8 @@ def model_arch(model):return "inkling"if "kimi" in model_type:return "kimi"+ if "deepseek_v4" in model_type or ("deepseek" in model_type and "v4" in model_type):+ return "deepseek_v4"return "glm"@@ -2263,7 +2310,8 @@ class APIHandler(BaseHTTPRequestHandler):tools = body.get("tools") or body.get("functions") or Nonetool_choice = body.get("tool_choice")renderer = (render_chat_inkling if ARCH == "inkling" else- render_chat_kimi if ARCH == "kimi" else render_chat)+ render_chat_kimi if ARCH == "kimi" else+ render_chat_v4 if ARCH == "deepseek_v4" else render_chat)audio_clips = [] if ARCH == "inkling" else Noneif audio_clips is not None:prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,@@ -2541,7 +2589,7 @@ def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_keraise ValueError("queue_timeout must be positive")if not 1 <= kv_slots <= 16:raise ValueError("kv_slots must be between 1 and 16")- if ARCH in ("inkling", "kimi") and kv_slots != 1:+ if ARCH in ("inkling", "kimi", "deepseek_v4") and kv_slots != 1:raise ValueError(f"{ARCH} engine currently supports exactly one KV slot")if host not in ("127.0.0.1", "localhost", "::1") and not api_key:# (#SEC-6) Fail closed: an unauthenticated engine on a non-loopback bind exposes@@ -2578,7 +2626,7 @@ def main():parser = argparse.ArgumentParser(description=__doc__)parser.add_argument("--model", default=os.environ.get("COLI_MODEL"), required=not os.environ.get("COLI_MODEL"))parser.add_argument("--engine", default=str(default_engine()))- parser.add_argument("--arch", choices=("auto", "glm", "inkling", "kimi"), default="auto",+ parser.add_argument("--arch", choices=("auto", "glm", "inkling", "kimi", "deepseek_v4"), default="auto",help="chat-template family; auto reads model_type from the model's config.json")parser.add_argument("--host", default="127.0.0.1")parser.add_argument("--port", type=int, default=8000)@@ -2607,7 +2655,9 @@ def main():ARCH = model_arch(args.model)if args.model_id is None:args.model_id = ("inkling-colibri" if ARCH == "inkling" else- "kimi-k3-colibri" if ARCH == "kimi" else "glm-5.2-colibri")+ "kimi-k3-colibri" if ARCH == "kimi" else+ "deepseek-v4-colibri" if ARCH == "deepseek_v4" else+ "glm-5.2-colibri")serve(args.model, args.host, args.port, args.model_id, args.api_key,args.cap,args.max_tokens,args.engine,cors_origins=args.cors_origin,max_queue=args.max_queue,queue_timeout=args.queue_timeout,kv_slots=args.kv_slots,diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.pyindex 629cfd72..440fd521 100644--- a/c/tests/test_v4_cli.py+++ b/c/tests/test_v4_cli.py@@ -1,19 +1,19 @@+import argparseimport importlib.machineryimport importlib.util-import argparse-import subprocess-import sys+import json+import osimport tempfileimport unittestfrom pathlib import PathHERE = Path(__file__).resolve().parent.parent-CLI = HERE / "v4"+CLI = HERE / "coli"def load_cli():- loader = importlib.machinery.SourceFileLoader("v4_cli", str(CLI))+ loader = importlib.machinery.SourceFileLoader("coli_v4_cli_test", str(CLI))spec = importlib.util.spec_from_loader(loader.name, loader)module = importlib.util.module_from_spec(spec)loader.exec_module(module)@@ -21,104 +21,70 @@ def load_cli():class V4CliTest(unittest.TestCase):- def run_cli(self, *args):- return subprocess.run(- [sys.executable, str(CLI), *args],- cwd=HERE,- text=True,- capture_output=True,- check=False,- timeout=10,+ @classmethod+ def setUpClass(cls):+ cls.cli = load_cli()++ def make_model(self, model_type="deepseek_v4"):+ directory = tempfile.TemporaryDirectory()+ root = Path(directory.name)+ (root / "config.json").write_text(+ json.dumps({"model_type": model_type}), encoding="utf-8")+ (root / "tokenizer.json").write_text("{}", encoding="utf-8")+ return directory, root- def test_chat_help_uses_explicit_model_and_ram(self):- result = self.run_cli("chat", "--help")- self.assertEqual(result.returncode, 0, result.stderr)- self.assertIn("--model", result.stdout)- self.assertIn("--ram", result.stdout)-- def test_model_is_required(self):- result = self.run_cli("chat", "--ram", "24")- self.assertNotEqual(result.returncode, 0)- self.assertIn("--model", result.stderr)-- def test_missing_model_is_reported_before_engine(self):- with tempfile.TemporaryDirectory() as directory:- missing = str(Path(directory) / "missing")- result = self.run_cli("chat", "--model", missing, "--ram", "24")- self.assertNotEqual(result.returncode, 0)- self.assertIn("model not found", result.stderr)-- def test_multi_turn_prompt_format(self):- cli = load_cli()- history = cli.new_history("Be concise.")- prompt = cli.turn_prompt(history, "Hello", False)- self.assertEqual(- prompt,- cli.BOS + "Be concise." + cli.USER + "Hello" + cli.ASSISTANT + "</think>",- )- history = cli.completed_history(prompt, "Hi!")- next_prompt = cli.turn_prompt(history, "Again", True)- self.assertEqual(- next_prompt,- prompt + "Hi!" + cli.EOS + cli.USER + "Again" + cli.ASSISTANT + "<think>",- )+ def test_model_arch_detects_deepseek_v4(self):+ directory, root = self.make_model()+ try:+ self.assertEqual(self.cli.model_arch(str(root)), "deepseek_v4")+ finally:+ directory.cleanup()- def test_compact_stats_uses_wall_ttft_and_ram_plan(self):- cli = load_cli()- log = (- "ram_tiers available=24.00GiB dense=resident projected=23.95GiB\n"- "prefill_timing startup=2.0s wall_to_first=15.500s "- "target_tok_s=2.0 combined_tok_s=1.250000\n"- "decode_timing tokens=4 seconds=4.0 tok_s=1.000000\n"- "summary dspark_rounds=1 proposed=4 accepted=2 rate=0.500 enabled=1\n"+ def test_engine_for_selects_deepseek_v4_binary(self):+ directory, root = self.make_model()+ try:+ expected = "deepseek_v4.exe" if os.name == "nt" else "deepseek_v4"+ self.assertEqual(Path(self.cli.engine_for(str(root))).name, expected)+ finally:+ directory.cleanup()++ def test_v4_engine_environment_forwards_ram_and_context(self):+ args = argparse.Namespace(ngen=8, temp=0.0, ram=64, ctx=4096)+ env = self.cli.env_for_engine(args, "deepseek_v4")+ self.assertEqual(env["NGEN"], "8")+ self.assertEqual(env["RAM_GB"], "64")+ self.assertEqual(env["CTX"], "4096")++ def test_openai_renderer_uses_native_v4_multiturn_template(self):+ import openai_server++ prompt = openai_server.render_chat_v4(+ [+ {"role": "system", "content": "Be concise."},+ {"role": "user", "content": "Hello"},+ {"role": "assistant", "content": "Hi!"},+ {"role": "user", "content": "Again"},+ ],+ enable_thinking=True,)self.assertEqual(- cli.compact_stats(log),- "RAM 23.95/24.00 GiB | TTFT 15.50s | prefill 1.250 tok/s | "- "decode 1.000 tok/s | DSpark acceptance 50.0%",+ prompt,+ "<\uff5cbegin\u2581of\u2581sentence\uff5c>Be concise."+ "<\uff5cUser\uff5c>Hello<\uff5cAssistant\uff5c></think>Hi!"+ "<\uff5cend\u2581of\u2581sentence\uff5c>"+ "<\uff5cUser\uff5c>Again<\uff5cAssistant\uff5c><think>",)- def test_engine_command_prefers_prompt_file(self):- cli = load_cli()- args = argparse.Namespace(- model=str(HERE), draft_model=None, ngen=32, ram=0,- stop_sentence=False,- )- command = cli.engine_command(args, raw=True, prompt_file=r"C:\tmp\prompt.txt")- self.assertIn("--prompt-file", command)- self.assertIn("--no-dspark", command)- self.assertIn(r"C:\tmp\prompt.txt", command)- self.assertNotIn("你好", command)-- def test_engine_command_enables_explicit_dspark_model(self):- cli = load_cli()- args = argparse.Namespace(- model=str(HERE), draft_model=str(HERE / "draft"), ngen=1,- ram=0, stop_sentence=False,- )- command = cli.engine_command(args, prompt="hello")- self.assertIn("--draft-model", command)- self.assertIn(str((HERE / "draft").resolve()), command)- self.assertNotIn("--no-dspark", command)-- def test_engine_command_keeps_combined_dspark_default(self):- cli = load_cli()- args = argparse.Namespace(- model=str(HERE), draft_model=None, model_has_dspark=True,- ngen=1, ram=0, stop_sentence=False,- )- command = cli.engine_command(args, prompt="hello")- self.assertNotIn("--no-dspark", command)- self.assertNotIn("--draft-model", command)+ def test_openai_renderer_rejects_unwired_tools(self):+ import openai_server++ with self.assertRaises(openai_server.APIError):+ openai_server.render_chat_v4(+ [{"role": "user", "content": "hello"}],+ tools=[{"type": "function"}],+ )- def test_write_prompt_file_is_utf8(self):- cli = load_cli()- path = cli.write_prompt_file("你好")- try:- self.assertEqual(Path(path).read_text(encoding="utf-8"), "你好")- finally:- Path(path).unlink(missing_ok=True)if __name__ == "__main__":unittest.main()