Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(cli): route DeepSeek V4 through coli serving

DrewZ43c5db89e86a
diff --git a/c/coli b/c/coli
index 44a09179..31a5fd69 100755
--- a/c/coli
+++ b/c/coli
@@ -167,13 +167,16 @@ def model_arch(model):
return "inkling"
if "kimi" in model_type:
return "kimi"
+ if "deepseek_v4" in model_type or ("deepseek" in model_type and "v4" in model_type):
+ return "deepseek_v4"
return "glm"
def engine_for(model):
arch = model_arch(model)
if arch == "glm" or os.environ.get("COLI_ENGINE"):
return GLM
- name = "inkling" if arch == "inkling" else "kimi_k3"
+ name = {"inkling": "inkling", "kimi": "kimi_k3",
+ "deepseek_v4": "deepseek_v4"}[arch]
local = os.path.join(HERE, name + _EXE)
return local if os.path.exists(local) else os.path.join(_LIBEXEC, name + _EXE)
@@ -187,7 +190,8 @@ def need_model(model, engine=None):
engine = engine or engine_for(model)
if not os.path.exists(engine):
arch = model_arch(model)
- target = "kimi_k3" if arch == "kimi" else "inkling" if arch == "inkling" else "colibri"
+ target = ("kimi_k3" if arch == "kimi" else "inkling" if arch == "inkling" else
+ "deepseek-v4" if arch == "deepseek_v4" else "colibri")
sys.exit(f"{C.yel}{target} engine is not built.{C.r} Run: make -C c {target}")
def env_for_engine(a, arch):
@@ -196,6 +200,9 @@ def env_for_engine(a, arch):
env = os.environ.copy()
if a.ngen: env["NGEN"] = str(a.ngen)
if a.temp is not None: env["COLI_TEMP"] = str(a.temp)
+ if arch == "deepseek_v4":
+ if a.ram: env["RAM_GB"] = str(a.ram)
+ if a.ctx: env["CTX"] = str(a.ctx)
return env
def cuda_binary():
@@ -541,7 +548,10 @@ def cmd_build(a):
if not os.path.exists(os.path.join(HERE, "Makefile")):
sys.exit(f"{C.yel}coli build{C.r} only works from a source checkout (this is an installed copy).\n"
f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c colibri.")
- sys.exit(subprocess.call(["make","-C",HERE,"colibri"]))
+ arch=model_arch(a.model) if a.model else "glm"
+ target={"glm":"colibri","inkling":"inkling","kimi":"kimi_k3",
+ "deepseek_v4":"deepseek-v4"}[arch]
+ sys.exit(subprocess.call(["make","-C",HERE,target]))
def cmd_info(a):
banner("info")
@@ -665,9 +675,16 @@ def cmd_tune(a):
print(f" profile: {path}\n")
def cmd_run(a):
- need_model(a.model)
+ arch=model_arch(a.model)
+ engine=engine_for(a.model)
+ need_model(a.model,engine)
prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"')
banner("run")
+ if arch=="deepseek_v4":
+ cmd=[engine,os.path.abspath(a.model),prompt,"--max-tokens",str(a.ngen)]
+ if a.ram: cmd += ["--memory-gb",str(a.ram)]
+ if os.environ.get("COLI_THINK","0")=="1": cmd.append("--thinking")
+ sys.exit(subprocess.call(cmd,env=env_for_engine(a,arch)))
# template ufficiale GLM-5.2: niente \n dopo i ruoli; <think></think> = risposta diretta (nothink).
# THINK=1 lascia <think> aperto, stessa convenzione del serve mode (glm.c). EN: THINK=1 leaves
# <think> open so the engine emits its reasoning block; the default stays nothink.
@@ -804,7 +821,8 @@ def cmd_chat(a):
if arch!="glm":
engine=engine_for(a.model)
need_model(a.model,engine)
- model_id="kimi-k3-colibri" if arch=="kimi" else "inkling-colibri"
+ model_id=("kimi-k3-colibri" if arch=="kimi" else
+ "inkling-colibri" if arch=="inkling" else "deepseek-v4-colibri")
banner(f"chat · {model_id} · local server")
import openai_server
# --cap rides along so an explicit `coli chat --cap N` reaches the
@@ -985,7 +1003,9 @@ def cmd_serve(a):
import openai_server
openai_server.ARCH=arch
model_id=a.model_id or ("kimi-k3-colibri" if arch=="kimi" else
- "inkling-colibri" if arch=="inkling" else "glm-5.2-colibri")
+ "inkling-colibri" if arch=="inkling" else
+ "deepseek-v4-colibri" if arch=="deepseek_v4" else
+ "glm-5.2-colibri")
try:
openai_server.serve(a.model,a.host,a.port,model_id,a.api_key,
a.cap,a.ngen,engine,env_for_engine(a,arch),a.cors_origin,
@@ -1017,7 +1037,7 @@ def cmd_stop(a):
if "coli" in cmd and " serve" in cmd and pid!=os.getpid():
if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)"))
comm=open(f"/proc/{pd}/comm").read().strip()
- if comm in ("colibri","glm","exe","olmoe","inkling","kimi_k3"):
+ if comm in ("colibri","glm","exe","olmoe","inkling","kimi_k3","deepseek_v4"):
env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace")
if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)"))
except (OSError,PermissionError): continue
diff --git a/c/openai_server.py b/c/openai_server.py
index c129714b..ec946b2a 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -374,7 +374,7 @@ def parse_tool_calls(reply, tools=None):
return text.strip(), calls
-ARCH = "glm" # set in main(): "glm" | "inkling" | "kimi" (auto-detected)
+ARCH = "glm" # set in main(): glm | inkling | kimi | deepseek_v4
INK_THINK, INK_TEXT = "<|content_thinking|>", "<|content_text|>"
@@ -657,6 +657,51 @@ def render_chat_kimi(messages, enable_thinking=False, reasoning_effort=None, too
return "".join(parts)
+def render_chat_v4(messages, enable_thinking=False, reasoning_effort=None, tools=None,
+ tool_choice=None):
+ """DeepSeek V4's native multi-turn chat template.
+
+ The target engine receives this as a raw prompt. Prior assistant turns end
+ with the checkpoint's EOS marker; the final assistant marker selects the
+ thinking or direct-answer prefix for the new turn.
+ """
+ if not isinstance(messages, list) or not messages:
+ raise APIError(400, "`messages` must be a non-empty array.", "messages")
+ if tools or tool_choice not in (None, "none"):
+ raise APIError(400, "Tool use is not wired up for DeepSeek V4 yet.",
+ "tools", "unsupported_parameter")
+ bos = "<\uff5cbegin\u2581of\u2581sentence\uff5c>"
+ user = "<\uff5cUser\uff5c>"
+ assistant = "<\uff5cAssistant\uff5c>"
+ eos = "<\uff5cend\u2581of\u2581sentence\uff5c>"
+ parts = [bos]
+ for index, message in enumerate(messages):
+ if not isinstance(message, dict):
+ raise APIError(400, "Each message must be an object.", f"messages.{index}")
+ role = message.get("role")
+ if role not in ("system", "developer", "user", "assistant"):
+ raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
+ raw = message.get("content")
+ text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
+ if role in ("system", "developer"):
+ parts.append(text)
+ elif role == "user":
+ parts.extend((user, text))
+ else:
+ reasoning = message.get("reasoning_content")
+ if reasoning is not None and not isinstance(reasoning, str):
+ raise APIError(400, "`reasoning_content` must be a string.",
+ f"messages.{index}.reasoning_content")
+ parts.append(assistant)
+ if reasoning:
+ parts.extend(("<think>", reasoning, "</think>"))
+ else:
+ parts.append("</think>")
+ parts.extend((text, eos))
+ parts.extend((assistant, "<think>" if enable_thinking else "</think>"))
+ return "".join(parts)
+
+
def render_chat_inkling(messages, enable_thinking=False, reasoning_effort=None, tools=None,
tool_choice=None, audio_out=None):
"""Text-only subset of Inkling's chat_template.jinja: role tokens with
@@ -1340,6 +1385,8 @@ def model_arch(model):
return "inkling"
if "kimi" in model_type:
return "kimi"
+ if "deepseek_v4" in model_type or ("deepseek" in model_type and "v4" in model_type):
+ return "deepseek_v4"
return "glm"
@@ -2263,7 +2310,8 @@ class APIHandler(BaseHTTPRequestHandler):
tools = body.get("tools") or body.get("functions") or None
tool_choice = body.get("tool_choice")
renderer = (render_chat_inkling if ARCH == "inkling" else
- render_chat_kimi if ARCH == "kimi" else render_chat)
+ render_chat_kimi if ARCH == "kimi" else
+ render_chat_v4 if ARCH == "deepseek_v4" else render_chat)
audio_clips = [] if ARCH == "inkling" else None
if audio_clips is not None:
prompt = renderer(body.get("messages"), enable_thinking, reasoning_effort, tools,
@@ -2541,7 +2589,7 @@ def serve(model, host="127.0.0.1", port=8000, model_id="glm-5.2-colibri", api_ke
raise ValueError("queue_timeout must be positive")
if not 1 <= kv_slots <= 16:
raise ValueError("kv_slots must be between 1 and 16")
- if ARCH in ("inkling", "kimi") and kv_slots != 1:
+ if ARCH in ("inkling", "kimi", "deepseek_v4") and kv_slots != 1:
raise ValueError(f"{ARCH} engine currently supports exactly one KV slot")
if host not in ("127.0.0.1", "localhost", "::1") and not api_key:
# (#SEC-6) Fail closed: an unauthenticated engine on a non-loopback bind exposes
@@ -2578,7 +2626,7 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", default=os.environ.get("COLI_MODEL"), required=not os.environ.get("COLI_MODEL"))
parser.add_argument("--engine", default=str(default_engine()))
- parser.add_argument("--arch", choices=("auto", "glm", "inkling", "kimi"), default="auto",
+ parser.add_argument("--arch", choices=("auto", "glm", "inkling", "kimi", "deepseek_v4"), default="auto",
help="chat-template family; auto reads model_type from the model's config.json")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=8000)
@@ -2607,7 +2655,9 @@ def main():
ARCH = model_arch(args.model)
if args.model_id is None:
args.model_id = ("inkling-colibri" if ARCH == "inkling" else
- "kimi-k3-colibri" if ARCH == "kimi" else "glm-5.2-colibri")
+ "kimi-k3-colibri" if ARCH == "kimi" else
+ "deepseek-v4-colibri" if ARCH == "deepseek_v4" else
+ "glm-5.2-colibri")
serve(args.model, args.host, args.port, args.model_id, args.api_key,
args.cap,args.max_tokens,args.engine,cors_origins=args.cors_origin,
max_queue=args.max_queue,queue_timeout=args.queue_timeout,kv_slots=args.kv_slots,
diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.py
index 629cfd72..440fd521 100644
--- a/c/tests/test_v4_cli.py
+++ b/c/tests/test_v4_cli.py
@@ -1,19 +1,19 @@
+import argparse
import importlib.machinery
import importlib.util
-import argparse
-import subprocess
-import sys
+import json
+import os
import tempfile
import unittest
from pathlib import Path
HERE = Path(__file__).resolve().parent.parent
-CLI = HERE / "v4"
+CLI = HERE / "coli"
def load_cli():
- loader = importlib.machinery.SourceFileLoader("v4_cli", str(CLI))
+ loader = importlib.machinery.SourceFileLoader("coli_v4_cli_test", str(CLI))
spec = importlib.util.spec_from_loader(loader.name, loader)
module = importlib.util.module_from_spec(spec)
loader.exec_module(module)
@@ -21,104 +21,70 @@ def load_cli():
class V4CliTest(unittest.TestCase):
- def run_cli(self, *args):
- return subprocess.run(
- [sys.executable, str(CLI), *args],
- cwd=HERE,
- text=True,
- capture_output=True,
- check=False,
- timeout=10,
+ @classmethod
+ def setUpClass(cls):
+ cls.cli = load_cli()
+
+ def make_model(self, model_type="deepseek_v4"):
+ directory = tempfile.TemporaryDirectory()
+ root = Path(directory.name)
+ (root / "config.json").write_text(
+ json.dumps({"model_type": model_type}), encoding="utf-8"
)
+ (root / "tokenizer.json").write_text("{}", encoding="utf-8")
+ return directory, root
- def test_chat_help_uses_explicit_model_and_ram(self):
- result = self.run_cli("chat", "--help")
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertIn("--model", result.stdout)
- self.assertIn("--ram", result.stdout)
-
- def test_model_is_required(self):
- result = self.run_cli("chat", "--ram", "24")
- self.assertNotEqual(result.returncode, 0)
- self.assertIn("--model", result.stderr)
-
- def test_missing_model_is_reported_before_engine(self):
- with tempfile.TemporaryDirectory() as directory:
- missing = str(Path(directory) / "missing")
- result = self.run_cli("chat", "--model", missing, "--ram", "24")
- self.assertNotEqual(result.returncode, 0)
- self.assertIn("model not found", result.stderr)
-
- def test_multi_turn_prompt_format(self):
- cli = load_cli()
- history = cli.new_history("Be concise.")
- prompt = cli.turn_prompt(history, "Hello", False)
- self.assertEqual(
- prompt,
- cli.BOS + "Be concise." + cli.USER + "Hello" + cli.ASSISTANT + "</think>",
- )
- history = cli.completed_history(prompt, "Hi!")
- next_prompt = cli.turn_prompt(history, "Again", True)
- self.assertEqual(
- next_prompt,
- prompt + "Hi!" + cli.EOS + cli.USER + "Again" + cli.ASSISTANT + "<think>",
- )
+ def test_model_arch_detects_deepseek_v4(self):
+ directory, root = self.make_model()
+ try:
+ self.assertEqual(self.cli.model_arch(str(root)), "deepseek_v4")
+ finally:
+ directory.cleanup()
- def test_compact_stats_uses_wall_ttft_and_ram_plan(self):
- cli = load_cli()
- log = (
- "ram_tiers available=24.00GiB dense=resident projected=23.95GiB\n"
- "prefill_timing startup=2.0s wall_to_first=15.500s "
- "target_tok_s=2.0 combined_tok_s=1.250000\n"
- "decode_timing tokens=4 seconds=4.0 tok_s=1.000000\n"
- "summary dspark_rounds=1 proposed=4 accepted=2 rate=0.500 enabled=1\n"
+ def test_engine_for_selects_deepseek_v4_binary(self):
+ directory, root = self.make_model()
+ try:
+ expected = "deepseek_v4.exe" if os.name == "nt" else "deepseek_v4"
+ self.assertEqual(Path(self.cli.engine_for(str(root))).name, expected)
+ finally:
+ directory.cleanup()
+
+ def test_v4_engine_environment_forwards_ram_and_context(self):
+ args = argparse.Namespace(ngen=8, temp=0.0, ram=64, ctx=4096)
+ env = self.cli.env_for_engine(args, "deepseek_v4")
+ self.assertEqual(env["NGEN"], "8")
+ self.assertEqual(env["RAM_GB"], "64")
+ self.assertEqual(env["CTX"], "4096")
+
+ def test_openai_renderer_uses_native_v4_multiturn_template(self):
+ import openai_server
+
+ prompt = openai_server.render_chat_v4(
+ [
+ {"role": "system", "content": "Be concise."},
+ {"role": "user", "content": "Hello"},
+ {"role": "assistant", "content": "Hi!"},
+ {"role": "user", "content": "Again"},
+ ],
+ enable_thinking=True,
)
self.assertEqual(
- cli.compact_stats(log),
- "RAM 23.95/24.00 GiB | TTFT 15.50s | prefill 1.250 tok/s | "
- "decode 1.000 tok/s | DSpark acceptance 50.0%",
+ prompt,
+ "<\uff5cbegin\u2581of\u2581sentence\uff5c>Be concise."
+ "<\uff5cUser\uff5c>Hello<\uff5cAssistant\uff5c></think>Hi!"
+ "<\uff5cend\u2581of\u2581sentence\uff5c>"
+ "<\uff5cUser\uff5c>Again<\uff5cAssistant\uff5c><think>",
)
- def test_engine_command_prefers_prompt_file(self):
- cli = load_cli()
- args = argparse.Namespace(
- model=str(HERE), draft_model=None, ngen=32, ram=0,
- stop_sentence=False,
- )
- command = cli.engine_command(args, raw=True, prompt_file=r"C:\tmp\prompt.txt")
- self.assertIn("--prompt-file", command)
- self.assertIn("--no-dspark", command)
- self.assertIn(r"C:\tmp\prompt.txt", command)
- self.assertNotIn("你好", command)
-
- def test_engine_command_enables_explicit_dspark_model(self):
- cli = load_cli()
- args = argparse.Namespace(
- model=str(HERE), draft_model=str(HERE / "draft"), ngen=1,
- ram=0, stop_sentence=False,
- )
- command = cli.engine_command(args, prompt="hello")
- self.assertIn("--draft-model", command)
- self.assertIn(str((HERE / "draft").resolve()), command)
- self.assertNotIn("--no-dspark", command)
-
- def test_engine_command_keeps_combined_dspark_default(self):
- cli = load_cli()
- args = argparse.Namespace(
- model=str(HERE), draft_model=None, model_has_dspark=True,
- ngen=1, ram=0, stop_sentence=False,
- )
- command = cli.engine_command(args, prompt="hello")
- self.assertNotIn("--no-dspark", command)
- self.assertNotIn("--draft-model", command)
+ def test_openai_renderer_rejects_unwired_tools(self):
+ import openai_server
+
+ with self.assertRaises(openai_server.APIError):
+ openai_server.render_chat_v4(
+ [{"role": "user", "content": "hello"}],
+ tools=[{"type": "function"}],
+ )
- def test_write_prompt_file_is_utf8(self):
- cli = load_cli()
- path = cli.write_prompt_file("你好")
- try:
- self.assertEqual(Path(path).read_text(encoding="utf-8"), "你好")
- finally:
- Path(path).unlink(missing_ok=True)
if __name__ == "__main__":
unittest.main()