Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(kimi): continue an open assistant turn, framed in the engine

enif461050dc26f
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index 5543ad61..0bfd1177 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -2370,7 +2370,8 @@ static int sample_tok(const float *lo, int V, float temp, float top_p){
* (K3_THINK=0 opens <response> directly = non-thinking mode). The model then
* closes think, opens response, and finishes with <|end_of_msg|> (the eos). */
typedef struct { Tok *T; int *ids; int n, cap;
- int sp_open, sp_close, sp_sep, sp_eom; } ChatB;
+ int sp_open, sp_close, sp_sep, sp_eom;
+ int cont; } ChatB; /* cont: the final turn was left open (continuation) */
static void cb_special(ChatB *b, int id){
if(b->n>=b->cap){ fprintf(stderr,"chat prompt too long\n"); exit(1); }
b->ids[b->n++]=id;
@@ -2398,7 +2399,7 @@ static int chat_build(Tok *T, const char *sys, const char *user, int thinking,
int *ids, int cap, int *sp){
ChatB b={T,ids,0,cap,
chat_special(T,"<|open|>"), chat_special(T,"<|close|>"),
- chat_special(T,"<|sep|>"), chat_special(T,"<|end_of_msg|>")};
+ chat_special(T,"<|sep|>"), chat_special(T,"<|end_of_msg|>"), 0};
if(b.sp_open<0||b.sp_close<0||b.sp_sep<0||b.sp_eom<0){
fprintf(stderr,"chat: XTML special tokens not in tokenizer.json\n"); exit(1); }
sp[0]=b.sp_open; sp[1]=b.sp_close; sp[2]=b.sp_sep; sp[3]=b.sp_eom;
@@ -2475,7 +2476,7 @@ static int chat_build_wire(Tok *T, const char *wire, int nwire, int *thinking,
int *ids, int cap, int *sp){
ChatB b={T,ids,0,cap,
chat_special(T,"<|open|>"), chat_special(T,"<|close|>"),
- chat_special(T,"<|sep|>"), chat_special(T,"<|end_of_msg|>")};
+ chat_special(T,"<|sep|>"), chat_special(T,"<|end_of_msg|>"), 0};
/* -2, not -1: the caller must be able to tell a bad payload from a snapshot
* whose tokenizer has no XTML tokens. Serve reported both as "invalid K3
* chat payload", which sent at least one user hunting through a request
@@ -2489,6 +2490,7 @@ static int chat_build_wire(Tok *T, const char *wire, int nwire, int *thinking,
while(p<end){
const char *nl=memchr(p,'\n',(size_t)(end-p));
if(!nl) return -1;
+ if(b.cont && *p!='G') return -1; /* the open final turn must be last; only G may follow */
if(*p=='G'){
int v=0;
if(sscanf(p,"G %d",&v)!=1) return -1;
@@ -2504,6 +2506,25 @@ static int chat_build_wire(Tok *T, const char *wire, int nwire, int *thinking,
chat_assistant(&b,reason,text);
free(reason); free(text); p=nl+1+nr+nt; continue;
}
+ if(*p=='C'){ /* continuation: the FINAL assistant turn, left OPEN.
+ * C <reason-len> <text-len>\n<reason><text> -- the same
+ * body as A, but rendered as the trailing open turn (no
+ * <|close|>, no <|end_of_msg|>, and the caller appends no
+ * fresh generation cue). An old engine has no 'C' record:
+ * it falls to the 'M' branch below, fails to parse, and
+ * rejects the payload -- fail-closed, never miswired. */
+ int nr=-1, nt=-1;
+ if(sscanf(p,"C %d %d",&nr,&nt)!=2||nr<0||nt<0||nl+1+nr+nt>end) return -1;
+ char *reason=malloc((size_t)nr+1), *text=malloc((size_t)nt+1);
+ if(!reason||!text){ fprintf(stderr,"OOM chat continuation\n"); exit(1); }
+ memcpy(reason,nl+1,(size_t)nr); reason[nr]=0;
+ memcpy(text,nl+1+nr,(size_t)nt); text[nt]=0;
+ cb_open(&b,"message","assistant");
+ if(nr){ cb_open(&b,"think",NULL); cb_text(&b,reason); cb_close(&b,"think"); }
+ cb_open(&b,"response",NULL); cb_text(&b,text); /* OPEN: no close, no eom */
+ b.cont=1;
+ free(reason); free(text); p=nl+1+nr+nt; continue;
+ }
if(*p=='Y'){ /* typed system message (#1143): tool-declare / tool-choice */
int ntp=-1, nb=-1;
if(sscanf(p,"Y %d %d",&ntp,&nb)!=2||ntp<1||ntp>64||nb<0||nl+1+ntp+nb>end) return -1;
@@ -2601,8 +2622,10 @@ static int chat_build_wire(Tok *T, const char *wire, int nwire, int *thinking,
chat_message(&b,r,text,!strcmp(r,"assistant"));
free(text); p=nl+1+nb;
}
- cb_open(&b,"message","assistant");
- cb_open(&b,*thinking?"think":"response",NULL);
+ if(!b.cont){ /* a continuation already emitted the open final turn */
+ cb_open(&b,"message","assistant");
+ cb_open(&b,*thinking?"think":"response",NULL);
+ }
return b.n;
}
diff --git a/c/openai_server.py b/c/openai_server.py
index 41b41bf5..133587b0 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -1063,12 +1063,18 @@ def _k3_order_tool_results(messages):
def render_chat_kimi(messages, enable_thinking=False, reasoning_effort=None, tools=None,
- tool_choice=None):
+ tool_choice=None, add_generation_prompt=True):
"""Validated multi-turn K3 payload for the C engine.
K3's rank-BPE makes ordinary-text segment boundaries part of the tokenizer
contract. This private length-framed payload preserves roles, UTF-8 bytes,
and message boundaries; kimi_k3.c constructs the native XTML tokens.
+
+ add_generation_prompt=False continues a trailing assistant turn. Kimi frames turns
+ engine-side, so unlike the string renderers there is no terminator to drop here: the final
+ assistant turn is emitted as a `C` record (reasoning + text), which kimi_k3.c renders as
+ the open turn -- no <|close|>/<|end_of_msg|>, and no fresh generation cue. An engine that
+ predates the record rejects the payload rather than miswiring it.
"""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
@@ -1121,6 +1127,14 @@ def render_chat_kimi(messages, enable_thinking=False, reasoning_effort=None, too
if role == "assistant":
last_calls = calls or []
tool_index = 0
+ if not add_generation_prompt and index == len(messages) - 1:
+ # Continuation: the trailing assistant turn is left OPEN. resolve_generation_prompt
+ # has already refused tools/tool_calls and a non-assistant trailing turn, so this is
+ # a plain assistant turn; the C record carries its reasoning (if any) and text, and
+ # kimi_k3.c renders it as the open turn with no cue.
+ r = reasoning or ""
+ parts.append(f"C {len(r.encode('utf-8'))} {len(text.encode('utf-8'))}\n{r}{text}")
+ continue
if calls:
if len(calls) > 64:
raise APIError(400, "Too many tool calls in one message (max 64).",
@@ -2388,16 +2402,18 @@ def render_chat_dsv41(messages, enable_thinking=False, reasoning_effort=None, to
# and for speed (tests/test_glm53_chat_template.py says why), and the bill for that choice is
# exactly here: one template flag, one open-turn shape to derive per renderer. Each string
# renderer derives its own, pinned byte-for-byte against the checkpoint's template;
-# CONTINUATION_FAMILIES is the set that has done so. Kimi K3 is the exception -- its prompt is
-# framed engine-side (render_chat_kimi hands a K3CHAT1 record to kimi_k3.c, which assembles the
-# XTML tokens), so its open turn is a change in the C path, not here, and is left to that engine.
+# CONTINUATION_FAMILIES is the set that has done so. Kimi K3 differs in WHERE its shape lives:
+# its prompt is framed engine-side (render_chat_kimi hands a K3CHAT1 record to kimi_k3.c, which
+# assembles the XTML tokens), so its open turn is a `C` record here plus a branch in that C path,
+# pinned by tests/test_k3_chat_tools.c against the tiny tokenizer rather than by a template diff.
# Families whose renderer implements the add_generation_prompt=False (open-turn) branch. A
# trailing assistant turn on a family NOT in this set falls through to the ordinary render
# (the cue is appended, exactly as before this existed) rather than erroring -- continuation is
# on by default, and a family without its open-turn shape yet must not start rejecting requests
# nobody opted into. Each renderer adds itself here in the same commit that derives its shape.
-CONTINUATION_FAMILIES = {"glm53", "qwen38", "qwen36", "glm", "olmoe", "deepseek_v4", "inkling"}
+CONTINUATION_FAMILIES = {"glm53", "qwen38", "qwen36", "glm", "olmoe", "deepseek_v4", "inkling",
+ "kimi"}
def resolve_generation_prompt(messages, body):
@@ -2417,10 +2433,11 @@ def resolve_generation_prompt(messages, body):
(fold the trailing turn into a completed one and append a fresh cue). It is the only value
that turns this off; anything else, including unset, leaves it on.
- A family whose renderer has no open-turn shape yet (ARCH not in CONTINUATION_FAMILIES --
- Kimi K3, whose turn is framed engine-side) falls through to the ordinary render rather than
- erroring: continuation defaults on, so an unimplemented family must not start rejecting
- trailing-assistant requests that worked before.
+ A family whose renderer has no open-turn shape yet (ARCH not in CONTINUATION_FAMILIES)
+ falls through to the ordinary render rather than erroring: continuation defaults on, so a
+ family added before its open-turn shape must not start rejecting trailing-assistant
+ requests that worked before. Every shipped family is in the set today, Kimi K3 included --
+ its open turn is framed in kimi_k3.c (a `C` record), not derived in the renderer here.
"""
continuing = os.environ.get("COLI_CONTINUE_ASSISTANT", "1") != "0"
last = messages[-1] if isinstance(messages, list) and messages else None
@@ -2501,9 +2518,12 @@ def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None,
if ARCH == "deepseek_v4":
return render_chat_v4(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt)
- renderer = (render_chat_kimi if ARCH == "kimi" else
- render_chat_dsv41 if ARCH == "deepseek_v41" else render_chat)
- return renderer(messages, enable_thinking, reasoning_effort, tools, tool_choice)
+ if ARCH == "kimi":
+ return render_chat_kimi(messages, enable_thinking, reasoning_effort, tools,
+ tool_choice, add_generation_prompt=add_generation_prompt)
+ if ARCH == "deepseek_v41":
+ return render_chat_dsv41(messages, enable_thinking, reasoning_effort, tools, tool_choice)
+ return render_chat(messages, enable_thinking, reasoning_effort, tools, tool_choice)
# ---- Anthropic Messages API (#343) --------------------------------------------------------
diff --git a/c/tests/test_k3_chat_tools.c b/c/tests/test_k3_chat_tools.c
index b7cc74b6..de0c7df2 100644
--- a/c/tests/test_k3_chat_tools.c
+++ b/c/tests/test_k3_chat_tools.c
@@ -116,6 +116,22 @@ int main(void){
"G 0\n",want);
}
+ /* C: continuation -- the FINAL assistant turn is left OPEN. The prior turns render as
+ * usual, but the last turn has no <|close|>/<|end_of_msg|> and NO fresh generation cue
+ * is appended (contrast tail_plain/tail_think in every case above). */
+ expect(&T,"C leaves the final assistant turn open, no cue appended",
+ "K3CHAT1\nM user 6\nhello?C 0 2\nhiG 0\n",
+ "<|open|>message role=\"user\"<|sep|>hello?<|close|>message<|sep|><|end_of_msg|>"
+ "<|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>hi");
+
+ /* C with reasoning: the think block is closed, the response is left open -- the model
+ * resumes writing its answer, its reasoning already behind it. */
+ expect(&T,"C with reasoning closes think and leaves response open",
+ "K3CHAT1\nM user 6\nhello?C 3 2\nwhyhiG 0\n",
+ "<|open|>message role=\"user\"<|sep|>hello?<|close|>message<|sep|><|end_of_msg|>"
+ "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>why<|close|>think<|sep|>"
+ "<|open|>response<|sep|>hi");
+
/* Malformed records are refused, not misread. */
{
int ids[256], sp[4], thinking=0;
@@ -123,6 +139,8 @@ int main(void){
"K3CHAT1\nY 200 4\nshortG 0\n", /* lengths past the payload */
"K3CHAT1\nO 0 3 2\nfooxxG 0\n", /* index < 1 */
"K3CHAT1\nB 0 0 0 1\nX 3 0\nfooG 0\n", /* unknown call record */
+ "K3CHAT1\nC 0 2\nhiM user 1\nxG 0\n", /* a turn after the open final turn */
+ "K3CHAT1\nC 0 2\nhiC 0 1\nyG 0\n", /* two open final turns */
};
for(size_t i=0;i<sizeof(bad)/sizeof(*bad);i++){
int n=chat_build_wire(&T,bad[i],(int)strlen(bad[i]),&thinking,ids,256,sp);
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 3ff0d260..5bec807a 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -2148,11 +2148,12 @@ class TrailingAssistantTurnTest(unittest.TestCase):
def test_unimplemented_family_passes_through(self):
"""Continuation is on by default, so a family whose renderer has no open-turn shape
yet must render as before -- append the cue -- not reject a request nobody opted into.
- Each family leaves this list in the commit that derives its shape; Kimi K3 (framed
- engine-side, so its open turn is a change in kimi_k3.c) is the standing case."""
- for arch in ("kimi",):
- with self.on(), patch("openai_server.ARCH", arch):
- self.assertTrue(resolve_generation_prompt(self.OPEN_TURN, {}))
+ Every shipped family is now in CONTINUATION_FAMILIES (Kimi K3 too, via its C `C`
+ record), so the backstop is exercised with a hypothetical future arch: it must pass
+ through, not error, the day a new renderer lands before its open-turn shape does."""
+ self.assertNotIn("future_family", CONTINUATION_FAMILIES)
+ with self.on(), patch("openai_server.ARCH", "future_family"):
+ self.assertTrue(resolve_generation_prompt(self.OPEN_TURN, {}))
def test_continuation_open_turn_deepseek_v4(self):
"""deepseek_v4 has no authoritative vendored jinja template to diff against:
@@ -2225,12 +2226,18 @@ class TrailingAssistantTurnTest(unittest.TestCase):
SECOND user turn must survive into their common prefix. If the first assistant turn
lost its terminator in the open render, that prefix would break right after it, before
this text. Checked in both thinking modes; the set drives the loop so a newly added
- family is covered the day it joins."""
+ family is covered the day it joins.
+
+ Kimi K3 is excluded: render_chat_for_arch returns its engine-side K3CHAT1 wire, not a
+ string prompt, so this string-level invariant doesn't apply -- its open turn is pinned
+ at the token level in tests/test_k3_chat_tools.c against the tiny tokenizer instead."""
multi = [{"role": "user", "content": "1+1?"},
{"role": "assistant", "content": "2"},
{"role": "user", "content": "capitale della Francia?"},
{"role": "assistant", "content": "La capitale e'"}]
for arch in sorted(CONTINUATION_FAMILIES):
+ if arch == "kimi":
+ continue
for enable_thinking in (True, False):
where = (arch, enable_thinking)
with patch("openai_server.ARCH", arch):
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 8f0f17d4..00fd1752 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -465,7 +465,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe
| `COLI_DEBUG` | `0` (off) | Tee the engine transaction to stderr, by level. **`1`** = decoded model output stream only (byte-by-byte, on both the tool-call and plain paths). **`2`** = both sides β€” the fully-rendered prompt the engine received *and* the output, bracketed and correlated by request id, so stderr reads as the whole conversation. Invaluable for seeing what the model received vs. emitted during an OpenCode session. |
| `COLI_TOOL_SALVAGE` | `0` (off) | Opt-in de-mangler: reconstruct a malformed int4 tool call by mapping its lone payload onto the tool's primary parameter. Never rewrites well-formed output; recommended for int4 deployments. |
| `COLI_THINK` | `0` (off) | Make thinking the default when the client sends *neither* `reasoning_effort` nor `enable_thinking`. Any explicit client value still wins. |
-| `COLI_CONTINUE_ASSISTANT` | `1` (on) | On the OpenAI- and Anthropic-compatible chat endpoints, continue a trailing `assistant` message instead of opening a new turn β€” the checkpoint's own template with `add_generation_prompt=False`. On by default: a message list ending in a non-empty `assistant` turn continues, the same contract as Anthropic's API. Set `0` to restore the old behavior (append a fresh generation cue). Refused with `tools`/`tool_calls`, and the turn must carry text not ending in whitespace. A family whose renderer has no open-turn shape yet (Kimi K3, framed engine-side) falls through to the old behavior. Unrelated to `COLI_PREFILL_CHUNK`, which is the compute phase. |
+| `COLI_CONTINUE_ASSISTANT` | `1` (on) | On the OpenAI- and Anthropic-compatible chat endpoints, continue a trailing `assistant` message β€” render its turn open and resume from it, dropping the turn terminator and the generation cue β€” instead of opening a new turn, the same contract as Anthropic's API. On by default: a message list ending in a non-empty `assistant` turn continues. Set `0` to restore the old behavior (append a fresh generation cue). Refused with `tools`/`tool_calls`, and the turn must carry text not ending in whitespace. Every shipped family supports it, Kimi K3 included (its open turn is framed engine-side in `kimi_k3.c`). Unrelated to `COLI_PREFILL_CHUNK`, which is the compute phase. |
| `COLI_MODEL` | unset | Default model directory (fallback for `--model`). |
| `COLI_MODEL_ID` | `glm-5.2-colibri` | Model id reported by the API. |
| `COLI_API_KEY` | unset | Required bearer token for the server. |
diff --git a/docs/api.md b/docs/api.md
index 845c6ba9..2468e74a 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -40,8 +40,10 @@ completed one and append a fresh cue). Continuation is refused together with
`tools`/`tool_calls`, because the tool-call parsers read an assistant turn from
its start, and the turn must carry text not ending in whitespace: the template
strips trailing whitespace, so the model would resume from different bytes than
-the ones sent. A family whose renderer has no open-turn shape yet β€” Kimi K3,
-whose turn is framed engine-side β€” falls through to the old behaviour.
+the ones sent. A family whose renderer has no open-turn shape yet falls through
+to the old behaviour rather than erroring β€” though every shipped family supports
+continuation today, Kimi K3 included (its open turn is framed engine-side, in
+`kimi_k3.c`, not derived in the gateway renderer).
A continuation resumes from the exact bytes you send, which makes the split
point part of the prompt. Splitting mid-word puts the model at a token boundary