Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python

fix(server): keep the rendered chat template in ChatSession.prompt_ids

linyubupaa8392a81f5f3
diff --git a/src/edge0/server/chat.py b/src/edge0/server/chat.py
index 8ec9d2d..a1502bd 100644
--- a/src/edge0/server/chat.py
+++ b/src/edge0/server/chat.py
@@ -88,21 +88,33 @@ class ChatSession:
# renders the canonical no-think prompt — an EMPTY think
# block closer, the model's direct-answer form. Default OFF
# (CLI demo parity).
+ #
+ # The successful render MUST be kept: substituting the
+ # hardcoded ChatML drops that think-block closer, the model
+ # then opens its own <think> and the turn derails (issue #11:
+ # <think> leaking into content, then empty/garbled replies).
+ # ``_chat_text()`` is a last resort, never the success path.
+ msgs = [m.__dict__ for m in self.req.messages]
think = self.req.enable_thinking
if think is None:
think = False
try:
text = tok.apply_chat_template(
- [m.__dict__ for m in self.req.messages],
- tokenize=False, add_generation_prompt=True,
+ msgs, tokenize=False, add_generation_prompt=True,
enable_thinking=bool(think))
except TypeError:
# tokenizer template without the kwarg: plain render
- text = tok.apply_chat_template(
- [m.__dict__ for m in self.req.messages],
- tokenize=False, add_generation_prompt=True)
- else:
+ try:
+ text = tok.apply_chat_template(
+ msgs, tokenize=False, add_generation_prompt=True)
+ except Exception: # noqa: BLE001
+ text = self._chat_text()
+ except Exception: # noqa: BLE001 — render failed: last resort
text = self._chat_text()
+ else:
+ # Tokenizer without a chat template: plain ChatML. (This also
+ # used to leave ``text`` unbound -> NameError.)
+ text = self._chat_text()
ids = tok.encode(text)
if not ids:
ids = [tok.bos_token_id or 0]
diff --git a/tests/test_server.py b/tests/test_server.py
index 60a3ab4..bad235b 100644
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -47,6 +47,7 @@ class FakeTok:
def __init__(self, ids=(7, 8, 9)):
self.ids = list(ids)
self.bos_token_id = 0
+ self.encoded_text = None
def apply_chat_template(self, messages, tokenize=False,
add_generation_prompt=True,
@@ -57,24 +58,31 @@ class FakeTok:
return text
def encode(self, text):
+ self.encoded_text = text
return self.ids
def decode(self, tokens):
return "".join(f"T{t}" for t in tokens)
-class PlainTok(FakeTok):
+class PlainTok:
"""Tokenizer without apply_chat_template (stdlib fallback path).
- Deleting the method (vs raising) keeps hasattr() False so the
- server's plain-text fallback renders instead."""
+ A standalone class rather than a FakeTok subclass: deleting the
+ inherited method off the instance does not work, so ``hasattr`` would
+ stay True and the fallback would never be exercised."""
- def __init__(self):
- super().__init__()
- try:
- del self.apply_chat_template
- except AttributeError:
- pass
+ def __init__(self, ids=(7, 8, 9)):
+ self.ids = list(ids)
+ self.bos_token_id = 0
+ self.encoded_text = None
+
+ def encode(self, text):
+ self.encoded_text = text
+ return self.ids
+
+ def decode(self, tokens):
+ return "".join(f"T{t}" for t in tokens)
class FakeEngine:
@@ -172,6 +180,12 @@ def test_chat_session_prompt_ids_uses_template():
sess = ChatSession(eng, _req())
ids = sess.prompt_ids()
assert ids == [7, 8, 9]
+ # What actually gets encoded is the RENDERED template, not the
+ # hardcoded ChatML (issue #11: the latter loses the template's empty
+ # <think></think> closer and derails the turn).
+ assert eng._tok.encoded_text.startswith("<tpl>")
+ assert eng._tok.encoded_text.endswith("<|im_start|>assistant\n")
+ assert eng._tok.encoded_text != sess._chat_text()
# The template was called with enable_thinking=False (serve.py parity).
text = eng._tok.apply_chat_template([{"role": "user", "content": "hi"}])
assert "<|im_start|>assistant" in text
@@ -182,6 +196,9 @@ def test_chat_session_prompt_ids_fallback_text():
sess = ChatSession(eng, _req())
ids = sess.prompt_ids()
assert ids == [7, 8, 9]
+ # No template on the tokenizer -> hardcoded ChatML (this path used to
+ # raise NameError on the unbound ``text``).
+ assert eng._tok.encoded_text == sess._chat_text()
def test_chat_session_fallback_text_format():