Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
fix(server): keep the rendered chat template in ChatSession.prompt_ids
linyubupaa8392a81f5f3
diff --git a/src/edge0/server/chat.py b/src/edge0/server/chat.pyindex 8ec9d2d..a1502bd 100644--- a/src/edge0/server/chat.py+++ b/src/edge0/server/chat.py@@ -88,21 +88,33 @@ class ChatSession:# renders the canonical no-think prompt — an EMPTY think# block closer, the model's direct-answer form. Default OFF# (CLI demo parity).+ #+ # The successful render MUST be kept: substituting the+ # hardcoded ChatML drops that think-block closer, the model+ # then opens its own <think> and the turn derails (issue #11:+ # <think> leaking into content, then empty/garbled replies).+ # ``_chat_text()`` is a last resort, never the success path.+ msgs = [m.__dict__ for m in self.req.messages]think = self.req.enable_thinkingif think is None:think = Falsetry:text = tok.apply_chat_template(- [m.__dict__ for m in self.req.messages],- tokenize=False, add_generation_prompt=True,+ msgs, tokenize=False, add_generation_prompt=True,enable_thinking=bool(think))except TypeError:# tokenizer template without the kwarg: plain render- text = tok.apply_chat_template(- [m.__dict__ for m in self.req.messages],- tokenize=False, add_generation_prompt=True)- else:+ try:+ text = tok.apply_chat_template(+ msgs, tokenize=False, add_generation_prompt=True)+ except Exception: # noqa: BLE001+ text = self._chat_text()+ except Exception: # noqa: BLE001 — render failed: last resorttext = self._chat_text()+ else:+ # Tokenizer without a chat template: plain ChatML. (This also+ # used to leave ``text`` unbound -> NameError.)+ text = self._chat_text()ids = tok.encode(text)if not ids:ids = [tok.bos_token_id or 0]diff --git a/tests/test_server.py b/tests/test_server.pyindex 60a3ab4..bad235b 100644--- a/tests/test_server.py+++ b/tests/test_server.py@@ -47,6 +47,7 @@ class FakeTok:def __init__(self, ids=(7, 8, 9)):self.ids = list(ids)self.bos_token_id = 0+ self.encoded_text = Nonedef apply_chat_template(self, messages, tokenize=False,add_generation_prompt=True,@@ -57,24 +58,31 @@ class FakeTok:return textdef encode(self, text):+ self.encoded_text = textreturn self.idsdef decode(self, tokens):return "".join(f"T{t}" for t in tokens)-class PlainTok(FakeTok):+class PlainTok:"""Tokenizer without apply_chat_template (stdlib fallback path).- Deleting the method (vs raising) keeps hasattr() False so the- server's plain-text fallback renders instead."""+ A standalone class rather than a FakeTok subclass: deleting the+ inherited method off the instance does not work, so ``hasattr`` would+ stay True and the fallback would never be exercised."""- def __init__(self):- super().__init__()- try:- del self.apply_chat_template- except AttributeError:- pass+ def __init__(self, ids=(7, 8, 9)):+ self.ids = list(ids)+ self.bos_token_id = 0+ self.encoded_text = None++ def encode(self, text):+ self.encoded_text = text+ return self.ids++ def decode(self, tokens):+ return "".join(f"T{t}" for t in tokens)class FakeEngine:@@ -172,6 +180,12 @@ def test_chat_session_prompt_ids_uses_template():sess = ChatSession(eng, _req())ids = sess.prompt_ids()assert ids == [7, 8, 9]+ # What actually gets encoded is the RENDERED template, not the+ # hardcoded ChatML (issue #11: the latter loses the template's empty+ # <think></think> closer and derails the turn).+ assert eng._tok.encoded_text.startswith("<tpl>")+ assert eng._tok.encoded_text.endswith("<|im_start|>assistant\n")+ assert eng._tok.encoded_text != sess._chat_text()# The template was called with enable_thinking=False (serve.py parity).text = eng._tok.apply_chat_template([{"role": "user", "content": "hi"}])assert "<|im_start|>assistant" in text@@ -182,6 +196,9 @@ def test_chat_session_prompt_ids_fallback_text():sess = ChatSession(eng, _req())ids = sess.prompt_ids()assert ids == [7, 8, 9]+ # No template on the tokenizer -> hardcoded ChatML (this path used to+ # raise NameError on the unbound ``text``).+ assert eng._tok.encoded_text == sess._chat_text()def test_chat_session_fallback_text_format():