Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
fix: enhance render_example to handle unicode and empty reasoning cases
HenryNdubuakuaf6bed6f5942
diff --git a/needle/model/finetune.py b/needle/model/finetune.pyindex e76d986..fe20877 100644--- a/needle/model/finetune.py+++ b/needle/model/finetune.py@@ -193,16 +193,16 @@ def generate_main(args):def render_example(example):tools = example.get("tools", [])- tools_json = tools if isinstance(tools, str) else json.dumps(tools, separators=(",", ":"))+ tools_json = tools if isinstance(tools, str) else json.dumps(tools, separators=(",", ":"), ensure_ascii=False)answers = example.get("answers", example.get("function_calls", []))- answers_json = answers if isinstance(answers, str) else json.dumps(answers, separators=(",", ":"))- reasoning = example.get("reasoning", "")- system = example.get("system", "")+ answers_json = answers if isinstance(answers, str) else json.dumps(answers, separators=(",", ":"), ensure_ascii=False)+ reasoning = (example.get("reasoning") or "").strip()+ system = (example.get("system") or "").strip()prefix = IM_START + "system\n" + system + IM_END + "\n" if system else ""prompt = (prefix + IM_START + "user\n" + TOOLS_START + tools_json + TOOLS_END + "\n"+ example["query"] + IM_END + "\n" + IM_START + "assistant\n")- target = (THINK_START + reasoning + THINK_END + "\n"- + TOOL_CALL_START + answers_json + TOOL_CALL_END + IM_END)+ think = THINK_START + "\n" + reasoning + "\n" + THINK_END + "\n" if reasoning else ""+ target = think + TOOL_CALL_START + answers_json + TOOL_CALL_END + IM_ENDreturn prompt, targetdiff --git a/tests/test_render.py b/tests/test_render.pyindex b8623bd..156ed6e 100644--- a/tests/test_render.py+++ b/tests/test_render.py@@ -27,6 +27,44 @@ def test_render_example_markers():assert '"name":"f"' in target+def test_render_example_matches_training_template():+ from needle.model.finetune import render_example+ from needle.model.tokenizer import (IM_END, THINK_START, THINK_END,+ TOOL_CALL_START, TOOL_CALL_END)++ example = {"tools": [], "query": "q", "reasoning": "'q' -> f",+ "answers": [{"name": "f", "arguments": {}}]}+ _, target = render_example(example)+ assert target == (THINK_START + "\n'q' -> f\n" + THINK_END + "\n"+ + TOOL_CALL_START + '[{"name":"f","arguments":{}}]'+ + TOOL_CALL_END + IM_END)+++def test_render_example_empty_reasoning_omits_think():+ from needle.model.finetune import render_example+ from needle.model.tokenizer import THINK_START, TOOL_CALL_START++ for example in ({"tools": [], "query": "q", "answers": []},+ {"tools": [], "query": "q", "reasoning": "", "answers": []},+ {"tools": [], "query": "q", "reasoning": None, "answers": []},+ {"tools": [], "query": "q", "reasoning": " ", "answers": []}):+ _, target = render_example(example)+ assert THINK_START not in target+ assert target.startswith(TOOL_CALL_START)+++def test_render_example_keeps_unicode():+ from needle.model.finetune import render_example++ example = {"tools": [{"name": "f", "parameters": {"type": "object", "properties": {+ "name": {"type": "string", "description": "café"}}}}],+ "query": "add Tomás",+ "answers": [{"name": "f", "arguments": {"name": "Tomás"}}]}+ prompt, target = render_example(example)+ assert "café" in prompt+ assert '"Tomás"' in target++def test_render_example_accepts_function_calls_alias():from needle.model.finetune import render_exampleexample = {"tools": [], "query": "hi",