Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

fix: wire --tools and --temperature through needle run, drop dead --no-constrained flag (#88)

Emiliano G.O.4ac96b1f444d
diff --git a/needle/cli.py b/needle/cli.py
index ef19736..c733b29 100644
--- a/needle/cli.py
+++ b/needle/cli.py
@@ -125,8 +125,8 @@ def main():
p.add_argument("--tools", type=str, default=None, help="Tools JSON for tool-call generation")
p.add_argument("--max-len", type=int, default=512)
p.add_argument("--seed", type=int, default=0)
- p.add_argument("--no-constrained", action="store_true",
- help="Disable grammar-constrained decoding for tool names/arg keys")
+ p.add_argument("--temperature", type=float, default=0.0,
+ help="Sampling temperature (0 = greedy)")
p = sub.add_parser("finetune")
p.add_argument("jsonl_path", type=str, help="Path to JSONL training data")
diff --git a/needle/model/run.py b/needle/model/run.py
index eefa66d..b65ce5a 100644
--- a/needle/model/run.py
+++ b/needle/model/run.py
@@ -1,4 +1,5 @@
import os
+import json
import pickle
import re
import sys
@@ -211,16 +212,27 @@ def generate(model, params, tokenizer, prompt, max_new_tokens=256, temperature=0
return text
+def build_prompt(query, tools=None):
+ if not tools:
+ return query
+ from .finetune import render_example
+ prompt, _ = render_example({"query": query, "tools": tools})
+ return prompt
+
+
def main(args):
params, config = load_checkpoint(args.checkpoint)
model = SimpleAttentionNetwork(config)
tokenizer = get_tokenizer(config.vocab_size)
prompt = args.query or "The most surprising thing about"
+ if getattr(args, "tools", None):
+ with open(args.tools) as handle:
+ prompt = build_prompt(prompt, json.load(handle))
print(f"prompt: {prompt!r}")
generate(
model, params, tokenizer, prompt,
max_new_tokens=args.max_len,
- temperature=getattr(args, "temperature", 0.0),
+ temperature=args.temperature,
seed=args.seed,
)
diff --git a/tests/test_run.py b/tests/test_run.py
new file mode 100644
index 0000000..ce9159d
--- /dev/null
+++ b/tests/test_run.py
@@ -0,0 +1,41 @@
+import argparse
+import json
+
+
+def test_build_prompt_passthrough_without_tools():
+ from needle.model.run import build_prompt
+
+ assert build_prompt("hello world") == "hello world"
+ assert build_prompt("hello world", tools=[]) == "hello world"
+ assert build_prompt("hello world", tools=None) == "hello world"
+
+
+def test_build_prompt_matches_training_template():
+ from needle.model.finetune import render_example
+ from needle.model.run import build_prompt
+ from needle.model.tokenizer import IM_START, TOOLS_START
+
+ tools = [{"name": "f", "parameters": {"type": "object", "properties": {}}}]
+ expected, _ = render_example({"query": "do the thing", "tools": tools})
+ out = build_prompt("do the thing", tools=tools)
+
+ assert out == expected
+ assert IM_START in out
+ assert TOOLS_START in out
+ assert "do the thing" in out
+
+
+def test_main_loads_tools_file_and_runs(tiny_checkpoint, tmp_path, capsys):
+ from needle.model.run import main
+
+ tools = [{"name": "f", "parameters": {"type": "object", "properties": {}}}]
+ path = tmp_path / "tools.json"
+ path.write_text(json.dumps(tools))
+
+ main(argparse.Namespace(
+ checkpoint=tiny_checkpoint, query="use f", tools=str(path),
+ max_len=4, seed=0, temperature=0.0))
+ out = capsys.readouterr().out
+
+ assert "<tools>" in out
+ assert '"name":"f"' in out