Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

feat: enhance Needle class to handle weights and add tests for weight management

HenryNdubuakuf3dc9f9a4b3b
diff --git a/needle/__init__.py b/needle/__init__.py
index 169dea3..30aef70 100644
--- a/needle/__init__.py
+++ b/needle/__init__.py
@@ -69,6 +69,12 @@ class Needle:
global _active, _active_weights, _active_blob
if _active is self:
return
+ if not self._weights and _active_weights:
+ raise RuntimeError(
+ f"{_active_weights} is loaded and the engine cannot unload it, "
+ f"so this agent would silently answer with those weights; "
+ f"construct agents that want the base model before any tuned "
+ f"one, or run them in separate processes")
if self._weights and _active_weights != self._weights:
with open(self._weights, "rb") as handle:
blob = handle.read()
@@ -139,7 +145,8 @@ class Needle:
return response
def extract(self, text: str, schema: type | dict, max_new_tokens: int = 256) -> object:
- return extract(text, schema, max_new_tokens=max_new_tokens)
+ return extract(text, schema, max_new_tokens=max_new_tokens,
+ weights=self._weights)
def reset(self):
self._bind()
@@ -154,11 +161,13 @@ def _jsonable(value):
return str(value)
-def extract(text: str, schema: type | dict, system: str | None = None, max_new_tokens: int = 256) -> object:
+def extract(text: str, schema: type | dict, system: str | None = None,
+ max_new_tokens: int = 256, weights: str | None = None) -> object:
"""One-shot structured extraction: declare `schema` as the only tool and return
the parsed object (a Pydantic instance if `schema` is a model, else a dict).
- Re-initializes the shared engine with this single schema."""
- agent = Needle(tools=[schema], system=system)
+ Re-initializes the shared engine with this single schema. Defaults to whatever
+ weights are already loaded, since the engine cannot unload them."""
+ agent = Needle(tools=[schema], system=system, weights=weights or _active_weights)
response = agent.complete(text, max_new_tokens)
calls = response.get("function_calls") or []
if not calls:
diff --git a/tests/test_weights.py b/tests/test_weights.py
new file mode 100644
index 0000000..d00594a
--- /dev/null
+++ b/tests/test_weights.py
@@ -0,0 +1,96 @@
+import json
+import warnings
+
+import pytest
+
+ENVELOPE = json.dumps({"type": "call", "confidence": 0.9,
+ "function_calls": [{"name": "City",
+ "arguments": {"city": "Paris"}}]}).encode("utf-8")
+
+
+class _Stub:
+ def __init__(self, calls):
+ self.calls = calls
+
+ def needle_init(self, system, tools, index):
+ self.calls.append("init")
+ return 0
+
+ def needle_load(self, blob, size):
+ self.calls.append("load")
+ return 0
+
+ def needle_complete(self, text, max_new_tokens, buffer, size):
+ self.calls.append("complete")
+ buffer.value = ENVELOPE
+ return 0
+
+ def needle_reset(self):
+ self.calls.append("reset")
+
+
+def engine(monkeypatch):
+ import needle
+
+ calls = []
+ stub = _Stub(calls)
+ monkeypatch.setattr(needle, "_lib", lambda: stub)
+ monkeypatch.setattr(needle, "_active", None)
+ monkeypatch.setattr(needle, "_active_weights", None)
+ monkeypatch.setattr(needle, "_active_blob", None)
+ return calls
+
+
+def tuned(tmp_path):
+ path = tmp_path / "tuned.cact"
+ path.write_bytes(b"tuned weights")
+ return str(path)
+
+
+def _tuned_agent(path):
+ import needle
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ return needle.Needle(tools="[]", weights=path)
+
+
+def test_new_base_agent_refuses_once_tuned_weights_are_loaded(engine, tuned):
+ import needle
+
+ _tuned_agent(tuned)
+ with pytest.raises(RuntimeError, match="cannot unload"):
+ needle.Needle(tools="[]")
+
+
+def test_existing_base_agent_refuses_instead_of_answering_on_tuned_weights(engine, tuned):
+ import needle
+
+ base = needle.Needle(tools="[]")
+ base.complete("hello")
+ _tuned_agent(tuned)
+ with pytest.raises(RuntimeError, match="cannot unload"):
+ base.complete("hello")
+
+
+def test_tuned_agent_rebinds_without_reloading_its_own_weights(engine, tuned):
+ agent = _tuned_agent(tuned)
+ other = _tuned_agent(tuned)
+ other.complete("hello")
+ agent.complete("hello")
+ assert engine.count("load") == 1
+
+
+def test_extract_inherits_the_loaded_weights(engine, tuned):
+ import needle
+
+ _tuned_agent(tuned)
+ schema = {"name": "City", "parameters": {"type": "object",
+ "properties": {"city": {"type": "string"}}}}
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ assert needle.extract("the weather in Paris", schema) == {"city": "Paris"}
+ assert needle._active_weights == tuned
+ assert engine.count("load") == 1