Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
Project Xander
HenryNdubuakub5e29432116a
diff --git a/doc/apis.md b/doc/apis.mdindex 38837d2..e365ce1 100644--- a/doc/apis.md+++ b/doc/apis.md@@ -3,13 +3,28 @@| Call | What it does || --- | --- || `needle.Needle(tools=None, system=None, weights=None, tool_index_path=None, buffer_size=65536)` | Create an agent bound to one toolset. `tools` takes decorated functions, Pydantic models, raw JSON schema dicts, or a JSON string. `system` carries environment facts. `weights` loads a tuned `.cact`. `tool_index_path` persists tool embeddings for large catalogues. |-| `agent.run(query, max_steps=8, max_new_tokens=256)` | Full agentic loop: the model picks calls, Needle executes your Python functions and feeds results back, and the final response carries the executed tool results as `results`. |-| `agent.complete(text, max_new_tokens=256)` | One turn. You execute the call and feed the result back yourself via the next `complete(...)`. |+| `agent.complete(text="", max_new_tokens=256, audio=None, audio_format="wav", sample_rate=0, channels=1)` | Complete a text, speech, or mixed turn. `audio` accepts WAV bytes or a path; PCM16 and float32 buffers use the accompanying format, rate, and channel fields. |+| `agent.embed(text="", audio=None, audio_format="wav", sample_rate=0, channels=1)` | Embed text, serialized tool schemas, audio, or a mixed input with the Needle 3 retrieval head. |+| `agent.run(query="", max_steps=8, max_new_tokens=256, audio=None, audio_format="wav", sample_rate=0, channels=1)` | Full agentic loop: the model picks calls, Needle executes your Python functions and feeds results back, and the final response carries the executed tool results as `results`. || `agent.reset()` | Rewind the conversation, keep the tools loaded. || `needle.tool` | Decorator that turns a function into a tool schema (attached as `fn._needle_tool`). || `needle.Field(...)` | Per argument constraints, attached inline with `typing.Annotated` or passed as a default. || `needle.extract(text, schema, system=None, max_new_tokens=256, weights=None, strict=True)` | One shot extraction. Returns a Pydantic instance if `schema` is a model, else a dict, or `None` if nothing matched. Strict mode rejects temporal values that contradict literal source years and engine-reported ungrounded values. |+Audio tokenization is internal. Pass a WAV path or WAV bytes directly:++```python+agent.complete(audio="command.wav")+agent.embed(audio=wav_bytes)+```++Microphone PCM can be passed without wrapping it in a WAV container:++```python+agent.complete(audio=pcm16_bytes, audio_format="pcm16",+ sample_rate=48_000, channels=2)+```+When one tuned archive is active, `extract(..., weights=None)` inherits it for backwards compatibility. If both a Needle 2 and Needle 3 tune are active, pass `weights=` explicitly; the client refuses to guess between generations.## Declaring toolsdiff --git a/needle/__init__.py b/needle/__init__.pyindex 67c1f9b..ee0cfc3 100644--- a/needle/__init__.py+++ b/needle/__init__.py@@ -76,13 +76,71 @@ _lib_handles = {}_active = {}+class _NeedleAudio(ctypes.Structure):+ _fields_ = [+ ("data", ctypes.c_void_p),+ ("size", ctypes.c_uint64),+ ("sample_rate", ctypes.c_int),+ ("channels", ctypes.c_int),+ ("format", ctypes.c_int),+ ]+++_AUDIO_FORMATS = {"wav": 1, "pcm16": 2, "float32": 3}+++def _prepare_audio(audio, audio_format="wav", sample_rate=0, channels=1):+ if audio is None:+ return None+ if isinstance(audio, (str, os.PathLike)):+ with open(os.fspath(audio), "rb") as handle:+ data = handle.read()+ else:+ try:+ data = bytes(audio)+ except (TypeError, ValueError) as exc:+ raise TypeError("audio must be a path or bytes-like object") from exc+ if not data:+ raise ValueError("audio is empty")+ try:+ format_code = _AUDIO_FORMATS[str(audio_format).lower()]+ except KeyError as exc:+ raise ValueError("audio_format must be 'wav', 'pcm16', or 'float32'") from exc+ if format_code != _AUDIO_FORMATS["wav"] and (+ int(sample_rate) <= 0 or int(channels) <= 0):+ raise ValueError("PCM audio requires positive sample_rate and channels")+ return {"data": data, "format": format_code,+ "sample_rate": int(sample_rate), "channels": int(channels)}+++def _native_audio(payload):+ if payload is None:+ return None, None+ data = payload["data"]+ storage = ctypes.create_string_buffer(data, len(data))+ descriptor = _NeedleAudio(+ ctypes.cast(storage, ctypes.c_void_p), len(data),+ payload["sample_rate"], payload["channels"], payload["format"])+ return ctypes.byref(descriptor), (storage, descriptor)++def _lib(generation=2):generation = int(generation)if generation not in _lib_handles:lib = ctypes.CDLL(_library_path(generation))lib.needle_init.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p]lib.needle_init.restype = ctypes.c_int- lib.needle_complete.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_int]+ if generation >= 3:+ lib.needle_complete.argtypes = [+ ctypes.c_char_p, ctypes.POINTER(_NeedleAudio), ctypes.c_int,+ ctypes.c_char_p, ctypes.c_int]+ lib.needle_embed.argtypes = [+ ctypes.c_char_p, ctypes.POINTER(_NeedleAudio),+ ctypes.POINTER(ctypes.c_float), ctypes.c_int]+ lib.needle_embed.restype = ctypes.c_int+ else:+ lib.needle_complete.argtypes = [+ ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_int]lib.needle_complete.restype = ctypes.c_intlib.needle_reset.argtypes = []lib.needle_reset.restype = None@@ -117,7 +175,7 @@ class Needle:_library_path(self._generation), self._weights,self._system.decode("utf-8"), self._tools_json.decode("utf-8"),os.fspath(tool_index_path) if tool_index_path else None,- buffer_size)+ buffer_size, generation=self._generation)else:self._bind()@@ -154,21 +212,34 @@ class Needle:return {"n_tools": self._n_tools, "tuned": bool(self._weights),"generation": self._generation}- def complete(self, text: str, max_new_tokens: int = 256) -> dict:+ def complete(self, text: str = "", max_new_tokens: int = 256,+ audio=None, audio_format="wav", sample_rate=0,+ channels=1) -> dict:_track("complete", self._track_props())- return self._complete(text, max_new_tokens)+ payload = _prepare_audio(audio, audio_format, sample_rate, channels)+ return self._complete(text, max_new_tokens, payload)- def _complete(self, text: str, max_new_tokens: int = 256) -> dict:+ def _complete(self, text: str, max_new_tokens: int = 256,+ audio=None) -> dict:+ if audio is not None and self._generation < 3:+ raise ValueError("audio requires a Needle 3 model")self._bind()if self._worker is not None:- raw = self._worker.complete(text, max_new_tokens)+ raw = self._worker.complete(text, max_new_tokens, audio)else:lib = _lib(self._generation)- rc = lib.needle_complete(- text.encode("utf-8"), int(max_new_tokens), self._buffer,- len(self._buffer))+ if self._generation >= 3:+ native_audio, keepalive = _native_audio(audio)+ rc = lib.needle_complete(+ text.encode("utf-8"), native_audio, int(max_new_tokens),+ self._buffer, len(self._buffer))+ else:+ rc = lib.needle_complete(+ text.encode("utf-8"), int(max_new_tokens), self._buffer,+ len(self._buffer))if rc < 0:- raise RuntimeError(f"needle_complete failed (code {rc})")+ detail = self._buffer.value.decode("utf-8", "replace")+ raise RuntimeError(detail or f"needle_complete failed (code {rc})")raw = self._buffer.value.decode("utf-8")try:response = json.loads(raw)@@ -180,9 +251,31 @@ class Needle:response["confidence"] = Nonereturn response- def run(self, query: str, max_steps: int = 8, max_new_tokens: int = 256) -> dict:+ def embed(self, text: str = "", audio=None, audio_format="wav",+ sample_rate=0, channels=1) -> list[float]:+ if self._generation < 3:+ raise ValueError("embeddings require a Needle 3 model")+ payload = _prepare_audio(audio, audio_format, sample_rate, channels)+ self._bind()+ if self._worker is not None:+ return self._worker.embed(text, payload)+ lib = _lib(self._generation)+ native_audio, keepalive = _native_audio(payload)+ dim = lib.needle_embed(text.encode("utf-8"), native_audio, None, 0)+ if dim <= 0:+ raise RuntimeError(f"needle_embed failed (code {dim})")+ output = (ctypes.c_float * dim)()+ rc = lib.needle_embed(text.encode("utf-8"), native_audio, output, dim)+ if rc != dim:+ raise RuntimeError(f"needle_embed failed (code {rc})")+ return list(output)++ def run(self, query: str = "", max_steps: int = 8,+ max_new_tokens: int = 256, audio=None, audio_format="wav",+ sample_rate=0, channels=1) -> dict:_track("run", self._track_props())- response = self._complete(query, max_new_tokens)+ payload = _prepare_audio(audio, audio_format, sample_rate, channels)+ response = self._complete(query, max_new_tokens, payload)executed = []for _ in range(max_steps):calls = response.get("function_calls") or []diff --git a/needle/_worker.py b/needle/_worker.pyindex bb0ac8b..d950ecd 100644--- a/needle/_worker.py+++ b/needle/_worker.py@@ -1,5 +1,6 @@from __future__ import annotations+import base64import ctypesimport jsonimport os@@ -14,6 +15,36 @@ _HEADER = struct.Struct("!Q")_EOF = object()+class _NeedleAudio(ctypes.Structure):+ _fields_ = [+ ("data", ctypes.c_void_p),+ ("size", ctypes.c_uint64),+ ("sample_rate", ctypes.c_int),+ ("channels", ctypes.c_int),+ ("format", ctypes.c_int),+ ]+++def _native_audio(payload):+ if not payload:+ return None, None+ data = base64.b64decode(payload["data"], validate=True)+ storage = ctypes.create_string_buffer(data, len(data))+ descriptor = _NeedleAudio(+ ctypes.cast(storage, ctypes.c_void_p), len(data),+ int(payload["sample_rate"]), int(payload["channels"]),+ int(payload["format"]))+ return ctypes.byref(descriptor), (storage, descriptor)+++def _wire_audio(payload):+ if payload is None:+ return None+ return {"data": base64.b64encode(payload["data"]).decode("ascii"),+ "sample_rate": payload["sample_rate"],+ "channels": payload["channels"], "format": payload["format"]}++def _read_exact(stream, size):chunks = []remaining = size@@ -47,13 +78,22 @@ def _write_message(stream, message):stream.flush()-def _load_library(path):+def _load_library(path, generation=2):lib = ctypes.CDLL(path)lib.needle_init.argtypes = [ctypes.c_char_p, ctypes.c_char_p,ctypes.c_char_p]lib.needle_init.restype = ctypes.c_int- lib.needle_complete.argtypes = [ctypes.c_char_p, ctypes.c_int,- ctypes.c_char_p, ctypes.c_int]+ if int(generation) >= 3:+ lib.needle_complete.argtypes = [+ ctypes.c_char_p, ctypes.POINTER(_NeedleAudio), ctypes.c_int,+ ctypes.c_char_p, ctypes.c_int]+ lib.needle_embed.argtypes = [+ ctypes.c_char_p, ctypes.POINTER(_NeedleAudio),+ ctypes.POINTER(ctypes.c_float), ctypes.c_int]+ lib.needle_embed.restype = ctypes.c_int+ else:+ lib.needle_complete.argtypes = [ctypes.c_char_p, ctypes.c_int,+ ctypes.c_char_p, ctypes.c_int]lib.needle_complete.restype = ctypes.c_intlib.needle_reset.argtypes = []lib.needle_reset.restype = None@@ -70,7 +110,8 @@ def _child():config = _read_message(source)if config is None:return 1- lib = _load_library(config["library"])+ generation = int(config.get("generation", 2))+ lib = _load_library(config["library"], generation)with open(config["weights"], "rb") as handle:weights = handle.read()if lib.needle_load(weights, len(weights)) < 0:@@ -90,9 +131,15 @@ def _child():breakoperation = request.get("operation")if operation == "complete":- code = lib.needle_complete(- request["text"].encode("utf-8"),- int(request["max_new_tokens"]), output, len(output))+ if generation >= 3:+ audio, keepalive = _native_audio(request.get("audio"))+ code = lib.needle_complete(+ request["text"].encode("utf-8"), audio,+ int(request["max_new_tokens"]), output, len(output))+ else:+ code = lib.needle_complete(+ request["text"].encode("utf-8"),+ int(request["max_new_tokens"]), output, len(output))if code < 0:_write_message(protocol, {"status": "error",@@ -103,6 +150,34 @@ def _child():"status": "ok","response": output.value.decode("utf-8"),})+ elif operation == "embed":+ if generation < 3:+ _write_message(protocol, {+ "status": "error",+ "message": "embeddings require a Needle 3 model",+ })+ continue+ audio, keepalive = _native_audio(request.get("audio"))+ text = request["text"].encode("utf-8")+ dim = lib.needle_embed(text, audio, None, 0)+ if dim <= 0:+ _write_message(protocol, {+ "status": "error",+ "message": f"needle_embed failed (code {dim})",+ })+ continue+ embedding = (ctypes.c_float * dim)()+ code = lib.needle_embed(text, audio, embedding, dim)+ if code != dim:+ _write_message(protocol, {+ "status": "error",+ "message": f"needle_embed failed (code {code})",+ })+ else:+ _write_message(protocol, {+ "status": "ok",+ "embedding": list(embedding),+ })elif operation == "reset":lib.needle_reset()_write_message(protocol, {"status": "ok"})@@ -130,7 +205,7 @@ def _child():class FineTuneWorker:def __init__(self, library, weights, system, tools, tool_index,- buffer_size, startup_timeout=300):+ buffer_size, startup_timeout=300, generation=2):self._lock = threading.Lock()self._messages = queue.Queue()self._closed = False@@ -148,6 +223,7 @@ class FineTuneWorker:"tools": tools,"tool_index": tool_index,"buffer_size": int(buffer_size),+ "generation": int(generation),})ready = self._receive(startup_timeout)if ready.get("status") != "ready":@@ -205,13 +281,21 @@ class FineTuneWorker:raise RuntimeError(response.get("message", "Needle worker failed"))return response- def complete(self, text, max_new_tokens):+ def complete(self, text, max_new_tokens, audio=None):return self._request({"operation": "complete","text": text,"max_new_tokens": int(max_new_tokens),+ "audio": _wire_audio(audio),})["response"]+ def embed(self, text, audio=None):+ return self._request({+ "operation": "embed",+ "text": text,+ "audio": _wire_audio(audio),+ })["embedding"]+def reset(self):self._request({"operation": "reset"})diff --git a/tests/test_weights.py b/tests/test_weights.pyindex 437a8f1..b9b90d0 100644--- a/tests/test_weights.py+++ b/tests/test_weights.py@@ -20,8 +20,9 @@ class _Stub:self.calls.append("load")return 0- def needle_complete(self, text, max_new_tokens, buffer, size):+ def needle_complete(self, text, *args):self.calls.append("complete")+ buffer = args[-2]buffer.value = ENVELOPEreturn 0@@ -33,16 +34,25 @@ class _WorkerStub:next_pid = 1000def __init__(self, calls, library, weights, system, tools, tool_index,- buffer_size):+ buffer_size, generation=2):self.calls = callsself.pid = type(self).next_pidtype(self).next_pid += 1self.calls.append(("worker_start", self.pid, library, weights))- def complete(self, text, max_new_tokens):- self.calls.append(("worker_complete", self.pid, text))+ def complete(self, text, max_new_tokens, audio=None):+ if audio:+ self.calls.append(("worker_complete_audio", self.pid, text,+ audio))+ else:+ self.calls.append(("worker_complete", self.pid, text))return ENVELOPE.decode("utf-8")+ def embed(self, text, audio=None):+ self.calls.append(("worker_embed", self.pid, text,+ audio))+ return [0.6, 0.8]+def reset(self):self.calls.append(("worker_reset", self.pid))@@ -61,9 +71,10 @@ def engine(monkeypatch):lambda generation=2: f"/tmp/libneedle{generation}")monkeypatch.setattr(needle, "FineTuneWorker",- lambda library, weights, system, tools, tool_index, buffer_size:+ lambda library, weights, system, tools, tool_index, buffer_size,+ generation=2:_WorkerStub(calls, library, weights, system, tools, tool_index,- buffer_size))+ buffer_size, generation))monkeypatch.setattr(needle, "_active", {})return calls@@ -153,6 +164,16 @@ def test_v2_and_v3_weights_route_to_independent_engines(engine, tuned, tuned_v3,assert routed == [2, 3]+def test_v3_routes_audio_and_embeddings_to_its_worker(engine, tuned_v3):+ agent = _tuned_agent(tuned_v3)+ wav = b"RIFFordinary wav bytes"+ agent.complete("transcribe", audio=wav)+ assert agent.embed("schema", audio=wav) == [0.6, 0.8]+ payload = {"data": wav, "format": 1, "sample_rate": 0, "channels": 1}+ assert ("worker_complete_audio", agent._worker.pid, "transcribe", payload) in engine+ assert ("worker_embed", agent._worker.pid, "schema", payload) in engine++def test_unknown_archive_tag_is_rejected_before_loading(engine, tmp_path):path = tmp_path / "unknown.cact"path.write_bytes(b"NOPE")