Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

Project Xander

HenryNdubuakub5e29432116a
diff --git a/doc/apis.md b/doc/apis.md
index 38837d2..e365ce1 100644
--- a/doc/apis.md
+++ b/doc/apis.md
@@ -3,13 +3,28 @@
| Call | What it does |
| --- | --- |
| `needle.Needle(tools=None, system=None, weights=None, tool_index_path=None, buffer_size=65536)` | Create an agent bound to one toolset. `tools` takes decorated functions, Pydantic models, raw JSON schema dicts, or a JSON string. `system` carries environment facts. `weights` loads a tuned `.cact`. `tool_index_path` persists tool embeddings for large catalogues. |
-| `agent.run(query, max_steps=8, max_new_tokens=256)` | Full agentic loop: the model picks calls, Needle executes your Python functions and feeds results back, and the final response carries the executed tool results as `results`. |
-| `agent.complete(text, max_new_tokens=256)` | One turn. You execute the call and feed the result back yourself via the next `complete(...)`. |
+| `agent.complete(text="", max_new_tokens=256, audio=None, audio_format="wav", sample_rate=0, channels=1)` | Complete a text, speech, or mixed turn. `audio` accepts WAV bytes or a path; PCM16 and float32 buffers use the accompanying format, rate, and channel fields. |
+| `agent.embed(text="", audio=None, audio_format="wav", sample_rate=0, channels=1)` | Embed text, serialized tool schemas, audio, or a mixed input with the Needle 3 retrieval head. |
+| `agent.run(query="", max_steps=8, max_new_tokens=256, audio=None, audio_format="wav", sample_rate=0, channels=1)` | Full agentic loop: the model picks calls, Needle executes your Python functions and feeds results back, and the final response carries the executed tool results as `results`. |
| `agent.reset()` | Rewind the conversation, keep the tools loaded. |
| `needle.tool` | Decorator that turns a function into a tool schema (attached as `fn._needle_tool`). |
| `needle.Field(...)` | Per argument constraints, attached inline with `typing.Annotated` or passed as a default. |
| `needle.extract(text, schema, system=None, max_new_tokens=256, weights=None, strict=True)` | One shot extraction. Returns a Pydantic instance if `schema` is a model, else a dict, or `None` if nothing matched. Strict mode rejects temporal values that contradict literal source years and engine-reported ungrounded values. |
+Audio tokenization is internal. Pass a WAV path or WAV bytes directly:
+
+```python
+agent.complete(audio="command.wav")
+agent.embed(audio=wav_bytes)
+```
+
+Microphone PCM can be passed without wrapping it in a WAV container:
+
+```python
+agent.complete(audio=pcm16_bytes, audio_format="pcm16",
+ sample_rate=48_000, channels=2)
+```
+
When one tuned archive is active, `extract(..., weights=None)` inherits it for backwards compatibility. If both a Needle 2 and Needle 3 tune are active, pass `weights=` explicitly; the client refuses to guess between generations.
## Declaring tools
diff --git a/needle/__init__.py b/needle/__init__.py
index 67c1f9b..ee0cfc3 100644
--- a/needle/__init__.py
+++ b/needle/__init__.py
@@ -76,13 +76,71 @@ _lib_handles = {}
_active = {}
+class _NeedleAudio(ctypes.Structure):
+ _fields_ = [
+ ("data", ctypes.c_void_p),
+ ("size", ctypes.c_uint64),
+ ("sample_rate", ctypes.c_int),
+ ("channels", ctypes.c_int),
+ ("format", ctypes.c_int),
+ ]
+
+
+_AUDIO_FORMATS = {"wav": 1, "pcm16": 2, "float32": 3}
+
+
+def _prepare_audio(audio, audio_format="wav", sample_rate=0, channels=1):
+ if audio is None:
+ return None
+ if isinstance(audio, (str, os.PathLike)):
+ with open(os.fspath(audio), "rb") as handle:
+ data = handle.read()
+ else:
+ try:
+ data = bytes(audio)
+ except (TypeError, ValueError) as exc:
+ raise TypeError("audio must be a path or bytes-like object") from exc
+ if not data:
+ raise ValueError("audio is empty")
+ try:
+ format_code = _AUDIO_FORMATS[str(audio_format).lower()]
+ except KeyError as exc:
+ raise ValueError("audio_format must be 'wav', 'pcm16', or 'float32'") from exc
+ if format_code != _AUDIO_FORMATS["wav"] and (
+ int(sample_rate) <= 0 or int(channels) <= 0):
+ raise ValueError("PCM audio requires positive sample_rate and channels")
+ return {"data": data, "format": format_code,
+ "sample_rate": int(sample_rate), "channels": int(channels)}
+
+
+def _native_audio(payload):
+ if payload is None:
+ return None, None
+ data = payload["data"]
+ storage = ctypes.create_string_buffer(data, len(data))
+ descriptor = _NeedleAudio(
+ ctypes.cast(storage, ctypes.c_void_p), len(data),
+ payload["sample_rate"], payload["channels"], payload["format"])
+ return ctypes.byref(descriptor), (storage, descriptor)
+
+
def _lib(generation=2):
generation = int(generation)
if generation not in _lib_handles:
lib = ctypes.CDLL(_library_path(generation))
lib.needle_init.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p]
lib.needle_init.restype = ctypes.c_int
- lib.needle_complete.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_int]
+ if generation >= 3:
+ lib.needle_complete.argtypes = [
+ ctypes.c_char_p, ctypes.POINTER(_NeedleAudio), ctypes.c_int,
+ ctypes.c_char_p, ctypes.c_int]
+ lib.needle_embed.argtypes = [
+ ctypes.c_char_p, ctypes.POINTER(_NeedleAudio),
+ ctypes.POINTER(ctypes.c_float), ctypes.c_int]
+ lib.needle_embed.restype = ctypes.c_int
+ else:
+ lib.needle_complete.argtypes = [
+ ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_int]
lib.needle_complete.restype = ctypes.c_int
lib.needle_reset.argtypes = []
lib.needle_reset.restype = None
@@ -117,7 +175,7 @@ class Needle:
_library_path(self._generation), self._weights,
self._system.decode("utf-8"), self._tools_json.decode("utf-8"),
os.fspath(tool_index_path) if tool_index_path else None,
- buffer_size)
+ buffer_size, generation=self._generation)
else:
self._bind()
@@ -154,21 +212,34 @@ class Needle:
return {"n_tools": self._n_tools, "tuned": bool(self._weights),
"generation": self._generation}
- def complete(self, text: str, max_new_tokens: int = 256) -> dict:
+ def complete(self, text: str = "", max_new_tokens: int = 256,
+ audio=None, audio_format="wav", sample_rate=0,
+ channels=1) -> dict:
_track("complete", self._track_props())
- return self._complete(text, max_new_tokens)
+ payload = _prepare_audio(audio, audio_format, sample_rate, channels)
+ return self._complete(text, max_new_tokens, payload)
- def _complete(self, text: str, max_new_tokens: int = 256) -> dict:
+ def _complete(self, text: str, max_new_tokens: int = 256,
+ audio=None) -> dict:
+ if audio is not None and self._generation < 3:
+ raise ValueError("audio requires a Needle 3 model")
self._bind()
if self._worker is not None:
- raw = self._worker.complete(text, max_new_tokens)
+ raw = self._worker.complete(text, max_new_tokens, audio)
else:
lib = _lib(self._generation)
- rc = lib.needle_complete(
- text.encode("utf-8"), int(max_new_tokens), self._buffer,
- len(self._buffer))
+ if self._generation >= 3:
+ native_audio, keepalive = _native_audio(audio)
+ rc = lib.needle_complete(
+ text.encode("utf-8"), native_audio, int(max_new_tokens),
+ self._buffer, len(self._buffer))
+ else:
+ rc = lib.needle_complete(
+ text.encode("utf-8"), int(max_new_tokens), self._buffer,
+ len(self._buffer))
if rc < 0:
- raise RuntimeError(f"needle_complete failed (code {rc})")
+ detail = self._buffer.value.decode("utf-8", "replace")
+ raise RuntimeError(detail or f"needle_complete failed (code {rc})")
raw = self._buffer.value.decode("utf-8")
try:
response = json.loads(raw)
@@ -180,9 +251,31 @@ class Needle:
response["confidence"] = None
return response
- def run(self, query: str, max_steps: int = 8, max_new_tokens: int = 256) -> dict:
+ def embed(self, text: str = "", audio=None, audio_format="wav",
+ sample_rate=0, channels=1) -> list[float]:
+ if self._generation < 3:
+ raise ValueError("embeddings require a Needle 3 model")
+ payload = _prepare_audio(audio, audio_format, sample_rate, channels)
+ self._bind()
+ if self._worker is not None:
+ return self._worker.embed(text, payload)
+ lib = _lib(self._generation)
+ native_audio, keepalive = _native_audio(payload)
+ dim = lib.needle_embed(text.encode("utf-8"), native_audio, None, 0)
+ if dim <= 0:
+ raise RuntimeError(f"needle_embed failed (code {dim})")
+ output = (ctypes.c_float * dim)()
+ rc = lib.needle_embed(text.encode("utf-8"), native_audio, output, dim)
+ if rc != dim:
+ raise RuntimeError(f"needle_embed failed (code {rc})")
+ return list(output)
+
+ def run(self, query: str = "", max_steps: int = 8,
+ max_new_tokens: int = 256, audio=None, audio_format="wav",
+ sample_rate=0, channels=1) -> dict:
_track("run", self._track_props())
- response = self._complete(query, max_new_tokens)
+ payload = _prepare_audio(audio, audio_format, sample_rate, channels)
+ response = self._complete(query, max_new_tokens, payload)
executed = []
for _ in range(max_steps):
calls = response.get("function_calls") or []
diff --git a/needle/_worker.py b/needle/_worker.py
index bb0ac8b..d950ecd 100644
--- a/needle/_worker.py
+++ b/needle/_worker.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import base64
import ctypes
import json
import os
@@ -14,6 +15,36 @@ _HEADER = struct.Struct("!Q")
_EOF = object()
+class _NeedleAudio(ctypes.Structure):
+ _fields_ = [
+ ("data", ctypes.c_void_p),
+ ("size", ctypes.c_uint64),
+ ("sample_rate", ctypes.c_int),
+ ("channels", ctypes.c_int),
+ ("format", ctypes.c_int),
+ ]
+
+
+def _native_audio(payload):
+ if not payload:
+ return None, None
+ data = base64.b64decode(payload["data"], validate=True)
+ storage = ctypes.create_string_buffer(data, len(data))
+ descriptor = _NeedleAudio(
+ ctypes.cast(storage, ctypes.c_void_p), len(data),
+ int(payload["sample_rate"]), int(payload["channels"]),
+ int(payload["format"]))
+ return ctypes.byref(descriptor), (storage, descriptor)
+
+
+def _wire_audio(payload):
+ if payload is None:
+ return None
+ return {"data": base64.b64encode(payload["data"]).decode("ascii"),
+ "sample_rate": payload["sample_rate"],
+ "channels": payload["channels"], "format": payload["format"]}
+
+
def _read_exact(stream, size):
chunks = []
remaining = size
@@ -47,13 +78,22 @@ def _write_message(stream, message):
stream.flush()
-def _load_library(path):
+def _load_library(path, generation=2):
lib = ctypes.CDLL(path)
lib.needle_init.argtypes = [ctypes.c_char_p, ctypes.c_char_p,
ctypes.c_char_p]
lib.needle_init.restype = ctypes.c_int
- lib.needle_complete.argtypes = [ctypes.c_char_p, ctypes.c_int,
- ctypes.c_char_p, ctypes.c_int]
+ if int(generation) >= 3:
+ lib.needle_complete.argtypes = [
+ ctypes.c_char_p, ctypes.POINTER(_NeedleAudio), ctypes.c_int,
+ ctypes.c_char_p, ctypes.c_int]
+ lib.needle_embed.argtypes = [
+ ctypes.c_char_p, ctypes.POINTER(_NeedleAudio),
+ ctypes.POINTER(ctypes.c_float), ctypes.c_int]
+ lib.needle_embed.restype = ctypes.c_int
+ else:
+ lib.needle_complete.argtypes = [ctypes.c_char_p, ctypes.c_int,
+ ctypes.c_char_p, ctypes.c_int]
lib.needle_complete.restype = ctypes.c_int
lib.needle_reset.argtypes = []
lib.needle_reset.restype = None
@@ -70,7 +110,8 @@ def _child():
config = _read_message(source)
if config is None:
return 1
- lib = _load_library(config["library"])
+ generation = int(config.get("generation", 2))
+ lib = _load_library(config["library"], generation)
with open(config["weights"], "rb") as handle:
weights = handle.read()
if lib.needle_load(weights, len(weights)) < 0:
@@ -90,9 +131,15 @@ def _child():
break
operation = request.get("operation")
if operation == "complete":
- code = lib.needle_complete(
- request["text"].encode("utf-8"),
- int(request["max_new_tokens"]), output, len(output))
+ if generation >= 3:
+ audio, keepalive = _native_audio(request.get("audio"))
+ code = lib.needle_complete(
+ request["text"].encode("utf-8"), audio,
+ int(request["max_new_tokens"]), output, len(output))
+ else:
+ code = lib.needle_complete(
+ request["text"].encode("utf-8"),
+ int(request["max_new_tokens"]), output, len(output))
if code < 0:
_write_message(protocol, {
"status": "error",
@@ -103,6 +150,34 @@ def _child():
"status": "ok",
"response": output.value.decode("utf-8"),
})
+ elif operation == "embed":
+ if generation < 3:
+ _write_message(protocol, {
+ "status": "error",
+ "message": "embeddings require a Needle 3 model",
+ })
+ continue
+ audio, keepalive = _native_audio(request.get("audio"))
+ text = request["text"].encode("utf-8")
+ dim = lib.needle_embed(text, audio, None, 0)
+ if dim <= 0:
+ _write_message(protocol, {
+ "status": "error",
+ "message": f"needle_embed failed (code {dim})",
+ })
+ continue
+ embedding = (ctypes.c_float * dim)()
+ code = lib.needle_embed(text, audio, embedding, dim)
+ if code != dim:
+ _write_message(protocol, {
+ "status": "error",
+ "message": f"needle_embed failed (code {code})",
+ })
+ else:
+ _write_message(protocol, {
+ "status": "ok",
+ "embedding": list(embedding),
+ })
elif operation == "reset":
lib.needle_reset()
_write_message(protocol, {"status": "ok"})
@@ -130,7 +205,7 @@ def _child():
class FineTuneWorker:
def __init__(self, library, weights, system, tools, tool_index,
- buffer_size, startup_timeout=300):
+ buffer_size, startup_timeout=300, generation=2):
self._lock = threading.Lock()
self._messages = queue.Queue()
self._closed = False
@@ -148,6 +223,7 @@ class FineTuneWorker:
"tools": tools,
"tool_index": tool_index,
"buffer_size": int(buffer_size),
+ "generation": int(generation),
})
ready = self._receive(startup_timeout)
if ready.get("status") != "ready":
@@ -205,13 +281,21 @@ class FineTuneWorker:
raise RuntimeError(response.get("message", "Needle worker failed"))
return response
- def complete(self, text, max_new_tokens):
+ def complete(self, text, max_new_tokens, audio=None):
return self._request({
"operation": "complete",
"text": text,
"max_new_tokens": int(max_new_tokens),
+ "audio": _wire_audio(audio),
})["response"]
+ def embed(self, text, audio=None):
+ return self._request({
+ "operation": "embed",
+ "text": text,
+ "audio": _wire_audio(audio),
+ })["embedding"]
+
def reset(self):
self._request({"operation": "reset"})
diff --git a/tests/test_weights.py b/tests/test_weights.py
index 437a8f1..b9b90d0 100644
--- a/tests/test_weights.py
+++ b/tests/test_weights.py
@@ -20,8 +20,9 @@ class _Stub:
self.calls.append("load")
return 0
- def needle_complete(self, text, max_new_tokens, buffer, size):
+ def needle_complete(self, text, *args):
self.calls.append("complete")
+ buffer = args[-2]
buffer.value = ENVELOPE
return 0
@@ -33,16 +34,25 @@ class _WorkerStub:
next_pid = 1000
def __init__(self, calls, library, weights, system, tools, tool_index,
- buffer_size):
+ buffer_size, generation=2):
self.calls = calls
self.pid = type(self).next_pid
type(self).next_pid += 1
self.calls.append(("worker_start", self.pid, library, weights))
- def complete(self, text, max_new_tokens):
- self.calls.append(("worker_complete", self.pid, text))
+ def complete(self, text, max_new_tokens, audio=None):
+ if audio:
+ self.calls.append(("worker_complete_audio", self.pid, text,
+ audio))
+ else:
+ self.calls.append(("worker_complete", self.pid, text))
return ENVELOPE.decode("utf-8")
+ def embed(self, text, audio=None):
+ self.calls.append(("worker_embed", self.pid, text,
+ audio))
+ return [0.6, 0.8]
+
def reset(self):
self.calls.append(("worker_reset", self.pid))
@@ -61,9 +71,10 @@ def engine(monkeypatch):
lambda generation=2: f"/tmp/libneedle{generation}")
monkeypatch.setattr(
needle, "FineTuneWorker",
- lambda library, weights, system, tools, tool_index, buffer_size:
+ lambda library, weights, system, tools, tool_index, buffer_size,
+ generation=2:
_WorkerStub(calls, library, weights, system, tools, tool_index,
- buffer_size))
+ buffer_size, generation))
monkeypatch.setattr(needle, "_active", {})
return calls
@@ -153,6 +164,16 @@ def test_v2_and_v3_weights_route_to_independent_engines(engine, tuned, tuned_v3,
assert routed == [2, 3]
+def test_v3_routes_audio_and_embeddings_to_its_worker(engine, tuned_v3):
+ agent = _tuned_agent(tuned_v3)
+ wav = b"RIFFordinary wav bytes"
+ agent.complete("transcribe", audio=wav)
+ assert agent.embed("schema", audio=wav) == [0.6, 0.8]
+ payload = {"data": wav, "format": 1, "sample_rate": 0, "channels": 1}
+ assert ("worker_complete_audio", agent._worker.pid, "transcribe", payload) in engine
+ assert ("worker_embed", agent._worker.pid, "schema", payload) in engine
+
+
def test_unknown_archive_tag_is_rejected_before_loading(engine, tmp_path):
path = tmp_path / "unknown.cact"
path.write_bytes(b"NOPE")