Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

feat: enhance stateless agent functionality and improve libc detection

HenryNdubuaku475eb9b1708f
diff --git a/needle/__init__.py b/needle/__init__.py
index 9d3e7e5..973897b 100644
--- a/needle/__init__.py
+++ b/needle/__init__.py
@@ -21,6 +21,8 @@ class ExtractionValidationError(ValueError):
"""The engine produced structured values that are not grounded in the input."""
+UNRESET_TURNS = 4
+
_CACT_GENERATIONS = {
0x05E12A83: 2,
0x05E12A84: 3,
@@ -133,10 +135,36 @@ def _load_base(lib, generation):
_active.pop(generation, None)
+def _load_cdll(generation):
+ """Load the engine, retrying with the other libc build if the first is wrong.
+
+ A musl system that reports itself as glibc (or the reverse) otherwise fails
+ at load with a missing symbol such as strtoll_l.
+ """
+ path = _library_path(generation)
+ try:
+ return ctypes.CDLL(path)
+ except OSError as first:
+ from .agent import fetch
+
+ other = fetch.other_libc_tag()
+ if other is None:
+ raise
+ try:
+ alt = fetch.fetch_library(dest_dir=os.path.dirname(path) or ".",
+ tag=other, generation=generation)
+ handle = ctypes.CDLL(alt)
+ except Exception:
+ raise first from None
+ warnings.warn(f"the {fetch._platform_tag()} engine did not load ({first}); "
+ f"using the {other} build instead", stacklevel=3)
+ return handle
+
+
def _lib(generation=2):
generation = int(generation)
if generation not in _lib_handles:
- lib = ctypes.CDLL(_library_path(generation))
+ lib = _load_cdll(generation)
lib.needle_init.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p]
lib.needle_init.restype = ctypes.c_int
lib.needle_complete.argtypes = [
@@ -156,8 +184,10 @@ def _lib(generation=2):
class Needle:
def __init__(self, tools=None, system=None, weights=None, tool_index_path=None, buffer_size=65536,
- auto_date=True, generation=None):
+ auto_date=True, generation=None, stateless=False):
self._functions = {}
+ self._stateless = bool(stateless)
+ self._turns = 0
self._weights = os.fspath(weights) if weights is not None else None
self._tuned = self._weights is not None
if self._tuned:
@@ -233,8 +263,20 @@ class Needle:
def complete(self, text: str = "", max_new_tokens: int = 512) -> dict:
_track("complete", self._track_props())
+ if self._stateless:
+ self.reset()
+ self._count_query()
return self._complete(text, max_new_tokens)
+ def _count_query(self):
+ self._turns += 1
+ if self._turns == UNRESET_TURNS + 1:
+ warnings.warn(
+ f"{UNRESET_TURNS} queries on this agent without reset(): every turn stays in "
+ "the conversation, so unrelated queries lose accuracy and confidence. "
+ "Call reset() between independent queries, or construct with stateless=True",
+ stacklevel=3)
+
def _complete(self, text: str, max_new_tokens: int = 512,
ground: bool = True) -> dict:
self._bind()
@@ -282,6 +324,9 @@ class Needle:
def run(self, query: str = "", max_steps: int = 8,
max_new_tokens: int = 512, strict: bool = True) -> dict:
_track("run", self._track_props())
+ if self._stateless:
+ self.reset()
+ self._count_query()
response = self._complete(query, max_new_tokens)
executed = []
for _ in range(max_steps):
@@ -322,6 +367,7 @@ class Needle:
def reset(self):
self._bind()
+ self._turns = 0
if self._worker is not None:
self._worker.reset()
else:
diff --git a/needle/agent/fetch.py b/needle/agent/fetch.py
index 99baeee..210d2da 100644
--- a/needle/agent/fetch.py
+++ b/needle/agent/fetch.py
@@ -1,5 +1,6 @@
import os
import platform
+import sysconfig
import sys
import zipfile
@@ -45,16 +46,31 @@ def _lib_name():
def _is_musl():
+ """True on a musl system (Alpine and friends).
+
+ platform.libc_ver() scans the interpreter binary and reports glibc on some
+ musl builds, so the loader and the filesystem are asked first.
+ """
if sys.platform != "linux":
return False
- libc, _ = platform.libc_ver()
- if libc:
- return False
try:
with open("/proc/self/maps", "rb") as maps:
- return b"musl" in maps.read()
+ blob = maps.read()
+ if b"musl" in blob:
+ return True
+ if b"/libc.so.6" in blob or b"/ld-linux" in blob:
+ return False
except OSError:
+ pass
+ import glob
+ if glob.glob("/lib/ld-musl-*.so.1"):
return True
+ if glob.glob("/lib*/libc.so.6") or glob.glob("/lib/*-linux-gnu/libc.so.6"):
+ return False
+ if "musl" in (sysconfig.get_config_var("HOST_GNU_TYPE") or ""):
+ return True
+ libc, _ = platform.libc_ver()
+ return not libc
def _platform_tag():
@@ -69,6 +85,16 @@ def _platform_tag():
return family + arch
+def other_libc_tag():
+ """The tag for the other Linux libc family, or None off Linux."""
+ tag = _platform_tag()
+ if tag.startswith("manylinux2014_"):
+ return tag.replace("manylinux2014_", "musllinux_1_2_")
+ if tag.startswith("musllinux_1_2_"):
+ return tag.replace("musllinux_1_2_", "manylinux2014_")
+ return None
+
+
def engine_repo(generation=2):
try:
return ENGINE_REPOS[int(generation)]
diff --git a/needle/model/finetune.py b/needle/model/finetune.py
index ee64b24..9a0c212 100644
--- a/needle/model/finetune.py
+++ b/needle/model/finetune.py
@@ -343,6 +343,48 @@ def merge_lora(params, lora, scale):
return unflatten_dict(flat)
+def _calls_of(text):
+ """The tool calls in a rendered completion, or None when there is no call block."""
+ if TOOL_CALL_START not in text:
+ return []
+ body = text.split(TOOL_CALL_START, 1)[1].split(TOOL_CALL_END)[0]
+ try:
+ calls = json.loads(body)
+ except json.JSONDecodeError:
+ return None
+ if not isinstance(calls, list):
+ return None
+ return [(str(c.get("name")), json.dumps(c.get("arguments") or {}, sort_keys=True))
+ for c in calls if isinstance(c, dict)]
+
+
+def _score_quantised(model, params, lora, scale, tokenizer, data_path, n_val, rng_seed=0):
+ """Exact-call accuracy of the held-out split under the export's numerics.
+
+ The float model is not what ships: `needle build` writes CQ W4 weights, so a
+ fine-tune is scored through the same quantisation the archive uses.
+ """
+ import numpy as np
+ from .run import generate
+ from .quantize import cq_ste_params, WEIGHT_BITS
+
+ examples = list(read_examples(data_path))
+ if not examples:
+ return 0, 0
+ order = np.random.default_rng(rng_seed).permutation(len(examples))
+ held = [examples[i] for i in order[:n_val]]
+ merged = cq_ste_params(merge_lora(params, lora, scale), WEIGHT_BITS)
+ correct = 0
+ for example in held:
+ prompt, _ = render_example({**example, "answers": []})
+ text = generate(model, merged, tokenizer, prompt, max_new_tokens=96,
+ temperature=0.0, stream=False)
+ want = _calls_of(TOOL_CALL_START + json.dumps(
+ example.get("answers", []), separators=(",", ":")) + TOOL_CALL_END)
+ correct += _calls_of(text) == want
+ return correct, len(held)
+
+
def finetune_local(args, progress=None):
import jax
import jax.numpy as jnp
@@ -446,6 +488,13 @@ def finetune_local(args, progress=None):
else:
emit(f" {'epoch':<9} {epoch + 1}/{args.epochs} loss {last:.4f}")
+ if n_val > 0 and getattr(args, "score", True):
+ correct, total = _score_quantised(model, params, lora, scale, tokenizer,
+ data_path, n_val, rng_seed=seed)
+ if total:
+ emit(f" {'accuracy':<9} {correct}/{total} held-out calls exact, scored on the "
+ f"W{WEIGHT_BITS} weights the archive ships")
+
os.makedirs(args.checkpoint_dir, exist_ok=True)
out = args.out or os.path.join(args.checkpoint_dir, "needle_lora.safetensors")
write_adapter(out, {
diff --git a/tests/test_fetch.py b/tests/test_fetch.py
index eeaf876..159b5eb 100644
--- a/tests/test_fetch.py
+++ b/tests/test_fetch.py
@@ -1,3 +1,7 @@
+import sys
+
+import pytest
+
def test_lib_path_env_override(tmp_path, monkeypatch):
import needle
@@ -157,3 +161,70 @@ def test_engine_gate_honours_the_library_override(tmp_path, monkeypatch):
monkeypatch.setenv("NEEDLE3_LIB_PATH", str(tmp_path / "gone"))
assert not _engine_available()
+
+
+def test_musl_detected_when_libc_ver_claims_glibc(monkeypatch, tmp_path):
+ import platform
+ from needle.agent import fetch
+
+ monkeypatch.setattr(platform, "libc_ver", lambda *a, **k: ("glibc", "2.9"))
+ monkeypatch.setattr(sys, "platform", "linux")
+ maps = tmp_path / "maps"
+ maps.write_bytes(b"7f0000000000-7f0000001000 r-xp /lib/ld-musl-x86_64.so.1\n")
+ real_open = open
+ monkeypatch.setattr("builtins.open",
+ lambda p, *a, **k: real_open(maps, *a, **k) if p == "/proc/self/maps"
+ else real_open(p, *a, **k))
+ assert fetch._is_musl() is True
+
+
+def test_glibc_detected_from_maps(monkeypatch, tmp_path):
+ import platform
+ from needle.agent import fetch
+
+ monkeypatch.setattr(platform, "libc_ver", lambda *a, **k: ("", ""))
+ monkeypatch.setattr(sys, "platform", "linux")
+ maps = tmp_path / "maps"
+ maps.write_bytes(b"7f0000000000-7f0000001000 r-xp /usr/lib/x86_64-linux-gnu/libc.so.6\n")
+ real_open = open
+ monkeypatch.setattr("builtins.open",
+ lambda p, *a, **k: real_open(maps, *a, **k) if p == "/proc/self/maps"
+ else real_open(p, *a, **k))
+ assert fetch._is_musl() is False
+
+
+def test_other_libc_tag_swaps_families(monkeypatch):
+ from needle.agent import fetch
+
+ monkeypatch.setattr(fetch, "_platform_tag", lambda: "manylinux2014_aarch64")
+ assert fetch.other_libc_tag() == "musllinux_1_2_aarch64"
+ monkeypatch.setattr(fetch, "_platform_tag", lambda: "musllinux_1_2_x86_64")
+ assert fetch.other_libc_tag() == "manylinux2014_x86_64"
+ monkeypatch.setattr(fetch, "_platform_tag", lambda: "macosx_11_0_arm64")
+ assert fetch.other_libc_tag() is None
+
+
+def test_engine_load_falls_back_to_the_other_libc(monkeypatch, tmp_path):
+ import ctypes
+ import needle
+ from needle.agent import fetch
+
+ good = tmp_path / "musl" / "libneedle.so"
+ good.parent.mkdir()
+ good.write_bytes(b"x")
+ monkeypatch.setattr(needle, "_library_path", lambda generation=2: str(tmp_path / "libneedle.so"))
+ monkeypatch.setattr(fetch, "other_libc_tag", lambda: "musllinux_1_2_x86_64")
+ monkeypatch.setattr(fetch, "fetch_library",
+ lambda version=None, dest_dir=None, tag=None, generation=2: str(good))
+ loaded = {}
+
+ def fake_cdll(path):
+ if path.endswith("musl/libneedle.so"):
+ loaded["path"] = path
+ return "handle"
+ raise OSError("Error relocating libneedle.so: strtoll_l: symbol not found")
+
+ monkeypatch.setattr(ctypes, "CDLL", fake_cdll)
+ with pytest.warns(UserWarning, match="using the musllinux"):
+ assert needle._load_cdll(3) == "handle"
+ assert loaded["path"].endswith("musl/libneedle.so")
diff --git a/tests/test_weights.py b/tests/test_weights.py
index 6f15696..7c703cf 100644
--- a/tests/test_weights.py
+++ b/tests/test_weights.py
@@ -291,3 +291,49 @@ def test_platform_tuned_archive_keeps_confidence(engine, tmp_path, fake_cact):
with pytest.warns(UserWarning, match="confidence as None"):
uncalibrated = needle.Needle(tools="[]", weights=str(tmp_path / "local.cact"))
assert uncalibrated.complete("hello").get("confidence", "kept") is None
+
+
+def test_stateless_agent_resets_before_every_query(engine):
+ import needle
+
+ agent = needle.Needle(tools="[]", generation=3, stateless=True)
+ agent.complete("one")
+ agent.complete("two")
+ resets = [c for c in engine if c == "reset"]
+ assert len(resets) == 2
+
+
+def test_unreset_queries_warn_once(engine):
+ import needle
+
+ agent = needle.Needle(tools="[]", generation=3)
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ for _ in range(needle.UNRESET_TURNS):
+ agent.complete("q")
+ with pytest.warns(UserWarning, match="without reset"):
+ agent.complete("q")
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ agent.complete("q")
+ agent.reset()
+ for _ in range(needle.UNRESET_TURNS):
+ agent.complete("q")
+
+
+def test_tool_result_turns_do_not_count_as_queries(engine):
+ import needle
+
+ calls = []
+
+ @needle.tool
+ def ping(x: str):
+ "Ping."
+ calls.append(x)
+ return {"ok": True}
+
+ agent = needle.Needle(tools=[ping], generation=3)
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ for _ in range(needle.UNRESET_TURNS):
+ agent.run("go")