Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

test(environments): pin the suite scoring rule and document a red suite (#124)

Som Samantray42bf1f2d0a77
diff --git a/llms.txt b/llms.txt
index e8d9889..49c12d3 100644
--- a/llms.txt
+++ b/llms.txt
@@ -225,6 +225,8 @@ from needle.environments import smart_home
smart_home.agent.complete("dim the study lights to 30 percent")
```
+`run_tests()` returns `True` when at least `round(0.9 * len(TEST_CASES))` cases pass - 29 of the shipped 32 - and no `critical` case fails; `min_confidence` treats a call below the threshold as a refusal. A suite scores the model's output, not the harness, so a red suite usually reports the model's fit to the tool surface rather than a defect in the code that scores it (rule out a reply shape the harness does not expect before blaming the model). The threshold and the `critical` effect are pinned by engine-free tests in `tests/test_environments.py`. The shipped base model does not pass all six suites on engine 2.0.4 at the default gate: five fall short, missing calls the query did state and inventing values it did not.
+
Adapt by swapping the Literal values for your own and keeping the shapes: closed sets as enums, bounded numbers, verbatim copy for free text, five tools or fewer.
## CLI summary
diff --git a/tests/test_environments.py b/tests/test_environments.py
index e7b1d66..d50b5fa 100644
--- a/tests/test_environments.py
+++ b/tests/test_environments.py
@@ -1,9 +1,12 @@
+import json
import re
+import types
import pytest
from conftest import requires_engine
-from needle import environments
+from needle import environments, tool
+from needle.environments import _harness
CATEGORIES = {"positive", "missing", "irrelevant", "negation", "invalid", "parallel"}
@@ -70,3 +73,123 @@ def test_smart_home_smoke():
response = smart_home.agent.complete("turn on the kitchen lights")
assert response.get("type") in ("call", "text", "refuse")
+
+
+_MATCHING = {"name": "set_level", "arguments": {"level": 5}}
+_OTHER = {"name": "set_level", "arguments": {"level": 9}}
+_PASS = "pass"
+_LOW = "low"
+_FAIL = "fail"
+
+
+@tool
+def set_level(level: int):
+ """Set the level.
+
+ Args:
+ level: The level to set.
+ """
+ return {"ok": True, "level": level}
+
+
+class _Stub:
+ """Engine stand-in whose reply is chosen from the query, so a suite's
+ outcome is decided by its own cases rather than by a model."""
+
+ def needle_init(self, system, tools, index):
+ return 0
+
+ def needle_load(self, blob, size):
+ return 0
+
+ def needle_complete(self, text, *args):
+ # The output buffer is the second-to-last argument in every
+ # needle_complete signature, for both engine generations.
+ buffer = args[-2]
+ prompt = text.decode("utf-8").strip()
+ matched = not prompt.startswith(_FAIL)
+ confidence = 0.2 if prompt.startswith(_LOW) else 0.9
+ buffer.value = json.dumps({
+ "type": "call",
+ "confidence": confidence,
+ "function_calls": [_MATCHING if matched else _OTHER],
+ }).encode("utf-8")
+ return 0
+
+ def needle_reset(self):
+ pass
+
+
+def stub(monkeypatch):
+ import needle
+
+ engine = _Stub()
+ monkeypatch.setenv("NEEDLE_TELEMETRY", "0")
+ # agent_for sets this with setdefault, so claiming it here keeps the change
+ # from leaking into the rest of the process.
+ monkeypatch.setenv("NEEDLE_STRICT_VALIDATE", "1")
+ monkeypatch.setattr(needle, "_lib", lambda generation=2: engine)
+ monkeypatch.setattr(needle, "_active", {})
+ monkeypatch.setattr(_harness, "_agents", {})
+
+
+def _synthetic(name, passes, failures, critical=0):
+ """A suite whose queries drive the stub: a _PASS query is answered with the
+ call its case expects, a _FAIL query with a different one."""
+ cases = [{"query": f"{_PASS} {i}", "calls": [_MATCHING], "category": "positive"}
+ for i in range(passes)]
+ for i in range(failures):
+ case = {"query": f"{_FAIL} {i}", "calls": [_MATCHING], "category": "synthetic"}
+ if i < critical:
+ case["critical"] = True
+ cases.append(case)
+ return types.SimpleNamespace(__name__=name, TOOLS=[set_level], SYSTEM="",
+ TEST_CASES=cases)
+
+
+def test_suite_passes_at_29_of_32(stub):
+ assert _harness.run_tests(_synthetic("a", 29, 3), verbose=False) is True
+
+
+def test_suite_fails_at_28_of_32(stub):
+ assert _harness.run_tests(_synthetic("b", 28, 4), verbose=False) is False
+
+
+def test_one_critical_failure_fails_a_suite_that_clears_the_rate(stub):
+ assert _harness.run_tests(_synthetic("c", 31, 1, critical=1), verbose=False) is False
+
+
+def test_min_confidence_treats_a_low_confidence_call_as_a_refusal(stub):
+ module = types.SimpleNamespace(
+ __name__="d", TOOLS=[set_level], SYSTEM="",
+ TEST_CASES=[{"query": _LOW, "calls": [], "category": "missing"}])
+
+ assert _harness.run_tests(module, 0.0, verbose=False) is False
+ assert _harness.run_tests(module, 0.4, verbose=False) is True
+
+
+def test_min_confidence_keeps_a_call_at_the_threshold(stub):
+ module = types.SimpleNamespace(
+ __name__="e", TOOLS=[set_level], SYSTEM="",
+ TEST_CASES=[{"query": _PASS, "calls": [_MATCHING], "category": "positive"}])
+
+ # "at or above the threshold" is a strict comparison, so a call whose
+ # confidence equals the gate is acted on rather than treated as a refusal.
+ assert _harness.run_tests(module, 0.9, verbose=False) is True
+
+
+def test_aggregate_reports_failure_when_any_environment_fails(monkeypatch):
+ class _Module:
+ def __init__(self, ok):
+ self._ok = ok
+
+ def run_tests(self, min_confidence=0.0, verbose=True):
+ return self._ok
+
+ failing = environments._NAMES[-1]
+ monkeypatch.setattr(environments, "_load", lambda name: _Module(name != failing))
+ assert environments.run_tests(verbose=False) is False
+
+ monkeypatch.setattr(environments, "_load", lambda name: _Module(True))
+ assert environments.run_tests(verbose=False) is True