Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
test(environments): pin the suite scoring rule and document a red suite (#124)
Som Samantray42bf1f2d0a77
diff --git a/llms.txt b/llms.txtindex e8d9889..49c12d3 100644--- a/llms.txt+++ b/llms.txt@@ -225,6 +225,8 @@ from needle.environments import smart_homesmart_home.agent.complete("dim the study lights to 30 percent")```+`run_tests()` returns `True` when at least `round(0.9 * len(TEST_CASES))` cases pass - 29 of the shipped 32 - and no `critical` case fails; `min_confidence` treats a call below the threshold as a refusal. A suite scores the model's output, not the harness, so a red suite usually reports the model's fit to the tool surface rather than a defect in the code that scores it (rule out a reply shape the harness does not expect before blaming the model). The threshold and the `critical` effect are pinned by engine-free tests in `tests/test_environments.py`. The shipped base model does not pass all six suites on engine 2.0.4 at the default gate: five fall short, missing calls the query did state and inventing values it did not.+Adapt by swapping the Literal values for your own and keeping the shapes: closed sets as enums, bounded numbers, verbatim copy for free text, five tools or fewer.## CLI summarydiff --git a/tests/test_environments.py b/tests/test_environments.pyindex e7b1d66..d50b5fa 100644--- a/tests/test_environments.py+++ b/tests/test_environments.py@@ -1,9 +1,12 @@+import jsonimport re+import typesimport pytestfrom conftest import requires_engine-from needle import environments+from needle import environments, tool+from needle.environments import _harnessCATEGORIES = {"positive", "missing", "irrelevant", "negation", "invalid", "parallel"}@@ -70,3 +73,123 @@ def test_smart_home_smoke():response = smart_home.agent.complete("turn on the kitchen lights")assert response.get("type") in ("call", "text", "refuse")+++_MATCHING = {"name": "set_level", "arguments": {"level": 5}}+_OTHER = {"name": "set_level", "arguments": {"level": 9}}+_PASS = "pass"+_LOW = "low"+_FAIL = "fail"+++@tool+def set_level(level: int):+ """Set the level.++ Args:+ level: The level to set.+ """+ return {"ok": True, "level": level}+++class _Stub:+ """Engine stand-in whose reply is chosen from the query, so a suite's+ outcome is decided by its own cases rather than by a model."""++ def needle_init(self, system, tools, index):+ return 0++ def needle_load(self, blob, size):+ return 0++ def needle_complete(self, text, *args):+ # The output buffer is the second-to-last argument in every+ # needle_complete signature, for both engine generations.+ buffer = args[-2]+ prompt = text.decode("utf-8").strip()+ matched = not prompt.startswith(_FAIL)+ confidence = 0.2 if prompt.startswith(_LOW) else 0.9+ buffer.value = json.dumps({+ "type": "call",+ "confidence": confidence,+ "function_calls": [_MATCHING if matched else _OTHER],+ }).encode("utf-8")+ return 0++ def needle_reset(self):+ pass+++def stub(monkeypatch):+ import needle++ engine = _Stub()+ monkeypatch.setenv("NEEDLE_TELEMETRY", "0")+ # agent_for sets this with setdefault, so claiming it here keeps the change+ # from leaking into the rest of the process.+ monkeypatch.setenv("NEEDLE_STRICT_VALIDATE", "1")+ monkeypatch.setattr(needle, "_lib", lambda generation=2: engine)+ monkeypatch.setattr(needle, "_active", {})+ monkeypatch.setattr(_harness, "_agents", {})+++def _synthetic(name, passes, failures, critical=0):+ """A suite whose queries drive the stub: a _PASS query is answered with the+ call its case expects, a _FAIL query with a different one."""+ cases = [{"query": f"{_PASS} {i}", "calls": [_MATCHING], "category": "positive"}+ for i in range(passes)]+ for i in range(failures):+ case = {"query": f"{_FAIL} {i}", "calls": [_MATCHING], "category": "synthetic"}+ if i < critical:+ case["critical"] = True+ cases.append(case)+ return types.SimpleNamespace(__name__=name, TOOLS=[set_level], SYSTEM="",+ TEST_CASES=cases)+++def test_suite_passes_at_29_of_32(stub):+ assert _harness.run_tests(_synthetic("a", 29, 3), verbose=False) is True+++def test_suite_fails_at_28_of_32(stub):+ assert _harness.run_tests(_synthetic("b", 28, 4), verbose=False) is False+++def test_one_critical_failure_fails_a_suite_that_clears_the_rate(stub):+ assert _harness.run_tests(_synthetic("c", 31, 1, critical=1), verbose=False) is False+++def test_min_confidence_treats_a_low_confidence_call_as_a_refusal(stub):+ module = types.SimpleNamespace(+ __name__="d", TOOLS=[set_level], SYSTEM="",+ TEST_CASES=[{"query": _LOW, "calls": [], "category": "missing"}])++ assert _harness.run_tests(module, 0.0, verbose=False) is False+ assert _harness.run_tests(module, 0.4, verbose=False) is True+++def test_min_confidence_keeps_a_call_at_the_threshold(stub):+ module = types.SimpleNamespace(+ __name__="e", TOOLS=[set_level], SYSTEM="",+ TEST_CASES=[{"query": _PASS, "calls": [_MATCHING], "category": "positive"}])++ # "at or above the threshold" is a strict comparison, so a call whose+ # confidence equals the gate is acted on rather than treated as a refusal.+ assert _harness.run_tests(module, 0.9, verbose=False) is True+++def test_aggregate_reports_failure_when_any_environment_fails(monkeypatch):+ class _Module:+ def __init__(self, ok):+ self._ok = ok++ def run_tests(self, min_confidence=0.0, verbose=True):+ return self._ok++ failing = environments._NAMES[-1]+ monkeypatch.setattr(environments, "_load", lambda name: _Module(name != failing))+ assert environments.run_tests(verbose=False) is False++ monkeypatch.setattr(environments, "_load", lambda name: _Module(True))+ assert environments.run_tests(verbose=False) is True