Skip to content

Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.

fix(grounding): stop rejecting thousands-separated numbers (#123)

Som Samantray956840ff176b
diff --git a/needle/__init__.py b/needle/__init__.py
index a008020..29c5565 100644
--- a/needle/__init__.py
+++ b/needle/__init__.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import ctypes
+import decimal
import json
import os
import re
@@ -296,6 +297,11 @@ class Needle:
for call in calls:
name = str(call.get("name"))
fabricated = sorted(ungrounded.get(name, ()))
+ if strict and fabricated:
+ grounded = _grounded_number_paths(
+ call.get("arguments") or {}, query, self._system_text)
+ fabricated = [path for path in fabricated
+ if path not in grounded]
if strict and fabricated:
results.append({"error": "ungrounded " + ", ".join(fabricated)})
continue
@@ -434,6 +440,49 @@ def _temporal_grounding(text, schema, arguments, system=None):
return _walk_grounding(schema, arguments, years)
+_NUMBER_TOKEN = re.compile(
+ r"(?<![\w.,])(?:[-+]?\d{1,3}(?:,\d{3})+(?:\.\d+)?|[-+]?\d+(?:\.\d+)?)(?!\d)")
+
+
+def _numeric_paths(arguments, path=""):
+ """Yield ``(path, value)`` for every numeric leaf, arguments-relative.
+
+ Paths match the form both call sites derive after stripping the engine's
+ tool prefix, so a flagged ``Tool.path`` resolves to a helper path.
+ """
+ if isinstance(arguments, bool):
+ return
+ if isinstance(arguments, (int, float)):
+ yield path, arguments
+ elif isinstance(arguments, dict):
+ for key, item in arguments.items():
+ yield from _numeric_paths(item, f"{path}.{key}" if path else str(key))
+ elif isinstance(arguments, list):
+ for index, item in enumerate(arguments):
+ yield from _numeric_paths(item, f"{path}[{index}]")
+
+
+def _source_numbers(*sources):
+ """Numeric values written in the source texts, separators normalized."""
+ text = "\n".join(source for source in sources if source)
+ return {decimal.Decimal(match.group(0).replace(",", ""))
+ for match in _NUMBER_TOKEN.finditer(text)}
+
+
+def _grounded_number_paths(arguments, *sources):
+ numbers = _source_numbers(*sources)
+ if not numbers:
+ return set()
+ grounded = set()
+ for path, value in _numeric_paths(arguments):
+ try:
+ if decimal.Decimal(str(value)) in numbers:
+ grounded.add(path)
+ except (decimal.InvalidOperation, ValueError):
+ continue
+ return grounded
+
+
def _ungrounded_paths(response):
validation = response.get("validation") or {}
grouped = {}
@@ -468,8 +517,12 @@ def _annotate_ungrounded(response, tool_schemas, seen_years, system=None):
def _validate_extraction(text, schema, arguments, response, system=None):
checked, failures = _temporal_grounding(text, schema, arguments, system)
validation = response.get("validation") or {}
- for name in validation.get("ungrounded") or []:
+ flagged = validation.get("ungrounded") or []
+ grounded = _grounded_number_paths(arguments, text, system) if flagged else set()
+ for name in flagged:
path = name.split(".", 1)[-1]
+ if path in grounded:
+ continue
if path not in checked or path in failures:
failures.add(path)
if validation.get("negation"):
diff --git a/tests/test_grounding.py b/tests/test_grounding.py
index 9d0abca..df62771 100644
--- a/tests/test_grounding.py
+++ b/tests/test_grounding.py
@@ -1,5 +1,6 @@
import datetime
import json
+from typing import List
import pydantic
import pytest
@@ -42,6 +43,43 @@ def stub(monkeypatch):
class Invoice(pydantic.BaseModel):
vendor: str
due_date: datetime.date
+ total: float = 0.0
+
+
+class LineItem(pydantic.BaseModel):
+ price: float
+
+
+class Order(pydantic.BaseModel):
+ items: List[LineItem]
+
+
+def set_thermostat(temperature: int, mode: str = "auto"):
+ """Set the thermostat.
+
+ Args:
+ temperature: target temperature in Celsius
+ mode: heating strategy to use
+ """
+ return {"temperature": temperature, "mode": mode}
+
+
+def _invoice(total, due_date="2026-09-01"):
+ return {"vendor": "Acme Corp", "total": total, "due_date": due_date}
+
+
+def _envelope(arguments, name, ungrounded):
+ return {"type": "call", "confidence": 0.9,
+ "function_calls": [{"name": name, "arguments": arguments}],
+ "validation": {"ungrounded": list(ungrounded), "negation": False}}
+
+
+def _extract_envelope(arguments, ungrounded=("Invoice.total",), name="Invoice"):
+ return _envelope(arguments, name, ungrounded)
+
+
+def _run_envelope(arguments, ungrounded=("set_thermostat.temperature",)):
+ return _envelope(arguments, "set_thermostat", ungrounded)
def _call(due_date, name="Invoice"):
@@ -128,3 +166,171 @@ def test_engine_reported_fabrications_are_kept_and_block_execution(stub):
assert response["results"] == [{"error": "ungrounded vendor"}]
assert stub.calls[-1] == ("complete", json.dumps([{"error": "ungrounded vendor"}]))
+
+
+def test_extract_clears_a_thousands_separated_number(stub):
+ import needle
+
+ stub.envelopes = [_extract_envelope(_invoice(1200.0))]
+ invoice = needle.extract("Invoice from Acme Corp, $1,200.00, due 2026-09-01", Invoice)
+
+ assert invoice.total == 1200.0
+ assert invoice.due_date == datetime.date(2026, 9, 1)
+
+
+def test_extract_clears_a_thousands_separated_integer(stub):
+ import needle
+
+ stub.envelopes = [_extract_envelope(_invoice(1200))]
+ invoice = needle.extract("Invoice from Acme Corp, $1,200, due 2026-09-01", Invoice)
+
+ assert invoice.total == 1200
+
+
+def test_extract_leaves_an_unseparated_number_working(stub):
+ import needle
+
+ stub.envelopes = [_extract_envelope(_invoice(1200.0))]
+ invoice = needle.extract("Invoice from Acme Corp, $1200.00, due 2026-09-01", Invoice)
+
+ assert invoice.total == 1200.0
+
+
+def test_extract_still_rejects_a_fabricated_number(stub):
+ import needle
+
+ stub.envelopes = [_extract_envelope(_invoice(9999.0))]
+ with pytest.raises(needle.ExtractionValidationError) as excinfo:
+ needle.extract("Invoice from Acme Corp, $1,200.00, due 2026-09-01", Invoice)
+
+ assert "total" in str(excinfo.value)
+
+
+def test_extract_still_rejects_a_non_numeric_engine_flag(stub):
+ import needle
+
+ stub.envelopes = [_extract_envelope(_invoice(1200.0),
+ ungrounded=("Invoice.vendor",))]
+ with pytest.raises(needle.ExtractionValidationError) as excinfo:
+ needle.extract("Invoice from Acme Corp, $1,200.00, due 2026-09-01", Invoice)
+
+ assert "vendor" in str(excinfo.value)
+
+
+def test_extract_clears_a_nested_separated_number(stub):
+ import needle
+
+ stub.envelopes = [_extract_envelope({"items": [{"price": 1100.0}]},
+ ungrounded=("Order.items[0].price",),
+ name="Order")]
+ order = needle.extract("Order with one item priced $1,100.00", Order)
+
+ assert order.items[0].price == 1100.0
+
+
+def test_extract_does_not_clear_a_digit_run_inside_a_longer_number(stub):
+ import needle
+
+ for source in ("Invoice from Acme Corp, $12,000.00, due 2026-09-01",
+ "Invoice from Acme Corp, $11,200.00, due 2026-09-01",
+ "Invoice from Acme Corp, $1,2000.00, due 2026-09-01"):
+ stub.envelopes = [_extract_envelope(_invoice(1200.0))]
+ with pytest.raises(needle.ExtractionValidationError):
+ needle.extract(source, Invoice)
+
+
+def test_extract_does_not_clear_a_number_absent_from_the_source(stub):
+ import needle
+
+ stub.envelopes = [_extract_envelope(_invoice(1200.0))]
+ with pytest.raises(needle.ExtractionValidationError):
+ needle.extract("Invoice from Acme Corp, due 2026-09-01", Invoice)
+
+
+def test_extract_does_not_read_a_date_hyphen_as_a_minus_sign(stub):
+ import needle
+
+ for value in (-9.0, -1.0):
+ stub.envelopes = [_extract_envelope(_invoice(value))]
+ with pytest.raises(needle.ExtractionValidationError):
+ needle.extract("Invoice from Acme Corp, $1,200.00, due 2026-09-01",
+ Invoice)
+
+
+def test_extract_does_not_read_a_range_hyphen_as_a_minus_sign(stub):
+ import needle
+
+ stub.envelopes = [_extract_envelope(_invoice(-1400.0))]
+ with pytest.raises(needle.ExtractionValidationError):
+ needle.extract("Invoice from Acme Corp, $1,200.00-1,400.00, due 2026-09-01",
+ Invoice)
+
+
+def test_extract_clears_a_written_negative(stub):
+ import needle
+
+ stub.envelopes = [_extract_envelope(_invoice(-1200.0))]
+ invoice = needle.extract("Invoice from Acme Corp, -1,200.00, due 2026-09-01",
+ Invoice)
+
+ assert invoice.total == -1200.0
+
+
+def test_extract_clears_a_zero_against_its_source(stub):
+ import needle
+
+ stub.envelopes = [_extract_envelope(_invoice(0.0))]
+ invoice = needle.extract("Invoice from Acme Corp, $0.00, due 2026-09-01", Invoice)
+
+ assert invoice.total == 0.0
+
+
+def test_run_executes_a_grounded_number_and_refuses_an_ungrounded_sibling(stub):
+ import needle
+
+ envelope = {"type": "call", "confidence": 0.9,
+ "function_calls": [
+ {"name": "set_thermostat", "arguments": {"temperature": 21}},
+ {"name": "set_thermostat", "arguments": {"temperature": 22}}],
+ "validation": {"ungrounded": ["set_thermostat.temperature"],
+ "negation": False}}
+ stub.envelopes = [envelope, {"type": "respond", "function_calls": []}]
+ agent = needle.Needle(tools=[set_thermostat])
+ response = agent.run("make it 21 and cool the room")
+
+ assert response["results"] == [{"temperature": 21, "mode": "auto"},
+ {"error": "ungrounded temperature"}]
+
+
+def test_run_executes_a_separated_number_grounded_in_the_query(stub):
+ import needle
+
+ stub.envelopes = [_run_envelope({"temperature": 1200}),
+ {"type": "respond", "function_calls": []}]
+ agent = needle.Needle(tools=[set_thermostat])
+ response = agent.run("set it to 1,200")
+
+ assert response["results"] == [{"temperature": 1200, "mode": "auto"}]
+
+
+def test_run_refuses_a_number_absent_from_the_query(stub):
+ import needle
+
+ stub.envelopes = [_run_envelope({"temperature": 99}),
+ {"type": "respond", "function_calls": []}]
+ agent = needle.Needle(tools=[set_thermostat])
+ response = agent.run("make it 21 and cool the room")
+
+ assert response["results"] == [{"error": "ungrounded temperature"}]
+
+
+def test_run_still_refuses_a_non_numeric_engine_flag(stub):
+ import needle
+
+ stub.envelopes = [_run_envelope({"temperature": 21},
+ ungrounded=("set_thermostat.mode",)),
+ {"type": "respond", "function_calls": []}]
+ agent = needle.Needle(tools=[set_thermostat])
+ response = agent.run("make it 21 and cool the room")
+
+ assert response["results"] == [{"error": "ungrounded mode"}]