Automation foundation model for tiny devices: 2-bit, 8-29 MB, tool calls, ASR, structured extraction and embeddings on phones, wearables, smart homes, robots, cars and microcontrollers.
fix(grounding): stop rejecting thousands-separated numbers (#123)
Som Samantray956840ff176b
diff --git a/needle/__init__.py b/needle/__init__.pyindex a008020..29c5565 100644--- a/needle/__init__.py+++ b/needle/__init__.py@@ -1,6 +1,7 @@from __future__ import annotationsimport ctypes+import decimalimport jsonimport osimport re@@ -296,6 +297,11 @@ class Needle:for call in calls:name = str(call.get("name"))fabricated = sorted(ungrounded.get(name, ()))+ if strict and fabricated:+ grounded = _grounded_number_paths(+ call.get("arguments") or {}, query, self._system_text)+ fabricated = [path for path in fabricated+ if path not in grounded]if strict and fabricated:results.append({"error": "ungrounded " + ", ".join(fabricated)})continue@@ -434,6 +440,49 @@ def _temporal_grounding(text, schema, arguments, system=None):return _walk_grounding(schema, arguments, years)+_NUMBER_TOKEN = re.compile(+ r"(?<![\w.,])(?:[-+]?\d{1,3}(?:,\d{3})+(?:\.\d+)?|[-+]?\d+(?:\.\d+)?)(?!\d)")+++def _numeric_paths(arguments, path=""):+ """Yield ``(path, value)`` for every numeric leaf, arguments-relative.++ Paths match the form both call sites derive after stripping the engine's+ tool prefix, so a flagged ``Tool.path`` resolves to a helper path.+ """+ if isinstance(arguments, bool):+ return+ if isinstance(arguments, (int, float)):+ yield path, arguments+ elif isinstance(arguments, dict):+ for key, item in arguments.items():+ yield from _numeric_paths(item, f"{path}.{key}" if path else str(key))+ elif isinstance(arguments, list):+ for index, item in enumerate(arguments):+ yield from _numeric_paths(item, f"{path}[{index}]")+++def _source_numbers(*sources):+ """Numeric values written in the source texts, separators normalized."""+ text = "\n".join(source for source in sources if source)+ return {decimal.Decimal(match.group(0).replace(",", ""))+ for match in _NUMBER_TOKEN.finditer(text)}+++def _grounded_number_paths(arguments, *sources):+ numbers = _source_numbers(*sources)+ if not numbers:+ return set()+ grounded = set()+ for path, value in _numeric_paths(arguments):+ try:+ if decimal.Decimal(str(value)) in numbers:+ grounded.add(path)+ except (decimal.InvalidOperation, ValueError):+ continue+ return grounded++def _ungrounded_paths(response):validation = response.get("validation") or {}grouped = {}@@ -468,8 +517,12 @@ def _annotate_ungrounded(response, tool_schemas, seen_years, system=None):def _validate_extraction(text, schema, arguments, response, system=None):checked, failures = _temporal_grounding(text, schema, arguments, system)validation = response.get("validation") or {}- for name in validation.get("ungrounded") or []:+ flagged = validation.get("ungrounded") or []+ grounded = _grounded_number_paths(arguments, text, system) if flagged else set()+ for name in flagged:path = name.split(".", 1)[-1]+ if path in grounded:+ continueif path not in checked or path in failures:failures.add(path)if validation.get("negation"):diff --git a/tests/test_grounding.py b/tests/test_grounding.pyindex 9d0abca..df62771 100644--- a/tests/test_grounding.py+++ b/tests/test_grounding.py@@ -1,5 +1,6 @@import datetimeimport json+from typing import Listimport pydanticimport pytest@@ -42,6 +43,43 @@ def stub(monkeypatch):class Invoice(pydantic.BaseModel):vendor: strdue_date: datetime.date+ total: float = 0.0+++class LineItem(pydantic.BaseModel):+ price: float+++class Order(pydantic.BaseModel):+ items: List[LineItem]+++def set_thermostat(temperature: int, mode: str = "auto"):+ """Set the thermostat.++ Args:+ temperature: target temperature in Celsius+ mode: heating strategy to use+ """+ return {"temperature": temperature, "mode": mode}+++def _invoice(total, due_date="2026-09-01"):+ return {"vendor": "Acme Corp", "total": total, "due_date": due_date}+++def _envelope(arguments, name, ungrounded):+ return {"type": "call", "confidence": 0.9,+ "function_calls": [{"name": name, "arguments": arguments}],+ "validation": {"ungrounded": list(ungrounded), "negation": False}}+++def _extract_envelope(arguments, ungrounded=("Invoice.total",), name="Invoice"):+ return _envelope(arguments, name, ungrounded)+++def _run_envelope(arguments, ungrounded=("set_thermostat.temperature",)):+ return _envelope(arguments, "set_thermostat", ungrounded)def _call(due_date, name="Invoice"):@@ -128,3 +166,171 @@ def test_engine_reported_fabrications_are_kept_and_block_execution(stub):assert response["results"] == [{"error": "ungrounded vendor"}]assert stub.calls[-1] == ("complete", json.dumps([{"error": "ungrounded vendor"}]))+++def test_extract_clears_a_thousands_separated_number(stub):+ import needle++ stub.envelopes = [_extract_envelope(_invoice(1200.0))]+ invoice = needle.extract("Invoice from Acme Corp, $1,200.00, due 2026-09-01", Invoice)++ assert invoice.total == 1200.0+ assert invoice.due_date == datetime.date(2026, 9, 1)+++def test_extract_clears_a_thousands_separated_integer(stub):+ import needle++ stub.envelopes = [_extract_envelope(_invoice(1200))]+ invoice = needle.extract("Invoice from Acme Corp, $1,200, due 2026-09-01", Invoice)++ assert invoice.total == 1200+++def test_extract_leaves_an_unseparated_number_working(stub):+ import needle++ stub.envelopes = [_extract_envelope(_invoice(1200.0))]+ invoice = needle.extract("Invoice from Acme Corp, $1200.00, due 2026-09-01", Invoice)++ assert invoice.total == 1200.0+++def test_extract_still_rejects_a_fabricated_number(stub):+ import needle++ stub.envelopes = [_extract_envelope(_invoice(9999.0))]+ with pytest.raises(needle.ExtractionValidationError) as excinfo:+ needle.extract("Invoice from Acme Corp, $1,200.00, due 2026-09-01", Invoice)++ assert "total" in str(excinfo.value)+++def test_extract_still_rejects_a_non_numeric_engine_flag(stub):+ import needle++ stub.envelopes = [_extract_envelope(_invoice(1200.0),+ ungrounded=("Invoice.vendor",))]+ with pytest.raises(needle.ExtractionValidationError) as excinfo:+ needle.extract("Invoice from Acme Corp, $1,200.00, due 2026-09-01", Invoice)++ assert "vendor" in str(excinfo.value)+++def test_extract_clears_a_nested_separated_number(stub):+ import needle++ stub.envelopes = [_extract_envelope({"items": [{"price": 1100.0}]},+ ungrounded=("Order.items[0].price",),+ name="Order")]+ order = needle.extract("Order with one item priced $1,100.00", Order)++ assert order.items[0].price == 1100.0+++def test_extract_does_not_clear_a_digit_run_inside_a_longer_number(stub):+ import needle++ for source in ("Invoice from Acme Corp, $12,000.00, due 2026-09-01",+ "Invoice from Acme Corp, $11,200.00, due 2026-09-01",+ "Invoice from Acme Corp, $1,2000.00, due 2026-09-01"):+ stub.envelopes = [_extract_envelope(_invoice(1200.0))]+ with pytest.raises(needle.ExtractionValidationError):+ needle.extract(source, Invoice)+++def test_extract_does_not_clear_a_number_absent_from_the_source(stub):+ import needle++ stub.envelopes = [_extract_envelope(_invoice(1200.0))]+ with pytest.raises(needle.ExtractionValidationError):+ needle.extract("Invoice from Acme Corp, due 2026-09-01", Invoice)+++def test_extract_does_not_read_a_date_hyphen_as_a_minus_sign(stub):+ import needle++ for value in (-9.0, -1.0):+ stub.envelopes = [_extract_envelope(_invoice(value))]+ with pytest.raises(needle.ExtractionValidationError):+ needle.extract("Invoice from Acme Corp, $1,200.00, due 2026-09-01",+ Invoice)+++def test_extract_does_not_read_a_range_hyphen_as_a_minus_sign(stub):+ import needle++ stub.envelopes = [_extract_envelope(_invoice(-1400.0))]+ with pytest.raises(needle.ExtractionValidationError):+ needle.extract("Invoice from Acme Corp, $1,200.00-1,400.00, due 2026-09-01",+ Invoice)+++def test_extract_clears_a_written_negative(stub):+ import needle++ stub.envelopes = [_extract_envelope(_invoice(-1200.0))]+ invoice = needle.extract("Invoice from Acme Corp, -1,200.00, due 2026-09-01",+ Invoice)++ assert invoice.total == -1200.0+++def test_extract_clears_a_zero_against_its_source(stub):+ import needle++ stub.envelopes = [_extract_envelope(_invoice(0.0))]+ invoice = needle.extract("Invoice from Acme Corp, $0.00, due 2026-09-01", Invoice)++ assert invoice.total == 0.0+++def test_run_executes_a_grounded_number_and_refuses_an_ungrounded_sibling(stub):+ import needle++ envelope = {"type": "call", "confidence": 0.9,+ "function_calls": [+ {"name": "set_thermostat", "arguments": {"temperature": 21}},+ {"name": "set_thermostat", "arguments": {"temperature": 22}}],+ "validation": {"ungrounded": ["set_thermostat.temperature"],+ "negation": False}}+ stub.envelopes = [envelope, {"type": "respond", "function_calls": []}]+ agent = needle.Needle(tools=[set_thermostat])+ response = agent.run("make it 21 and cool the room")++ assert response["results"] == [{"temperature": 21, "mode": "auto"},+ {"error": "ungrounded temperature"}]+++def test_run_executes_a_separated_number_grounded_in_the_query(stub):+ import needle++ stub.envelopes = [_run_envelope({"temperature": 1200}),+ {"type": "respond", "function_calls": []}]+ agent = needle.Needle(tools=[set_thermostat])+ response = agent.run("set it to 1,200")++ assert response["results"] == [{"temperature": 1200, "mode": "auto"}]+++def test_run_refuses_a_number_absent_from_the_query(stub):+ import needle++ stub.envelopes = [_run_envelope({"temperature": 99}),+ {"type": "respond", "function_calls": []}]+ agent = needle.Needle(tools=[set_thermostat])+ response = agent.run("make it 21 and cool the room")++ assert response["results"] == [{"error": "ungrounded temperature"}]+++def test_run_still_refuses_a_non_numeric_engine_flag(stub):+ import needle++ stub.envelopes = [_run_envelope({"temperature": 21},+ ungrounded=("set_thermostat.mode",)),+ {"type": "respond", "function_calls": []}]+ agent = needle.Needle(tools=[set_thermostat])+ response = agent.run("make it 21 and cool the room")++ assert response["results"] == [{"error": "ungrounded mode"}]