Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(structured): report the calibrated confidence `DecisionResult` promises

Bruce-Yiie1199ffaec74
diff --git a/docs/structured.md b/docs/structured.md
index 6ac170db..28ba0e84 100644
--- a/docs/structured.md
+++ b/docs/structured.md
@@ -146,20 +146,33 @@ with per-field confidence, probabilities, the raw answers, and the usage and rou
```python
result = agent.decide(state, schema=Ticket, return_details=True)
-result.values["department"] # "billing"
-result.confidence["department"] # 0.94
-result.probabilities["department"] # {"billing": 0.94, "support": 0.06, "sales": 0.0}
-result.usage # {"input_tokens": 42, "output_tokens": 0}
-result.routing # the Router decision, when a Router answered
+result.values["department"] # "billing"
+result.answer_confidence["department"] # 0.94 calibrated max(p): gate on this one
+result.confidence["department"] # 0.71 normalized entropy, which depends on label count
+result.probabilities["department"] # {"billing": 0.94, "support": 0.06, "sales": 0.0}
+result.usage # {"input_tokens": 42, "output_tokens": 0}
+result.routing # the Router decision, when a Router answered
```
-You can gate on it, for example escalate a field whose confidence is below a threshold:
+`confidence` and `answer_confidence` are different quantities, and the names follow the definitions.
+`answer_confidence` is the calibrated `max(p)`: it is what temperature scaling fits, what the
+reported ECE measures, and what `min_confidence` is defined against. `confidence` is normalized
+entropy, which depends on how many options the question had -- `tests/test_confidence.py` pins that
+a two-option distribution comes back as 0.90 on a `noul` and 0.53 on an equivalent `choice` -- so it
+does not compare against a threshold, and it is not what the calibration figures describe. A field
+that reported no usable `answer_confidence` maps to `None`, which is not the same as a reported `0.0`.
+
+Gate on the calibrated one:
```python
-if result.confidence["department"] < 0.6:
+if result.answer_confidence["department"] < 0.6:
result.values["department"] = "human-review"
```
+Both numbers are reported because both are sometimes what you want: `confidence` answers "how
+concentrated is this distribution", `answer_confidence` answers "how much should I trust this
+answer", and only the second is comparable across question shapes.
+
## How it maps internally
- Enum and `Literal` become `choice` questions with the values as string labels; the label is
diff --git a/laya/confidence.py b/laya/confidence.py
index 253329bf..c96e5ff0 100644
--- a/laya/confidence.py
+++ b/laya/confidence.py
@@ -4,7 +4,40 @@ Pure Python: safe to import without PyTorch so that Router and structured
decisions stay lightweight and free of torch import overhead.
"""
import math
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
+
+
+def calibrated_confidence(answer: Dict[str, Any]) -> Optional[float]:
+ """The calibrated `answer_confidence` of `answer`, or None if it did not report one.
+
+ `answer_confidence` is the calibrated max(p) quantity: it is what temperature scaling fits,
+ what the reported ECE measures, and what the `min_confidence` gate is defined against (#361).
+ `confidence`, by contrast, is normalized entropy on a scale that depends on the number of
+ options -- `common.confidence_from_probs` calls it "not calibrated", and #394 is an open issue
+ saying a confidence threshold does not transfer across option counts. So this deliberately
+ does **not** fall back to `confidence`: reporting the entropy number under the calibrated name
+ would be worse than reporting nothing, because the name is what a caller filters on.
+
+ A `bool` is not a confidence and a NaN is not a decision, so both give None rather than 0.0.
+ """
+ conf = answer.get("answer_confidence")
+ if isinstance(conf, (int, float)) and not isinstance(conf, bool) and math.isfinite(conf):
+ return float(conf)
+ return None
+
+
+def _gate_confidence(answer: Dict[str, Any]) -> Optional[float]:
+ """The number the abstention gate compares against `min_confidence`, or None if there is none.
+
+ The calibrated `answer_confidence` first, falling back to the entropy `confidence` so an
+ answer that carries only the older field is still gated rather than silently passed.
+ """
+ conf = calibrated_confidence(answer)
+ if conf is None:
+ conf = answer.get("confidence")
+ if isinstance(conf, (int, float)) and not isinstance(conf, bool) and math.isfinite(conf):
+ return float(conf)
+ return conf
def check_min_confidence(v: Any) -> float:
@@ -33,8 +66,6 @@ def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) ->
for a in answers.values():
if not isinstance(a, dict):
continue
- conf = a.get("answer_confidence")
- if conf is None:
- conf = a.get("confidence")
- if isinstance(conf, (int, float)) and not isinstance(conf, bool) and math.isfinite(conf) and conf < min_confidence:
+ conf = _gate_confidence(a)
+ if conf is not None and conf < min_confidence:
a["low_confidence"] = True
diff --git a/laya/structured.py b/laya/structured.py
index 30e5d3ba..5f296d9d 100644
--- a/laya/structured.py
+++ b/laya/structured.py
@@ -20,7 +20,7 @@ from collections.abc import Sequence as SequenceABC
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Sequence, Tuple
-from .confidence import check_min_confidence, flag_low_confidence
+from .confidence import calibrated_confidence, check_min_confidence, flag_low_confidence
MAX_PROPERTIES = 32
MAX_OPTIONS = 32
@@ -37,6 +37,13 @@ class DecisionResult:
`values` is the schema-shaped output. `confidence` and `probabilities` are keyed by field,
and `answers` is Laya's raw answer per field.
+
+ `answer_confidence` is the calibrated max(p) confidence per field -- the quantity temperature
+ scaling fits, the reported ECE measures, and the `min_confidence` gate is defined against --
+ under its own name. `confidence` keeps the normalized-entropy value it has always had, because
+ that is not a calibrated quantity and a caller filtering this artifact to decide what to
+ escalate has to be able to tell the two apart. A field that reported no usable calibrated
+ confidence maps to `None`, which is distinct from a reported `0.0`.
"""
values: Dict[str, Any]
@@ -45,6 +52,9 @@ class DecisionResult:
answers: Dict[str, Any]
usage: Optional[Dict[str, int]] = None
routing: Optional[Dict[str, Any]] = None
+ # Appended with a default so every existing construction of this dataclass keeps working, and
+ # the first six fields keep their positions.
+ answer_confidence: Dict[str, Optional[float]] = field(default_factory=dict)
@dataclass
@@ -232,9 +242,13 @@ def answer_to_pydantic(model: Any, answers: Dict[str, Any]) -> Any:
def _details(values: Dict[str, Any], answers: Dict[str, Any], result: Dict[str, Any]) -> DecisionResult:
confidence: Dict[str, float] = {}
+ answer_confidence: Dict[str, Optional[float]] = {}
probabilities: Dict[str, Dict[str, Any]] = {}
for name, answer in answers.items():
confidence[name] = float(answer.get("confidence", 0.0))
+ # Read through the module that owns the definition, so this and the `min_confidence` gate
+ # cannot disagree about what "calibrated" means.
+ answer_confidence[name] = calibrated_confidence(answer)
if answer.get("type") == "noul":
p = float(answer.get("noul", 0.0))
probabilities[name] = {"false": round(1.0 - p, 4), "true": round(p, 4)}
@@ -247,6 +261,7 @@ def _details(values: Dict[str, Any], answers: Dict[str, Any], result: Dict[str,
answers=dict(answers),
usage=result.get("usage"),
routing=result.get("routing"),
+ answer_confidence=answer_confidence,
)
diff --git a/tests/test_structured.py b/tests/test_structured.py
index fcf4ae2c..b51b2a69 100644
--- a/tests/test_structured.py
+++ b/tests/test_structured.py
@@ -202,6 +202,58 @@ check_true("decide/details answers flag set", det.answers["urgency"].get("low_co
check("decide/details high conf value kept", det.values["department"], "billing")
check_true("decide/details high conf flag unset", det.answers["department"].get("low_confidence") is not True)
+# --------------------------------------------------------------------- calibrated confidence
+# The decision: "which of my structured decisions are safe to automate?" is answered by filtering
+# the details artifact, so the number that filter reads has to be the calibrated one.
+#
+# `docs/structured.md` calls this bridge one that returns "typed values with calibrated
+# confidence". `DecisionResult.confidence` is built from the answer's `confidence`, and
+# `laya/common.py:confidence_from_probs` documents that field as "not calibrated: it is not what
+# temperature scaling fits and not what the reported ECE measures. See `answer_confidence`".
+# Laya's own gate is defined against `answer_confidence` (#361/#456), and #394 is an open issue
+# saying a confidence threshold does not transfer across option counts -- which is the entropy
+# definition's failure mode specifically.
+#
+# So a caller filtering `details.confidence` to decide what to escalate filters on a quantity the
+# repo documents as neither temperature-scaled nor ECE-measured, under the name of the one that is.
+CAL = {"department": {"type": "choice", "choice": "billing", "confidence": 0.30, "answer_confidence": 0.95},
+ "urgency": {"type": "score", "score": 2.0, "confidence": 0.90, "answer_confidence": 0.40,
+ "probabilities": {"0": 0.1, "1": 0.5, "2": 0.4}, "legend": {}}}
+cal_det = decide(FakeRunner(CAL), "s", schema=SCHEMA, return_details=True)
+
+# The two numbers differ on the same field, which is the whole point: the entropy value ranks
+# `urgency` above `department` and the calibrated one reverses it.
+check_true("details/the two confidences really do disagree",
+ cal_det.confidence["department"] < cal_det.confidence["urgency"])
+check_true("details/…and the calibrated one reverses that order",
+ cal_det.answer_confidence["department"] > cal_det.answer_confidence["urgency"])
+check("details/calibrated is reported per field", cal_det.answer_confidence["department"], 0.95)
+
+# The name means what it says, so a caller that sorts by it sorts by the quantity Laya's
+# calibration figures describe.
+check("details/entropy field is left exactly as it was", cal_det.confidence["department"], 0.30)
+
+# A field with no usable confidence is `None`, not 0.0. `confidence` defaults to 0.0, so today an
+# absent confidence and a genuinely zero one are the same value, and a caller filtering on "below
+# 0.4, escalate" escalates both without being able to tell why.
+NO_CONF = {"department": {"type": "choice", "choice": "billing", "confidence": 0.0}}
+no_det = decide(FakeRunner(NO_CONF), "s", schema=SCHEMA, return_details=True)
+check("details/absent calibrated confidence is None, not 0.0", no_det.answer_confidence["department"], None)
+check_true("details/…and stays distinct from a reported zero",
+ no_det.answer_confidence["department"] is not cal_det.answer_confidence["urgency"])
+
+# A `bool` is not a confidence, and a NaN is not a decision.
+BAD_CONF = {"department": {"type": "choice", "choice": "billing", "answer_confidence": True}}
+bad_det = decide(FakeRunner(BAD_CONF), "s", schema=SCHEMA, return_details=True)
+check("details/a bool is not reported as a confidence", bad_det.answer_confidence["department"], None)
+NAN_CONF = {"department": {"type": "choice", "choice": "billing", "answer_confidence": float("nan")}}
+nan_det = decide(FakeRunner(NAN_CONF), "s", schema=SCHEMA, return_details=True)
+check("details/a NaN is not reported as a confidence", nan_det.answer_confidence["department"], None)
+
+# Every field is accounted for, so a caller can iterate `answer_confidence` without a KeyError on
+# a field the model did not answer.
+check_true("details/one entry per field", set(cal_det.answer_confidence) == set(CAL))
+
# Direct projection with low_confidence: True in answer
answers_with_flag = {
"department": {"type": "choice", "choice": "billing", "confidence": 0.4, "low_confidence": True},
diff --git a/tests/test_structured_api.py b/tests/test_structured_api.py
index 16d67b02..17760b85 100644
--- a/tests/test_structured_api.py
+++ b/tests/test_structured_api.py
@@ -74,11 +74,22 @@ check_true("decide/accepts **predict_kwargs",
any(p.kind == inspect.Parameter.VAR_KEYWORD for p in inspect.signature(structured.decide).parameters.values()))
# --------------------------------------------------------------- DecisionResult
-FIELDS = ["values", "confidence", "probabilities", "answers", "usage", "routing"]
+FIELDS = ["values", "confidence", "probabilities", "answers", "usage", "routing", "answer_confidence"]
check("DecisionResult fields", [f.name for f in dataclasses.fields(structured.DecisionResult)], FIELDS)
for optional in ("usage", "routing"):
check("DecisionResult/%s default None" % optional,
structured.DecisionResult.__dataclass_fields__[optional].default, None)
+# `answer_confidence` is appended with a default, so any existing construction of this dataclass
+# still works: the first six fields keep their positions and every caller may omit it.
+check_true("DecisionResult/answer_confidence is optional",
+ structured.DecisionResult.__dataclass_fields__["answer_confidence"].default_factory
+ is not dataclasses.MISSING)
+check("DecisionResult/existing construction still works",
+ [f.name for f in dataclasses.fields(structured.DecisionResult(
+ {"v": 1}, {"c": 1}, {"p": {}}, {"a": {}}))][:4],
+ ["values", "confidence", "probabilities", "answers"])
+check("DecisionResult/…and defaults answer_confidence", structured.DecisionResult(
+ {"v": 1}, {"c": 1}, {"p": {}}, {"a": {}}).answer_confidence, {})
# --------------------------------------------------------------- methods
for label, cls in (("Agent", laya.Agent), ("Router", laya.Router)):