Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(structured): report the calibrated confidence `DecisionResult` promises
Bruce-Yiie1199ffaec74
diff --git a/docs/structured.md b/docs/structured.mdindex 6ac170db..28ba0e84 100644--- a/docs/structured.md+++ b/docs/structured.md@@ -146,20 +146,33 @@ with per-field confidence, probabilities, the raw answers, and the usage and rou```pythonresult = agent.decide(state, schema=Ticket, return_details=True)-result.values["department"] # "billing"-result.confidence["department"] # 0.94-result.probabilities["department"] # {"billing": 0.94, "support": 0.06, "sales": 0.0}-result.usage # {"input_tokens": 42, "output_tokens": 0}-result.routing # the Router decision, when a Router answered+result.values["department"] # "billing"+result.answer_confidence["department"] # 0.94 calibrated max(p): gate on this one+result.confidence["department"] # 0.71 normalized entropy, which depends on label count+result.probabilities["department"] # {"billing": 0.94, "support": 0.06, "sales": 0.0}+result.usage # {"input_tokens": 42, "output_tokens": 0}+result.routing # the Router decision, when a Router answered```-You can gate on it, for example escalate a field whose confidence is below a threshold:+`confidence` and `answer_confidence` are different quantities, and the names follow the definitions.+`answer_confidence` is the calibrated `max(p)`: it is what temperature scaling fits, what the+reported ECE measures, and what `min_confidence` is defined against. `confidence` is normalized+entropy, which depends on how many options the question had -- `tests/test_confidence.py` pins that+a two-option distribution comes back as 0.90 on a `noul` and 0.53 on an equivalent `choice` -- so it+does not compare against a threshold, and it is not what the calibration figures describe. A field+that reported no usable `answer_confidence` maps to `None`, which is not the same as a reported `0.0`.++Gate on the calibrated one:```python-if result.confidence["department"] < 0.6:+if result.answer_confidence["department"] < 0.6:result.values["department"] = "human-review"```+Both numbers are reported because both are sometimes what you want: `confidence` answers "how+concentrated is this distribution", `answer_confidence` answers "how much should I trust this+answer", and only the second is comparable across question shapes.+## How it maps internally- Enum and `Literal` become `choice` questions with the values as string labels; the label isdiff --git a/laya/confidence.py b/laya/confidence.pyindex 253329bf..c96e5ff0 100644--- a/laya/confidence.py+++ b/laya/confidence.py@@ -4,7 +4,40 @@ Pure Python: safe to import without PyTorch so that Router and structureddecisions stay lightweight and free of torch import overhead."""import math-from typing import Any, Dict, List+from typing import Any, Dict, List, Optional+++def calibrated_confidence(answer: Dict[str, Any]) -> Optional[float]:+ """The calibrated `answer_confidence` of `answer`, or None if it did not report one.++ `answer_confidence` is the calibrated max(p) quantity: it is what temperature scaling fits,+ what the reported ECE measures, and what the `min_confidence` gate is defined against (#361).+ `confidence`, by contrast, is normalized entropy on a scale that depends on the number of+ options -- `common.confidence_from_probs` calls it "not calibrated", and #394 is an open issue+ saying a confidence threshold does not transfer across option counts. So this deliberately+ does **not** fall back to `confidence`: reporting the entropy number under the calibrated name+ would be worse than reporting nothing, because the name is what a caller filters on.++ A `bool` is not a confidence and a NaN is not a decision, so both give None rather than 0.0.+ """+ conf = answer.get("answer_confidence")+ if isinstance(conf, (int, float)) and not isinstance(conf, bool) and math.isfinite(conf):+ return float(conf)+ return None+++def _gate_confidence(answer: Dict[str, Any]) -> Optional[float]:+ """The number the abstention gate compares against `min_confidence`, or None if there is none.++ The calibrated `answer_confidence` first, falling back to the entropy `confidence` so an+ answer that carries only the older field is still gated rather than silently passed.+ """+ conf = calibrated_confidence(answer)+ if conf is None:+ conf = answer.get("confidence")+ if isinstance(conf, (int, float)) and not isinstance(conf, bool) and math.isfinite(conf):+ return float(conf)+ return confdef check_min_confidence(v: Any) -> float:@@ -33,8 +66,6 @@ def flag_low_confidence(results: List[Dict[str, Any]], min_confidence: float) ->for a in answers.values():if not isinstance(a, dict):continue- conf = a.get("answer_confidence")- if conf is None:- conf = a.get("confidence")- if isinstance(conf, (int, float)) and not isinstance(conf, bool) and math.isfinite(conf) and conf < min_confidence:+ conf = _gate_confidence(a)+ if conf is not None and conf < min_confidence:a["low_confidence"] = Truediff --git a/laya/structured.py b/laya/structured.pyindex 30e5d3ba..5f296d9d 100644--- a/laya/structured.py+++ b/laya/structured.py@@ -20,7 +20,7 @@ from collections.abc import Sequence as SequenceABCfrom dataclasses import dataclass, fieldfrom typing import Any, Dict, List, Optional, Sequence, Tuple-from .confidence import check_min_confidence, flag_low_confidence+from .confidence import calibrated_confidence, check_min_confidence, flag_low_confidenceMAX_PROPERTIES = 32MAX_OPTIONS = 32@@ -37,6 +37,13 @@ class DecisionResult:`values` is the schema-shaped output. `confidence` and `probabilities` are keyed by field,and `answers` is Laya's raw answer per field.++ `answer_confidence` is the calibrated max(p) confidence per field -- the quantity temperature+ scaling fits, the reported ECE measures, and the `min_confidence` gate is defined against --+ under its own name. `confidence` keeps the normalized-entropy value it has always had, because+ that is not a calibrated quantity and a caller filtering this artifact to decide what to+ escalate has to be able to tell the two apart. A field that reported no usable calibrated+ confidence maps to `None`, which is distinct from a reported `0.0`."""values: Dict[str, Any]@@ -45,6 +52,9 @@ class DecisionResult:answers: Dict[str, Any]usage: Optional[Dict[str, int]] = Nonerouting: Optional[Dict[str, Any]] = None+ # Appended with a default so every existing construction of this dataclass keeps working, and+ # the first six fields keep their positions.+ answer_confidence: Dict[str, Optional[float]] = field(default_factory=dict)@dataclass@@ -232,9 +242,13 @@ def answer_to_pydantic(model: Any, answers: Dict[str, Any]) -> Any:def _details(values: Dict[str, Any], answers: Dict[str, Any], result: Dict[str, Any]) -> DecisionResult:confidence: Dict[str, float] = {}+ answer_confidence: Dict[str, Optional[float]] = {}probabilities: Dict[str, Dict[str, Any]] = {}for name, answer in answers.items():confidence[name] = float(answer.get("confidence", 0.0))+ # Read through the module that owns the definition, so this and the `min_confidence` gate+ # cannot disagree about what "calibrated" means.+ answer_confidence[name] = calibrated_confidence(answer)if answer.get("type") == "noul":p = float(answer.get("noul", 0.0))probabilities[name] = {"false": round(1.0 - p, 4), "true": round(p, 4)}@@ -247,6 +261,7 @@ def _details(values: Dict[str, Any], answers: Dict[str, Any], result: Dict[str,answers=dict(answers),usage=result.get("usage"),routing=result.get("routing"),+ answer_confidence=answer_confidence,)diff --git a/tests/test_structured.py b/tests/test_structured.pyindex fcf4ae2c..b51b2a69 100644--- a/tests/test_structured.py+++ b/tests/test_structured.py@@ -202,6 +202,58 @@ check_true("decide/details answers flag set", det.answers["urgency"].get("low_cocheck("decide/details high conf value kept", det.values["department"], "billing")check_true("decide/details high conf flag unset", det.answers["department"].get("low_confidence") is not True)+# --------------------------------------------------------------------- calibrated confidence+# The decision: "which of my structured decisions are safe to automate?" is answered by filtering+# the details artifact, so the number that filter reads has to be the calibrated one.+#+# `docs/structured.md` calls this bridge one that returns "typed values with calibrated+# confidence". `DecisionResult.confidence` is built from the answer's `confidence`, and+# `laya/common.py:confidence_from_probs` documents that field as "not calibrated: it is not what+# temperature scaling fits and not what the reported ECE measures. See `answer_confidence`".+# Laya's own gate is defined against `answer_confidence` (#361/#456), and #394 is an open issue+# saying a confidence threshold does not transfer across option counts -- which is the entropy+# definition's failure mode specifically.+#+# So a caller filtering `details.confidence` to decide what to escalate filters on a quantity the+# repo documents as neither temperature-scaled nor ECE-measured, under the name of the one that is.+CAL = {"department": {"type": "choice", "choice": "billing", "confidence": 0.30, "answer_confidence": 0.95},+ "urgency": {"type": "score", "score": 2.0, "confidence": 0.90, "answer_confidence": 0.40,+ "probabilities": {"0": 0.1, "1": 0.5, "2": 0.4}, "legend": {}}}+cal_det = decide(FakeRunner(CAL), "s", schema=SCHEMA, return_details=True)++# The two numbers differ on the same field, which is the whole point: the entropy value ranks+# `urgency` above `department` and the calibrated one reverses it.+check_true("details/the two confidences really do disagree",+ cal_det.confidence["department"] < cal_det.confidence["urgency"])+check_true("details/…and the calibrated one reverses that order",+ cal_det.answer_confidence["department"] > cal_det.answer_confidence["urgency"])+check("details/calibrated is reported per field", cal_det.answer_confidence["department"], 0.95)++# The name means what it says, so a caller that sorts by it sorts by the quantity Laya's+# calibration figures describe.+check("details/entropy field is left exactly as it was", cal_det.confidence["department"], 0.30)++# A field with no usable confidence is `None`, not 0.0. `confidence` defaults to 0.0, so today an+# absent confidence and a genuinely zero one are the same value, and a caller filtering on "below+# 0.4, escalate" escalates both without being able to tell why.+NO_CONF = {"department": {"type": "choice", "choice": "billing", "confidence": 0.0}}+no_det = decide(FakeRunner(NO_CONF), "s", schema=SCHEMA, return_details=True)+check("details/absent calibrated confidence is None, not 0.0", no_det.answer_confidence["department"], None)+check_true("details/…and stays distinct from a reported zero",+ no_det.answer_confidence["department"] is not cal_det.answer_confidence["urgency"])++# A `bool` is not a confidence, and a NaN is not a decision.+BAD_CONF = {"department": {"type": "choice", "choice": "billing", "answer_confidence": True}}+bad_det = decide(FakeRunner(BAD_CONF), "s", schema=SCHEMA, return_details=True)+check("details/a bool is not reported as a confidence", bad_det.answer_confidence["department"], None)+NAN_CONF = {"department": {"type": "choice", "choice": "billing", "answer_confidence": float("nan")}}+nan_det = decide(FakeRunner(NAN_CONF), "s", schema=SCHEMA, return_details=True)+check("details/a NaN is not reported as a confidence", nan_det.answer_confidence["department"], None)++# Every field is accounted for, so a caller can iterate `answer_confidence` without a KeyError on+# a field the model did not answer.+check_true("details/one entry per field", set(cal_det.answer_confidence) == set(CAL))+# Direct projection with low_confidence: True in answeranswers_with_flag = {"department": {"type": "choice", "choice": "billing", "confidence": 0.4, "low_confidence": True},diff --git a/tests/test_structured_api.py b/tests/test_structured_api.pyindex 16d67b02..17760b85 100644--- a/tests/test_structured_api.py+++ b/tests/test_structured_api.py@@ -74,11 +74,22 @@ check_true("decide/accepts **predict_kwargs",any(p.kind == inspect.Parameter.VAR_KEYWORD for p in inspect.signature(structured.decide).parameters.values()))# --------------------------------------------------------------- DecisionResult-FIELDS = ["values", "confidence", "probabilities", "answers", "usage", "routing"]+FIELDS = ["values", "confidence", "probabilities", "answers", "usage", "routing", "answer_confidence"]check("DecisionResult fields", [f.name for f in dataclasses.fields(structured.DecisionResult)], FIELDS)for optional in ("usage", "routing"):check("DecisionResult/%s default None" % optional,structured.DecisionResult.__dataclass_fields__[optional].default, None)+# `answer_confidence` is appended with a default, so any existing construction of this dataclass+# still works: the first six fields keep their positions and every caller may omit it.+check_true("DecisionResult/answer_confidence is optional",+ structured.DecisionResult.__dataclass_fields__["answer_confidence"].default_factory+ is not dataclasses.MISSING)+check("DecisionResult/existing construction still works",+ [f.name for f in dataclasses.fields(structured.DecisionResult(+ {"v": 1}, {"c": 1}, {"p": {}}, {"a": {}}))][:4],+ ["values", "confidence", "probabilities", "answers"])+check("DecisionResult/…and defaults answer_confidence", structured.DecisionResult(+ {"v": 1}, {"c": 1}, {"p": {}}, {"a": {}}).answer_confidence, {})# --------------------------------------------------------------- methodsfor label, cls in (("Agent", laya.Agent), ("Router", laya.Router)):