Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(examples): gate on answer_confidence, not entropy confidence
thread3d05b49dfeee89
diff --git a/examples/18_confidence_gating.py b/examples/18_confidence_gating.pyindex f165159f..8907486a 100644--- a/examples/18_confidence_gating.py+++ b/examples/18_confidence_gating.py@@ -6,13 +6,15 @@ automatically at 0.85 or above, hand the rest to a human.from _common import banner, describe, device_line, heading, loadbanner("18", "Confidence gating", """- The production pattern from the README. Laya's probabilities are trained with strictly- proper scoring rules, so `confidence` is a real probability, not a vibe: it is the- probability mass Laya puts on its own top answer.+ The production pattern from the README. Gate on `answer_confidence`: the calibrated+ probability Laya puts on the answer it reports, defined the same way on every question+ type. `confidence` is a different number on `choice` and `score` -- 1 minus normalised+ entropy, a measure of how concentrated the distribution is -- and the README warns against+ carrying a threshold over from it.- We route at >= 0.85 automatically and escalate everything below that to a human. Watch- the second email: its label is right, but Laya is not confident enough to act on it- alone -- which is exactly what the gate is for.+ We route at >= 0.85 automatically and escalate everything below that to a human. The gate+ is about certainty, not correctness: a ticket can come back with the label you expect and+ still go to a human because `answer_confidence` sits below the bar.""")THRESHOLD = 0.85@@ -63,25 +65,30 @@ auto, escalated, seen = [], [], {}for name, body in EMAILS:answers = agent.predict({"body": body}, QUESTIONS)["answers"]dept = answers["department"]["choice"]- conf = answers["department"]["confidence"]+ answer_conf = answers["department"]["answer_confidence"]seen[name] = answers["department"]- if conf >= THRESHOLD:+ if answer_conf >= THRESHOLD:decision = "AUTOMATE -> %s queue" % deptauto.append(name)else:decision = "ESCALATE -> human triage"escalated.append(name)- print(" %-18s %-8s conf=%.3f %s" % (name, dept, conf, decision))+ print(" %-18s %-8s answer_conf=%.3f %s" % (name, dept, answer_conf, decision))heading("summary")print(" threshold %.2f" % THRESHOLD)print(" automated (%d) %s" % (len(auto), ", ".join(auto)))print(" escalated (%d) %s" % (len(escalated), ", ".join(escalated)))-outage = seen["api outage"]-outage_p = outage["probabilities"][outage["choice"]]-print("""- The gate is about certainty, not correctness. "api outage" is labelled `%s`- with p=%.3f -- the right answer -- yet its confidence is %.3f, below the bar, so it- still goes to a human. That is the trade you tune with THRESHOLD: raise it and fewer- mistakes reach production, but more tickets cost a human first.- """ % (outage["choice"], outage_p, outage["confidence"]))+if escalated:+ soft = seen[escalated[0]]+ print("""+ The gate is about certainty, not correctness. "%s" comes back labelled `%s` with+ answer_confidence %.3f, below the %.2f bar, so it goes to a human -- even though that label+ may well be the right one. That is the trade you tune with THRESHOLD: raise it and fewer+ uncertain answers reach production, but more tickets cost a human first.+ """ % (escalated[0], soft["choice"], soft["answer_confidence"], THRESHOLD))+else:+ print("""+ Nothing fell below the %.2f bar on this batch. Lower THRESHOLD to make the gate stricter,+ or lengthen EMAILS until something uncertain shows up.+ """ % THRESHOLD)diff --git a/examples/40_caching_and_monitoring.py b/examples/40_caching_and_monitoring.pyindex 48c96e08..dc96b396 100644--- a/examples/40_caching_and_monitoring.py+++ b/examples/40_caching_and_monitoring.py@@ -16,8 +16,9 @@ banner("40", "Caching and monitoring", """inputs, keep the result, and pay for the forward pass once.Part A measures a real forward pass against a cache hit. Part B runs 15 tickets through- the triage preset and uses Laya's calibrated confidence to sort every answer into- auto-action, human review or escalation -- the operational shape example 41 builds on.+ the triage preset and gates on `answer_confidence`, the calibrated probability of the+ answer Laya reports, to sort every answer into auto-action, human review or escalation --+ the operational shape example 41 builds on.""")agent = load("english")@@ -128,11 +129,11 @@ print(" %-19s %-17s %-7s %-24s %s"for name, message in TICKETS:answers = agent.predict({"message": message}, QUESTIONS)["answers"]for qid, answer in answers.items():- b = bucket(answer["confidence"])+ b = bucket(answer["answer_confidence"])answer_buckets[b] += 1per_question.setdefault(qid, {"auto": 0, "review": 0, "escalate": 0})[b] += 1- weakest_id = min(DECISION_IDS, key=lambda q: answers[q]["confidence"])- weakest_conf = answers[weakest_id]["confidence"]+ weakest_id = min(DECISION_IDS, key=lambda q: answers[q]["answer_confidence"])+ weakest_conf = answers[weakest_id]["answer_confidence"]top_intent = max(answers["intent"]["probabilities"].values())rows.append((name, answers, weakest_id, weakest_conf, bucket(weakest_conf), top_intent))print(" %-19s %-17s %-7.2f %-24s %s"@@ -148,11 +149,11 @@ for qid, counts in per_question.items():print(" %-19s %-7s %-6d %-7d %d"% (qid, QUESTIONS[qid]["type"], counts["auto"], counts["review"], counts["escalate"]))frustration = per_question["frustration"]-print(" `frustration` is a %d-level score, so its entropy confidence is low even when the"+print(" `frustration` is a %d-level score, so the probability it puts on the reported level"% len(QUESTIONS["frustration"]["criteria"]))-print(" answer is informative: %d of %d landed in escalate. Gate on the choice and noul"+print(" is modest even when the answer is informative: %d of %d landed in escalate. Gate on"% (frustration["escalate"], len(TICKETS)))-print(" answers you actually act on, not on the score.")+print(" the choice and noul answers you actually act on, not on the score.")heading("states that look unusual")print(" %-19s %-17s %-7s %-24s %s"@@ -169,15 +170,15 @@ vague = next(answers for name, answers, *_ in rows if name == "vague")p = list(vague["intent"]["probabilities"].values())entropy = -sum(x * math.log(max(x, 1e-12)) for x in p) / math.log(len(p))print("""- Confidence is calibrated: the checkpoint is trained with proper scoring rules (example- 34 covers the temperatures), so 0.80 is a level of certainty you can gate on rather than- a vibe. For a choice question it is normalised Shannon entropy, confidence = 1 - H/log(k).- For the "vague" ticket the intent distribution has H/log(k) = %.2f, so confidence is %.2f:+ The monitor gates on `answer_confidence`: the calibrated probability of the answer Laya+ reports, defined the same way on every question type. `confidence` is a different measure --+ on `choice` and `score` it is 1 - H/log(k), how concentrated the distribution is -- and the+ README warns against carrying a threshold over from it. For the "vague" ticket the intent+ distribution has H/log(k) = %.2f, so `confidence` is %.2f while `answer_confidence` is %.2f:the model is stating that the message does not identify one intent, and the monitor- escalates it. For `noul`, confidence is max(p, 1-p) -- a different formula on a different- scale, which is why the table above is split by primitive.+ escalates it.The thresholds, the 0.5 spam-style gates and the auto/review/escalate split are ours.The probabilities are the model's, and they are the only reason a threshold meansanything. Example 41 turns exactly this monitor into a service policy.- """ % (entropy, 1 - entropy))+ """ % (entropy, 1 - entropy, vague["intent"]["answer_confidence"]))diff --git a/examples/41_production_triage_service.py b/examples/41_production_triage_service.pyindex f8074bf3..2d64314a 100644--- a/examples/41_production_triage_service.py+++ b/examples/41_production_triage_service.py@@ -31,7 +31,7 @@ INTENT_CONFIDENT = 0.35 # below this the intent itself is too soft to adef decide(answers):"""Map triage answers to an action. Everything here is application policy."""intent = answers["intent"]["choice"]- intent_conf = answers["intent"]["confidence"]+ intent_conf = answers["intent"]["answer_confidence"]churn = answers["churn_risk"]["noul"]urgent = answers["is_urgent"]["noul"]frustration = answers["frustration"]["score"]@@ -54,12 +54,13 @@ def decide(answers):reasons.append("intent=technical_help")return "route_to_team", "engineering", False, reasonsif intent == "information" and intent_conf >= INTENT_CONFIDENT:- reasons.append("intent=information conf=%.2f >= %.2f" % (intent_conf, INTENT_CONFIDENT))+ reasons.append("intent=information answer_confidence=%.2f >= %.2f"+ % (intent_conf, INTENT_CONFIDENT))return "auto_respond", "self_service", False, reasonsif frustration >= 2.5:reasons.append("frustration=%.2f" % frustration)return "route_to_team", "senior_support", False, reasons- reasons.append("fallback: intent=%s conf=%.2f" % (intent, intent_conf))+ reasons.append("fallback: intent=%s answer_confidence=%.2f" % (intent, intent_conf))return "route_to_team", "general_support", False, reasons# --- four tickets, one per branch -----------------------------------------------------------@@ -94,7 +95,7 @@ for ticket in TICKETS:"routing": {"model": result["routing"]["model"], "reason": result["routing"]["reason"]},"answers": {"intent": {"choice": answers["intent"]["choice"],- "confidence": answers["intent"]["confidence"]},+ "answer_confidence": answers["intent"]["answer_confidence"]},"is_urgent": {"noul": answers["is_urgent"]["noul"]},"frustration": {"score": answers["frustration"]["score"]},"refund_requested": {"noul": answers["refund_requested"]["noul"]},