Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(examples): gate on answer_confidence, not entropy confidence

thread3d05b49dfeee89
diff --git a/examples/18_confidence_gating.py b/examples/18_confidence_gating.py
index f165159f..8907486a 100644
--- a/examples/18_confidence_gating.py
+++ b/examples/18_confidence_gating.py
@@ -6,13 +6,15 @@ automatically at 0.85 or above, hand the rest to a human.
from _common import banner, describe, device_line, heading, load
banner("18", "Confidence gating", """
- The production pattern from the README. Laya's probabilities are trained with strictly
- proper scoring rules, so `confidence` is a real probability, not a vibe: it is the
- probability mass Laya puts on its own top answer.
+ The production pattern from the README. Gate on `answer_confidence`: the calibrated
+ probability Laya puts on the answer it reports, defined the same way on every question
+ type. `confidence` is a different number on `choice` and `score` -- 1 minus normalised
+ entropy, a measure of how concentrated the distribution is -- and the README warns against
+ carrying a threshold over from it.
- We route at >= 0.85 automatically and escalate everything below that to a human. Watch
- the second email: its label is right, but Laya is not confident enough to act on it
- alone -- which is exactly what the gate is for.
+ We route at >= 0.85 automatically and escalate everything below that to a human. The gate
+ is about certainty, not correctness: a ticket can come back with the label you expect and
+ still go to a human because `answer_confidence` sits below the bar.
""")
THRESHOLD = 0.85
@@ -63,25 +65,30 @@ auto, escalated, seen = [], [], {}
for name, body in EMAILS:
answers = agent.predict({"body": body}, QUESTIONS)["answers"]
dept = answers["department"]["choice"]
- conf = answers["department"]["confidence"]
+ answer_conf = answers["department"]["answer_confidence"]
seen[name] = answers["department"]
- if conf >= THRESHOLD:
+ if answer_conf >= THRESHOLD:
decision = "AUTOMATE -> %s queue" % dept
auto.append(name)
else:
decision = "ESCALATE -> human triage"
escalated.append(name)
- print(" %-18s %-8s conf=%.3f %s" % (name, dept, conf, decision))
+ print(" %-18s %-8s answer_conf=%.3f %s" % (name, dept, answer_conf, decision))
heading("summary")
print(" threshold %.2f" % THRESHOLD)
print(" automated (%d) %s" % (len(auto), ", ".join(auto)))
print(" escalated (%d) %s" % (len(escalated), ", ".join(escalated)))
-outage = seen["api outage"]
-outage_p = outage["probabilities"][outage["choice"]]
-print("""
- The gate is about certainty, not correctness. "api outage" is labelled `%s`
- with p=%.3f -- the right answer -- yet its confidence is %.3f, below the bar, so it
- still goes to a human. That is the trade you tune with THRESHOLD: raise it and fewer
- mistakes reach production, but more tickets cost a human first.
- """ % (outage["choice"], outage_p, outage["confidence"]))
+if escalated:
+ soft = seen[escalated[0]]
+ print("""
+ The gate is about certainty, not correctness. "%s" comes back labelled `%s` with
+ answer_confidence %.3f, below the %.2f bar, so it goes to a human -- even though that label
+ may well be the right one. That is the trade you tune with THRESHOLD: raise it and fewer
+ uncertain answers reach production, but more tickets cost a human first.
+ """ % (escalated[0], soft["choice"], soft["answer_confidence"], THRESHOLD))
+else:
+ print("""
+ Nothing fell below the %.2f bar on this batch. Lower THRESHOLD to make the gate stricter,
+ or lengthen EMAILS until something uncertain shows up.
+ """ % THRESHOLD)
diff --git a/examples/40_caching_and_monitoring.py b/examples/40_caching_and_monitoring.py
index 48c96e08..dc96b396 100644
--- a/examples/40_caching_and_monitoring.py
+++ b/examples/40_caching_and_monitoring.py
@@ -16,8 +16,9 @@ banner("40", "Caching and monitoring", """
inputs, keep the result, and pay for the forward pass once.
Part A measures a real forward pass against a cache hit. Part B runs 15 tickets through
- the triage preset and uses Laya's calibrated confidence to sort every answer into
- auto-action, human review or escalation -- the operational shape example 41 builds on.
+ the triage preset and gates on `answer_confidence`, the calibrated probability of the
+ answer Laya reports, to sort every answer into auto-action, human review or escalation --
+ the operational shape example 41 builds on.
""")
agent = load("english")
@@ -128,11 +129,11 @@ print(" %-19s %-17s %-7s %-24s %s"
for name, message in TICKETS:
answers = agent.predict({"message": message}, QUESTIONS)["answers"]
for qid, answer in answers.items():
- b = bucket(answer["confidence"])
+ b = bucket(answer["answer_confidence"])
answer_buckets[b] += 1
per_question.setdefault(qid, {"auto": 0, "review": 0, "escalate": 0})[b] += 1
- weakest_id = min(DECISION_IDS, key=lambda q: answers[q]["confidence"])
- weakest_conf = answers[weakest_id]["confidence"]
+ weakest_id = min(DECISION_IDS, key=lambda q: answers[q]["answer_confidence"])
+ weakest_conf = answers[weakest_id]["answer_confidence"]
top_intent = max(answers["intent"]["probabilities"].values())
rows.append((name, answers, weakest_id, weakest_conf, bucket(weakest_conf), top_intent))
print(" %-19s %-17s %-7.2f %-24s %s"
@@ -148,11 +149,11 @@ for qid, counts in per_question.items():
print(" %-19s %-7s %-6d %-7d %d"
% (qid, QUESTIONS[qid]["type"], counts["auto"], counts["review"], counts["escalate"]))
frustration = per_question["frustration"]
-print(" `frustration` is a %d-level score, so its entropy confidence is low even when the"
+print(" `frustration` is a %d-level score, so the probability it puts on the reported level"
% len(QUESTIONS["frustration"]["criteria"]))
-print(" answer is informative: %d of %d landed in escalate. Gate on the choice and noul"
+print(" is modest even when the answer is informative: %d of %d landed in escalate. Gate on"
% (frustration["escalate"], len(TICKETS)))
-print(" answers you actually act on, not on the score.")
+print(" the choice and noul answers you actually act on, not on the score.")
heading("states that look unusual")
print(" %-19s %-17s %-7s %-24s %s"
@@ -169,15 +170,15 @@ vague = next(answers for name, answers, *_ in rows if name == "vague")
p = list(vague["intent"]["probabilities"].values())
entropy = -sum(x * math.log(max(x, 1e-12)) for x in p) / math.log(len(p))
print("""
- Confidence is calibrated: the checkpoint is trained with proper scoring rules (example
- 34 covers the temperatures), so 0.80 is a level of certainty you can gate on rather than
- a vibe. For a choice question it is normalised Shannon entropy, confidence = 1 - H/log(k).
- For the "vague" ticket the intent distribution has H/log(k) = %.2f, so confidence is %.2f:
+ The monitor gates on `answer_confidence`: the calibrated probability of the answer Laya
+ reports, defined the same way on every question type. `confidence` is a different measure --
+ on `choice` and `score` it is 1 - H/log(k), how concentrated the distribution is -- and the
+ README warns against carrying a threshold over from it. For the "vague" ticket the intent
+ distribution has H/log(k) = %.2f, so `confidence` is %.2f while `answer_confidence` is %.2f:
the model is stating that the message does not identify one intent, and the monitor
- escalates it. For `noul`, confidence is max(p, 1-p) -- a different formula on a different
- scale, which is why the table above is split by primitive.
+ escalates it.
The thresholds, the 0.5 spam-style gates and the auto/review/escalate split are ours.
The probabilities are the model's, and they are the only reason a threshold means
anything. Example 41 turns exactly this monitor into a service policy.
- """ % (entropy, 1 - entropy))
+ """ % (entropy, 1 - entropy, vague["intent"]["answer_confidence"]))
diff --git a/examples/41_production_triage_service.py b/examples/41_production_triage_service.py
index f8074bf3..2d64314a 100644
--- a/examples/41_production_triage_service.py
+++ b/examples/41_production_triage_service.py
@@ -31,7 +31,7 @@ INTENT_CONFIDENT = 0.35 # below this the intent itself is too soft to a
def decide(answers):
"""Map triage answers to an action. Everything here is application policy."""
intent = answers["intent"]["choice"]
- intent_conf = answers["intent"]["confidence"]
+ intent_conf = answers["intent"]["answer_confidence"]
churn = answers["churn_risk"]["noul"]
urgent = answers["is_urgent"]["noul"]
frustration = answers["frustration"]["score"]
@@ -54,12 +54,13 @@ def decide(answers):
reasons.append("intent=technical_help")
return "route_to_team", "engineering", False, reasons
if intent == "information" and intent_conf >= INTENT_CONFIDENT:
- reasons.append("intent=information conf=%.2f >= %.2f" % (intent_conf, INTENT_CONFIDENT))
+ reasons.append("intent=information answer_confidence=%.2f >= %.2f"
+ % (intent_conf, INTENT_CONFIDENT))
return "auto_respond", "self_service", False, reasons
if frustration >= 2.5:
reasons.append("frustration=%.2f" % frustration)
return "route_to_team", "senior_support", False, reasons
- reasons.append("fallback: intent=%s conf=%.2f" % (intent, intent_conf))
+ reasons.append("fallback: intent=%s answer_confidence=%.2f" % (intent, intent_conf))
return "route_to_team", "general_support", False, reasons
# --- four tickets, one per branch -----------------------------------------------------------
@@ -94,7 +95,7 @@ for ticket in TICKETS:
"routing": {"model": result["routing"]["model"], "reason": result["routing"]["reason"]},
"answers": {
"intent": {"choice": answers["intent"]["choice"],
- "confidence": answers["intent"]["confidence"]},
+ "answer_confidence": answers["intent"]["answer_confidence"]},
"is_urgent": {"noul": answers["is_urgent"]["noul"]},
"frustration": {"score": answers["frustration"]["score"]},
"refund_requested": {"noul": answers["refund_requested"]["noul"]},