Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #973 from aashish254/fix-example18-calibrated-claim

Nandakishor9f17443351e7
diff --cc tests/test_confidence.py
index 4bb39df3,2104f654..123f8531
--- a/tests/test_confidence.py
+++ b/tests/test_confidence.py
@@@ -1121,249 -604,147 +1121,391 @@@ for _fn in (test_preset_is_four_flags_a
PASS.append("page-28/%s" % _fn.__name__)
+# --------------------------------------------- example 40 must not call a raw softmax calibrated
+# `examples/40_caching_and_monitoring.py` gates 15 tickets on `answer_confidence`. On main it calls
+# that field "the calibrated probability of the answer Laya reports" in both its banner and its
+# closing paragraph, and its `bucket()` docstring reads "application policy over a calibrated
+# number". `answer_confidence` is max(p) -- the quantity temperature scaling fits and ECE measures
+# -- but the README's own Calibration section says that reading holds *only* after temperatures are
+# fitted and validated, the shipped checkpoints are over-confident, and the loader emits at load
+# time `RuntimeWarning: ... choice:11+=0.10058... -> 0.5. Treat confidence from the affected
+# entries as uncalibrated.` The page now reads the temperature each shape was actually scaled by off
+# the loaded agent and states the calibration as conditional.
+#
+# No weights are loaded here: the four helpers are pulled from the example's AST and exec'd against
+# a fabricated agent, and `scale_for`'s lookup is checked against core's own `temp_bucket`, so the
+# gate drives the code the page runs instead of re-reading its prose.
+from laya.common import QTYPES as _QTYPES40, temp_bucket as _temp_bucket # noqa: E402
+
+EXAMPLE_40 = os.path.join(ROOT, "examples", "40_caching_and_monitoring.py")
+HELPERS_40 = ("option_count", "scale_for", "clamped_buckets", "entropy_confidence")
+# Names a helper may read from the page's world: the two core symbols it must defer to, and `math`
+# for the entropy. Any other non-builtin free name means this gate cannot drive that helper.
+CORE_GLOBALS_40 = {"temp_bucket", "QTYPES", "math"}
+
+# main's page, as literal source lines. Every ban is witnessed against this text; every positive
+# rule below is witnessed by showing this text does NOT satisfy it.
+OLD_PAGE_40 = '''
+ the triage preset and gates on `answer_confidence`, the calibrated probability of the
+ """Our thresholds, not the model's: application policy over a calibrated number."""
+ The monitor gates on `answer_confidence`: the calibrated probability of the answer Laya
+ distribution has H/log(k) = %.2f, so `confidence` is %.2f while
+ """ % (entropy, 1 - entropy, vague["intent"]["answer_confidence"]))
+'''
+
+UNCONDITIONAL_CALIBRATION = re.compile(r"the calibrated probability", re.I)
+POLICY_OVER_CALIBRATED = re.compile(r"over a calibrated number", re.I)
+SUBSTITUTED_CONFIDENCE = re.compile(r"1\s*-\s*entropy")
+DERIVES_SCALING = re.compile(r"temp_bucket\(")
+READS_REPORTED_CONFIDENCE = re.compile(r"\[[\"']confidence[\"']\]")
+HARDCODED_BUCKET = re.compile(r"[\"'](choice|score|noul):[0-9]")
+
+
+def _src40():
+ with open(EXAMPLE_40, encoding="utf-8") as fh:
+ return fh.read()
+
+
+def _helpers40():
+ """Exec only the example's top-level helpers -- its `load()` call needs weights."""
+ tree = ast.parse(_src40())
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_40]
+ ns = {"temp_bucket": _temp_bucket, "QTYPES": _QTYPES40, "math": math}
+ exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_40, "exec"), ns) # noqa: S102
+ return ns
+
+
+def _called40():
+ tree = ast.parse(_src40())
+ top = [n for n in tree.body if not isinstance(
+ n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))]
+ return {node.id for stmt in top for node in ast.walk(stmt)
+ if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}
+
+
+def _fn40(name):
+ for n in ast.parse(_src40()).body:
+ if isinstance(n, ast.FunctionDef) and n.name == name:
+ return n
+ raise KeyError(name)
+
+
+class _FakeAgent(object):
+ """Only the three temperature attributes `scale_for` and `clamped_buckets` read."""
+
+ def __init__(self, by_options, per_type, raw=None):
+ self.temperature_by_options = by_options
+ self.temperature = per_type
+ self.temperature_by_options_raw = dict(by_options) if raw is None else raw
+
+
+def test_page40_helpers_are_live_defer_to_core():
+ defined = sorted(n.name for n in ast.parse(_src40()).body
+ if isinstance(n, ast.FunctionDef) and n.name in HELPERS_40)
+ assert defined == sorted(HELPERS_40), "example 40 lost a helper: %s" % defined
+ ns = _helpers40()
+ unused = set(HELPERS_40) - _called40()
+ assert not unused, "defined but never called at module level: %s" % sorted(unused)
+ for name in HELPERS_40:
+ node = _fn40(name)
+ outside = sorted(n for n in _free_names(node)
+ if not hasattr(builtins, n) and n not in CORE_GLOBALS_40)
+ assert not outside, "%s reads %s from the page, so this gate cannot drive it" % (name, outside)
+ prints = [c for c in ast.walk(node)
+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == "print"]
+ assert not prints, "%s prints instead of returning" % name
+ # `scale_for` must resolve the bucket through core, not carry a typed-in table.
+ assert not HARDCODED_BUCKET.search(ast.dump(_fn40("scale_for"))), (
+ "scale_for hardcodes a bucket name instead of calling temp_bucket")
+
+
+def test_scale_for_replays_core_lookup():
+ """For every shape, scale_for returns exactly `temperature_by_options.get(bucket, per_type)`."""
+ ns = _helpers40()
+ per_type = [7.0, 8.0, 9.0] # choice, score, noul fallbacks
+ by_options = {"choice:6-10": 1.0, "noul:2": 1.9834, "score:3-5": 1.2514}
+ fake = _FakeAgent(by_options, per_type)
+ for qtype, k in (("choice", 6), ("noul", 2), ("score", 4), ("noul", 2)):
+ name, applied, in_map = ns["scale_for"](fake, qtype, k)
+ bucket = _temp_bucket(_QTYPES40[qtype], k)
+ assert name == bucket, (qtype, k, name, bucket)
+ assert in_map == (bucket in by_options), (bucket, in_map)
+ assert abs(applied - by_options.get(bucket, per_type[_QTYPES40[qtype]])) < 1e-9, (
+ "%s: %r not core's get(...)" % (bucket, applied))
+ # the fallback arm: an unmapped shape must fall to the per-type temperature, flagged not-in-map.
+ name, applied, in_map = ns["scale_for"](_FakeAgent({}, [0.5, 0.6, 0.7]), "choice", 12)
+ assert (name, applied, in_map) == ("choice:11+", 0.5, False), (name, applied, in_map)
+ # a mapped temperature of exactly 1.0 must be reported as 1.0, not nudged.
+ assert ns["scale_for"](_FakeAgent({"choice:6-10": 1.0}, [1.0, 1.0, 1.0]), "choice", 6) == (
+ "choice:6-10", 1.0, True)
+
+
+def test_option_count_matches_the_shipped_preset():
+ ns = _helpers40()
+ counts = {qid: ns["option_count"](q) for qid, q in laya.triage_questions().items()}
+ assert counts == {"intent": 6, "is_urgent": 2, "frustration": 4,
+ "refund_requested": 2, "churn_risk": 2}, counts
+
+
+def test_clamped_buckets_names_only_a_shipped_value_the_runtime_refused():
+ ns = _helpers40()
+ # `choice:11+` ships below TEMP_MIN and is clamped to 0.5; everything else ships unchanged.
+ applied = {"choice:11+": 0.5, "noul:2": 1.9834}
+ raw = {"choice:11+": 0.10058280825614929, "noul:2": 1.9834}
+ out = ns["clamped_buckets"](_FakeAgent(applied, [0.5, 0.5, 0.5], raw))
+ assert out == ["choice:11+=0.1006 -> 0.5000"], out
+ # and an honest checkpoint reports nothing.
+ same = {"noul:2": 1.9834}
+ assert ns["clamped_buckets"](_FakeAgent(same, [0.5, 0.5, 0.5], dict(same))) == []
+
+
+def test_entropy_confidence_recomputes_core_exactly():
+ ns = _helpers40()
+ for dist in ([0.5, 0.3, 0.2], [0.9, 0.1], [0.25, 0.25, 0.25, 0.25], [1.0, 0.0, 0.0]):
+ p = np.array(dist, dtype=float)
+ got = ns["entropy_confidence"](dist)
+ want = confidence_from_probs(p, len(dist))
+ assert close(got, want, 1e-9), (dist, got, want)
+ assert ns["entropy_confidence"]([1.0]) == 1.0, "k<2 is 1.0 by definition"
+
+
+def test_page40_drops_the_unconditional_calibration_claim():
+ """Each ban fires on main's page and not on this one -- a ban with no witness is a guess."""
+ src = _src40()
+ for name, rule in (("'the calibrated probability'", UNCONDITIONAL_CALIBRATION),
+ ("'over a calibrated number'", POLICY_OVER_CALIBRATED),
+ ("prints `1 - entropy` as the confidence", SUBSTITUTED_CONFIDENCE)):
+ assert rule.search(OLD_PAGE_40), "%s does not fire on the wording it bans" % name
+ assert not rule.search(src), "%s is still in example 40" % name
+
+
+def test_page40_reads_the_scaling_and_the_reported_field():
+ """The positive half: the page must derive the temperature and print the reported field."""
+ src = _src40()
+ assert DERIVES_SCALING.search(src), "the bucket must be resolved through core's temp_bucket"
+ assert READS_REPORTED_CONFIDENCE.search(src), "the page must read the reported `confidence`"
+ # and main's page satisfies none of that, so these rules could not have passed before.
+ assert not DERIVES_SCALING.search(OLD_PAGE_40)
+ assert not READS_REPORTED_CONFIDENCE.search(OLD_PAGE_40)
+
+
+for _fn in (test_page40_helpers_are_live_defer_to_core, test_scale_for_replays_core_lookup,
+ test_option_count_matches_the_shipped_preset,
+ test_clamped_buckets_names_only_a_shipped_value_the_runtime_refused,
+ test_entropy_confidence_recomputes_core_exactly,
+ test_page40_drops_the_unconditional_calibration_claim,
+ test_page40_reads_the_scaling_and_the_reported_field):
+ try:
+ _fn()
+ except AssertionError as e:
+ FAIL.append("page-40/%s: %s" % (_fn.__name__, e))
+ except Exception as e: # a crash is a failure, never a silent pass
+ FAIL.append("page-40/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
+ else:
+ PASS.append("page-40/%s" % _fn.__name__)
+# Gate: the two predict_long call sites' comments must describe the None-min_confidence
+# contract truthfully. The pre-fix wording claimed every answer "reports that it ran ungated"
+# / "say so on every answer"; confidence.py:196-197 documents and :211-212 implements the
+# opposite -- the gate writes nothing at all, so the payload carries no `abstention` field.
+import ast as _gate_ast # noqa: E402
+
+_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def _gate_gate_comment(rel_path):
+ """Return the run of comment lines that immediately precede the
+ apply_confidence_gate([result], None) call inside predict_long."""
+ with open(os.path.join(_REPO, rel_path)) as f:
+ lines = f.readlines()
+ tree = _gate_ast.parse("".join(lines))
+ for node in _gate_ast.walk(tree):
+ if isinstance(node, _gate_ast.FunctionDef) and node.name == "predict_long":
+ call_line = None
+ for sub in _gate_ast.walk(node):
+ if (isinstance(sub, _gate_ast.Call)
+ and getattr(sub.func, "id", getattr(sub.func, "attr", None)) == "apply_confidence_gate"
+ and len(sub.args) == 2
+ and isinstance(sub.args[1], _gate_ast.Constant)
+ and sub.args[1].value is None):
+ call_line = sub.lineno - 1
+ break
+ if call_line is None:
+ continue
+ out = []
+ i = call_line - 1
+ while i >= 0 and lines[i].lstrip().startswith("#"):
+ out.insert(0, lines[i].strip().lstrip("#").strip())
+ i -= 1
+ return " ".join(out)
+ return ""
+
+
+_agent_comment = _gate_gate_comment(os.path.join("laya", "agent.py"))
+_onnx_comment = _gate_gate_comment(os.path.join("laya", "onnx_agent.py"))
+
+for _label, _comment in (("agent.py", _agent_comment), ("onnx_agent.py", _onnx_comment)):
+ check_true("gate/comment/%s exists" % _label, len(_comment) > 0, "no comment found")
+ check_true("gate/comment/%s drops the ungated-reports claim" % _label,
+ "reports that it ran ungated" not in _comment
+ and "say so on every answer" not in _comment,
+ "pre-fix wording still on the call site: %r" % _comment)
+ check_true("gate/comment/%s names writes nothing" % _label,
+ "writes nothing" in _comment,
+ "gate comment must name the None-contract: %r" % _comment)
+ check_true("gate/comment/%s names the min_confidence argument" % _label,
+ "min_confidence" in _comment,
+ "gate comment must refer to the argument it is about: %r" % _comment)
+
+# Live witness: apply_confidence_gate([payload], None) leaves the payload byte-identical,
+# so the corrected comment is not itself a claim the code contradicts.
+_witness = {"model": "w", "answers": {"q": {"choice": "a", "confidence": 0.42}}, "usage": {}}
+_witness_before = copy.deepcopy(_witness)
+apply_confidence_gate([_witness], None)
+check("gate/live None writes nothing", _witness, _witness_before)
+check("gate/live None adds no abstention key", "abstention" in _witness["answers"]["q"], False)
+
++
+ # --------------------------------------------- example 18 must not call the threshold a measurement
+ # `examples/18_confidence_gating.py` is the intro-level gating pattern: it picks a 0.85 cutoff on
+ # `answer_confidence` and acts on it. On main it calls the field "the calibrated probability Laya
+ # puts on the answer it reports" and its module docstring says it "branches on Laya's calibrated
+ # confidence". The README's Calibration section (line 1586) says the opposite about the shipped
+ # state: mean ECE on `laya` is 0.466 as shipped, dropping to 0.081 only after refitting one
+ # temperature per (question type, option-count) bucket on held-out data, and both shipped
+ # checkpoints are over-confident. `answer_confidence` is the field temperature scaling fits and
+ # the abstention gate reads -- but "calibrated" is a property a fit earns, not a property of the
+ # field, and the loader emits `RuntimeWarning: ... Treat confidence from the affected entries as
+ # uncalibrated.` on this very checkpoint. The page now reads back the temperature it actually
+ # applies to each of its two questions and states the calibration claim as conditional.
+ from laya.common import QTYPES as _QTYPES18, temp_bucket as _temp_bucket18 # noqa: E402
+
+ EXAMPLE_18 = os.path.join(ROOT, "examples", "18_confidence_gating.py")
+ HELPERS_18 = ("option_count", "scale_for")
+ CORE_GLOBALS_18 = {"temp_bucket", "QTYPES", "math"}
+
+ OLD_PAGE_18 = '''
+ Answers a batch of support emails and branches on Laya\'s calibrated confidence: act
+ The production pattern from the README. Gate on `answer_confidence`: the calibrated
+ probability Laya puts on the answer it reports, defined the same way on every question
+ '''
+
+ UNQUALIFIED_CALIBRATION_18 = re.compile(
+ r"the\s+calibrated\s+probability|Laya['’]s\s+calibrated\s+confidence", re.I)
+ DERIVES_SCALING_18 = re.compile(r"temp_bucket\(")
+ NAMES_CONDITION_18 = re.compile(r"over-confident", re.I)
+ HARDCODED_BUCKET_18 = re.compile(r"[\"'](choice|score|noul):[0-9]")
+
+
+ def _src18():
+ with open(EXAMPLE_18, encoding="utf-8") as fh:
+ return fh.read()
+
+
+ def _helpers18():
+ tree = ast.parse(_src18())
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_18]
+ ns = {"temp_bucket": _temp_bucket18, "QTYPES": _QTYPES18, "math": math}
+ exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_18, "exec"), ns) # noqa: S102
+ return ns
+
+
+ class _FakeAgent18(object):
+ """Only the two temperature attributes `scale_for` reads."""
+
+ def __init__(self, by_options, per_type):
+ self.temperature_by_options = by_options
+ self.temperature = per_type
+
+
+ def _top_literal18(name):
+ for n in ast.parse(_src18()).body:
+ if isinstance(n, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id == name for t in n.targets):
+ return ast.literal_eval(n.value)
+ raise KeyError(name)
+
+
+ def _fn18(name):
+ for n in ast.parse(_src18()).body:
+ if isinstance(n, ast.FunctionDef) and n.name == name:
+ return n
+ raise KeyError(name)
+
+
+ def test_page18_helpers_are_live_and_defer_to_core():
+ defined = sorted(n.name for n in ast.parse(_src18()).body
+ if isinstance(n, ast.FunctionDef) and n.name in HELPERS_18)
+ assert defined == sorted(HELPERS_18), "example 18 lost a helper: %s" % defined
+ tree = ast.parse(_src18())
+ top_loads = {node.id for stmt in tree.body if not isinstance(
+ stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))
+ for node in ast.walk(stmt)
+ if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}
+ unused = set(HELPERS_18) - top_loads
+ assert not unused, "defined but never called at module level: %s" % sorted(unused)
+ for name in HELPERS_18:
+ node = _fn18(name)
+ outside = sorted(n for n in _free_names(node)
+ if not hasattr(builtins, n) and n not in CORE_GLOBALS_18)
+ assert not outside, "%s reads %s from the page" % (name, outside)
+ prints = [c for c in ast.walk(node)
+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == "print"]
+ assert not prints, "%s prints instead of returning" % name
+ assert not HARDCODED_BUCKET_18.search(ast.dump(_fn18("scale_for"))), (
+ "scale_for hardcodes a bucket name instead of calling temp_bucket")
+
+
+ def test_page18_questions_reach_core_bucket_names():
+ """Replay the page's own QUESTIONS through core's temp_bucket and through the example's helper."""
+ ns = _helpers18()
+ questions = _top_literal18("QUESTIONS")
+ per_type = [7.0, 8.0, 9.0]
+ by_options = {"choice:3-5": 1.7601518630981445, "noul:2": 1.983399510383606}
+ fake = _FakeAgent18(by_options, per_type)
+ shapes = {}
+ for qid, q in questions.items():
+ k = ns["option_count"](q)
+ name, applied, in_map = ns["scale_for"](fake, q["type"], k)
+ assert name == _temp_bucket18(_QTYPES18[q["type"]], k), (qid, name)
+ assert abs(applied - by_options[name]) < 1e-9, (qid, applied)
+ assert in_map is True, "%s: %s missing from map" % (qid, name)
+ shapes[qid] = (q["type"], k, name, applied)
+ assert shapes == {
+ "department": ("choice", 4, "choice:3-5", 1.7601518630981445),
+ "refund_requested": ("noul", 2, "noul:2", 1.983399510383606),
+ }, shapes
+
+
+ def test_page18_drops_the_unconditional_calibration_claim():
+ """The two phrases main uses fire on the ban and are gone here; main has no temp_bucket call."""
+ src = _src18()
+ assert UNQUALIFIED_CALIBRATION_18.search(OLD_PAGE_18), "the ban must fire on main's wording"
+ assert not UNQUALIFIED_CALIBRATION_18.search(src), (
+ "the page still calls `answer_confidence` calibrated without condition")
+ assert not DERIVES_SCALING_18.search(OLD_PAGE_18), "main's page has no scaling lookup"
+ assert not NAMES_CONDITION_18.search(OLD_PAGE_18), "main's page states no calibration condition"
+ assert DERIVES_SCALING_18.search(src), "the page must resolve the bucket via core's temp_bucket"
+ assert NAMES_CONDITION_18.search(src), (
+ "the page must carry the README's Calibration condition (mean ECE 0.466 -> 0.081, "
+ "both shipped checkpoints over-confident)")
+ # The README's exact language lives at line 1586; the page must not misquote it.
+ readme = open(os.path.join(ROOT, "README.md"), encoding="utf-8").read()
+ assert "Both checkpoints are over-confident as shipped" in readme
+ assert "0.466 -> 0.081" in readme
+
+
+ for _fn in (test_page18_helpers_are_live_and_defer_to_core,
+ test_page18_questions_reach_core_bucket_names,
+ test_page18_drops_the_unconditional_calibration_claim):
+ try:
+ _fn()
+ except AssertionError as e:
+ FAIL.append("page-18/%s: %s" % (_fn.__name__, e))
+ except Exception as e:
+ FAIL.append("page-18/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
+ else:
+ PASS.append("page-18/%s" % _fn.__name__)
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)