Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #973 from aashish254/fix-example18-calibrated-claim
Nandakishor9f17443351e7
diff --cc tests/test_confidence.pyindex 4bb39df3,2104f654..123f8531--- a/tests/test_confidence.py+++ b/tests/test_confidence.py@@@ -1121,249 -604,147 +1121,391 @@@ for _fn in (test_preset_is_four_flags_aPASS.append("page-28/%s" % _fn.__name__)+# --------------------------------------------- example 40 must not call a raw softmax calibrated+# `examples/40_caching_and_monitoring.py` gates 15 tickets on `answer_confidence`. On main it calls+# that field "the calibrated probability of the answer Laya reports" in both its banner and its+# closing paragraph, and its `bucket()` docstring reads "application policy over a calibrated+# number". `answer_confidence` is max(p) -- the quantity temperature scaling fits and ECE measures+# -- but the README's own Calibration section says that reading holds *only* after temperatures are+# fitted and validated, the shipped checkpoints are over-confident, and the loader emits at load+# time `RuntimeWarning: ... choice:11+=0.10058... -> 0.5. Treat confidence from the affected+# entries as uncalibrated.` The page now reads the temperature each shape was actually scaled by off+# the loaded agent and states the calibration as conditional.+#+# No weights are loaded here: the four helpers are pulled from the example's AST and exec'd against+# a fabricated agent, and `scale_for`'s lookup is checked against core's own `temp_bucket`, so the+# gate drives the code the page runs instead of re-reading its prose.+from laya.common import QTYPES as _QTYPES40, temp_bucket as _temp_bucket # noqa: E402++EXAMPLE_40 = os.path.join(ROOT, "examples", "40_caching_and_monitoring.py")+HELPERS_40 = ("option_count", "scale_for", "clamped_buckets", "entropy_confidence")+# Names a helper may read from the page's world: the two core symbols it must defer to, and `math`+# for the entropy. Any other non-builtin free name means this gate cannot drive that helper.+CORE_GLOBALS_40 = {"temp_bucket", "QTYPES", "math"}++# main's page, as literal source lines. Every ban is witnessed against this text; every positive+# rule below is witnessed by showing this text does NOT satisfy it.+OLD_PAGE_40 = '''+ the triage preset and gates on `answer_confidence`, the calibrated probability of the+ """Our thresholds, not the model's: application policy over a calibrated number."""+ The monitor gates on `answer_confidence`: the calibrated probability of the answer Laya+ distribution has H/log(k) = %.2f, so `confidence` is %.2f while+ """ % (entropy, 1 - entropy, vague["intent"]["answer_confidence"]))+'''++UNCONDITIONAL_CALIBRATION = re.compile(r"the calibrated probability", re.I)+POLICY_OVER_CALIBRATED = re.compile(r"over a calibrated number", re.I)+SUBSTITUTED_CONFIDENCE = re.compile(r"1\s*-\s*entropy")+DERIVES_SCALING = re.compile(r"temp_bucket\(")+READS_REPORTED_CONFIDENCE = re.compile(r"\[[\"']confidence[\"']\]")+HARDCODED_BUCKET = re.compile(r"[\"'](choice|score|noul):[0-9]")+++def _src40():+ with open(EXAMPLE_40, encoding="utf-8") as fh:+ return fh.read()+++def _helpers40():+ """Exec only the example's top-level helpers -- its `load()` call needs weights."""+ tree = ast.parse(_src40())+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_40]+ ns = {"temp_bucket": _temp_bucket, "QTYPES": _QTYPES40, "math": math}+ exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_40, "exec"), ns) # noqa: S102+ return ns+++def _called40():+ tree = ast.parse(_src40())+ top = [n for n in tree.body if not isinstance(+ n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))]+ return {node.id for stmt in top for node in ast.walk(stmt)+ if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}+++def _fn40(name):+ for n in ast.parse(_src40()).body:+ if isinstance(n, ast.FunctionDef) and n.name == name:+ return n+ raise KeyError(name)+++class _FakeAgent(object):+ """Only the three temperature attributes `scale_for` and `clamped_buckets` read."""++ def __init__(self, by_options, per_type, raw=None):+ self.temperature_by_options = by_options+ self.temperature = per_type+ self.temperature_by_options_raw = dict(by_options) if raw is None else raw+++def test_page40_helpers_are_live_defer_to_core():+ defined = sorted(n.name for n in ast.parse(_src40()).body+ if isinstance(n, ast.FunctionDef) and n.name in HELPERS_40)+ assert defined == sorted(HELPERS_40), "example 40 lost a helper: %s" % defined+ ns = _helpers40()+ unused = set(HELPERS_40) - _called40()+ assert not unused, "defined but never called at module level: %s" % sorted(unused)+ for name in HELPERS_40:+ node = _fn40(name)+ outside = sorted(n for n in _free_names(node)+ if not hasattr(builtins, n) and n not in CORE_GLOBALS_40)+ assert not outside, "%s reads %s from the page, so this gate cannot drive it" % (name, outside)+ prints = [c for c in ast.walk(node)+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == "print"]+ assert not prints, "%s prints instead of returning" % name+ # `scale_for` must resolve the bucket through core, not carry a typed-in table.+ assert not HARDCODED_BUCKET.search(ast.dump(_fn40("scale_for"))), (+ "scale_for hardcodes a bucket name instead of calling temp_bucket")+++def test_scale_for_replays_core_lookup():+ """For every shape, scale_for returns exactly `temperature_by_options.get(bucket, per_type)`."""+ ns = _helpers40()+ per_type = [7.0, 8.0, 9.0] # choice, score, noul fallbacks+ by_options = {"choice:6-10": 1.0, "noul:2": 1.9834, "score:3-5": 1.2514}+ fake = _FakeAgent(by_options, per_type)+ for qtype, k in (("choice", 6), ("noul", 2), ("score", 4), ("noul", 2)):+ name, applied, in_map = ns["scale_for"](fake, qtype, k)+ bucket = _temp_bucket(_QTYPES40[qtype], k)+ assert name == bucket, (qtype, k, name, bucket)+ assert in_map == (bucket in by_options), (bucket, in_map)+ assert abs(applied - by_options.get(bucket, per_type[_QTYPES40[qtype]])) < 1e-9, (+ "%s: %r not core's get(...)" % (bucket, applied))+ # the fallback arm: an unmapped shape must fall to the per-type temperature, flagged not-in-map.+ name, applied, in_map = ns["scale_for"](_FakeAgent({}, [0.5, 0.6, 0.7]), "choice", 12)+ assert (name, applied, in_map) == ("choice:11+", 0.5, False), (name, applied, in_map)+ # a mapped temperature of exactly 1.0 must be reported as 1.0, not nudged.+ assert ns["scale_for"](_FakeAgent({"choice:6-10": 1.0}, [1.0, 1.0, 1.0]), "choice", 6) == (+ "choice:6-10", 1.0, True)+++def test_option_count_matches_the_shipped_preset():+ ns = _helpers40()+ counts = {qid: ns["option_count"](q) for qid, q in laya.triage_questions().items()}+ assert counts == {"intent": 6, "is_urgent": 2, "frustration": 4,+ "refund_requested": 2, "churn_risk": 2}, counts+++def test_clamped_buckets_names_only_a_shipped_value_the_runtime_refused():+ ns = _helpers40()+ # `choice:11+` ships below TEMP_MIN and is clamped to 0.5; everything else ships unchanged.+ applied = {"choice:11+": 0.5, "noul:2": 1.9834}+ raw = {"choice:11+": 0.10058280825614929, "noul:2": 1.9834}+ out = ns["clamped_buckets"](_FakeAgent(applied, [0.5, 0.5, 0.5], raw))+ assert out == ["choice:11+=0.1006 -> 0.5000"], out+ # and an honest checkpoint reports nothing.+ same = {"noul:2": 1.9834}+ assert ns["clamped_buckets"](_FakeAgent(same, [0.5, 0.5, 0.5], dict(same))) == []+++def test_entropy_confidence_recomputes_core_exactly():+ ns = _helpers40()+ for dist in ([0.5, 0.3, 0.2], [0.9, 0.1], [0.25, 0.25, 0.25, 0.25], [1.0, 0.0, 0.0]):+ p = np.array(dist, dtype=float)+ got = ns["entropy_confidence"](dist)+ want = confidence_from_probs(p, len(dist))+ assert close(got, want, 1e-9), (dist, got, want)+ assert ns["entropy_confidence"]([1.0]) == 1.0, "k<2 is 1.0 by definition"+++def test_page40_drops_the_unconditional_calibration_claim():+ """Each ban fires on main's page and not on this one -- a ban with no witness is a guess."""+ src = _src40()+ for name, rule in (("'the calibrated probability'", UNCONDITIONAL_CALIBRATION),+ ("'over a calibrated number'", POLICY_OVER_CALIBRATED),+ ("prints `1 - entropy` as the confidence", SUBSTITUTED_CONFIDENCE)):+ assert rule.search(OLD_PAGE_40), "%s does not fire on the wording it bans" % name+ assert not rule.search(src), "%s is still in example 40" % name+++def test_page40_reads_the_scaling_and_the_reported_field():+ """The positive half: the page must derive the temperature and print the reported field."""+ src = _src40()+ assert DERIVES_SCALING.search(src), "the bucket must be resolved through core's temp_bucket"+ assert READS_REPORTED_CONFIDENCE.search(src), "the page must read the reported `confidence`"+ # and main's page satisfies none of that, so these rules could not have passed before.+ assert not DERIVES_SCALING.search(OLD_PAGE_40)+ assert not READS_REPORTED_CONFIDENCE.search(OLD_PAGE_40)+++for _fn in (test_page40_helpers_are_live_defer_to_core, test_scale_for_replays_core_lookup,+ test_option_count_matches_the_shipped_preset,+ test_clamped_buckets_names_only_a_shipped_value_the_runtime_refused,+ test_entropy_confidence_recomputes_core_exactly,+ test_page40_drops_the_unconditional_calibration_claim,+ test_page40_reads_the_scaling_and_the_reported_field):+ try:+ _fn()+ except AssertionError as e:+ FAIL.append("page-40/%s: %s" % (_fn.__name__, e))+ except Exception as e: # a crash is a failure, never a silent pass+ FAIL.append("page-40/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))+ else:+ PASS.append("page-40/%s" % _fn.__name__)+# Gate: the two predict_long call sites' comments must describe the None-min_confidence+# contract truthfully. The pre-fix wording claimed every answer "reports that it ran ungated"+# / "say so on every answer"; confidence.py:196-197 documents and :211-212 implements the+# opposite -- the gate writes nothing at all, so the payload carries no `abstention` field.+import ast as _gate_ast # noqa: E402++_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+++def _gate_gate_comment(rel_path):+ """Return the run of comment lines that immediately precede the+ apply_confidence_gate([result], None) call inside predict_long."""+ with open(os.path.join(_REPO, rel_path)) as f:+ lines = f.readlines()+ tree = _gate_ast.parse("".join(lines))+ for node in _gate_ast.walk(tree):+ if isinstance(node, _gate_ast.FunctionDef) and node.name == "predict_long":+ call_line = None+ for sub in _gate_ast.walk(node):+ if (isinstance(sub, _gate_ast.Call)+ and getattr(sub.func, "id", getattr(sub.func, "attr", None)) == "apply_confidence_gate"+ and len(sub.args) == 2+ and isinstance(sub.args[1], _gate_ast.Constant)+ and sub.args[1].value is None):+ call_line = sub.lineno - 1+ break+ if call_line is None:+ continue+ out = []+ i = call_line - 1+ while i >= 0 and lines[i].lstrip().startswith("#"):+ out.insert(0, lines[i].strip().lstrip("#").strip())+ i -= 1+ return " ".join(out)+ return ""+++_agent_comment = _gate_gate_comment(os.path.join("laya", "agent.py"))+_onnx_comment = _gate_gate_comment(os.path.join("laya", "onnx_agent.py"))++for _label, _comment in (("agent.py", _agent_comment), ("onnx_agent.py", _onnx_comment)):+ check_true("gate/comment/%s exists" % _label, len(_comment) > 0, "no comment found")+ check_true("gate/comment/%s drops the ungated-reports claim" % _label,+ "reports that it ran ungated" not in _comment+ and "say so on every answer" not in _comment,+ "pre-fix wording still on the call site: %r" % _comment)+ check_true("gate/comment/%s names writes nothing" % _label,+ "writes nothing" in _comment,+ "gate comment must name the None-contract: %r" % _comment)+ check_true("gate/comment/%s names the min_confidence argument" % _label,+ "min_confidence" in _comment,+ "gate comment must refer to the argument it is about: %r" % _comment)++# Live witness: apply_confidence_gate([payload], None) leaves the payload byte-identical,+# so the corrected comment is not itself a claim the code contradicts.+_witness = {"model": "w", "answers": {"q": {"choice": "a", "confidence": 0.42}}, "usage": {}}+_witness_before = copy.deepcopy(_witness)+apply_confidence_gate([_witness], None)+check("gate/live None writes nothing", _witness, _witness_before)+check("gate/live None adds no abstention key", "abstention" in _witness["answers"]["q"], False)++++ # --------------------------------------------- example 18 must not call the threshold a measurement+ # `examples/18_confidence_gating.py` is the intro-level gating pattern: it picks a 0.85 cutoff on+ # `answer_confidence` and acts on it. On main it calls the field "the calibrated probability Laya+ # puts on the answer it reports" and its module docstring says it "branches on Laya's calibrated+ # confidence". The README's Calibration section (line 1586) says the opposite about the shipped+ # state: mean ECE on `laya` is 0.466 as shipped, dropping to 0.081 only after refitting one+ # temperature per (question type, option-count) bucket on held-out data, and both shipped+ # checkpoints are over-confident. `answer_confidence` is the field temperature scaling fits and+ # the abstention gate reads -- but "calibrated" is a property a fit earns, not a property of the+ # field, and the loader emits `RuntimeWarning: ... Treat confidence from the affected entries as+ # uncalibrated.` on this very checkpoint. The page now reads back the temperature it actually+ # applies to each of its two questions and states the calibration claim as conditional.+ from laya.common import QTYPES as _QTYPES18, temp_bucket as _temp_bucket18 # noqa: E402++ EXAMPLE_18 = os.path.join(ROOT, "examples", "18_confidence_gating.py")+ HELPERS_18 = ("option_count", "scale_for")+ CORE_GLOBALS_18 = {"temp_bucket", "QTYPES", "math"}++ OLD_PAGE_18 = '''+ Answers a batch of support emails and branches on Laya\'s calibrated confidence: act+ The production pattern from the README. Gate on `answer_confidence`: the calibrated+ probability Laya puts on the answer it reports, defined the same way on every question+ '''++ UNQUALIFIED_CALIBRATION_18 = re.compile(+ r"the\s+calibrated\s+probability|Laya['’]s\s+calibrated\s+confidence", re.I)+ DERIVES_SCALING_18 = re.compile(r"temp_bucket\(")+ NAMES_CONDITION_18 = re.compile(r"over-confident", re.I)+ HARDCODED_BUCKET_18 = re.compile(r"[\"'](choice|score|noul):[0-9]")+++ def _src18():+ with open(EXAMPLE_18, encoding="utf-8") as fh:+ return fh.read()+++ def _helpers18():+ tree = ast.parse(_src18())+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_18]+ ns = {"temp_bucket": _temp_bucket18, "QTYPES": _QTYPES18, "math": math}+ exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_18, "exec"), ns) # noqa: S102+ return ns+++ class _FakeAgent18(object):+ """Only the two temperature attributes `scale_for` reads."""++ def __init__(self, by_options, per_type):+ self.temperature_by_options = by_options+ self.temperature = per_type+++ def _top_literal18(name):+ for n in ast.parse(_src18()).body:+ if isinstance(n, ast.Assign) and any(+ isinstance(t, ast.Name) and t.id == name for t in n.targets):+ return ast.literal_eval(n.value)+ raise KeyError(name)+++ def _fn18(name):+ for n in ast.parse(_src18()).body:+ if isinstance(n, ast.FunctionDef) and n.name == name:+ return n+ raise KeyError(name)+++ def test_page18_helpers_are_live_and_defer_to_core():+ defined = sorted(n.name for n in ast.parse(_src18()).body+ if isinstance(n, ast.FunctionDef) and n.name in HELPERS_18)+ assert defined == sorted(HELPERS_18), "example 18 lost a helper: %s" % defined+ tree = ast.parse(_src18())+ top_loads = {node.id for stmt in tree.body if not isinstance(+ stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))+ for node in ast.walk(stmt)+ if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}+ unused = set(HELPERS_18) - top_loads+ assert not unused, "defined but never called at module level: %s" % sorted(unused)+ for name in HELPERS_18:+ node = _fn18(name)+ outside = sorted(n for n in _free_names(node)+ if not hasattr(builtins, n) and n not in CORE_GLOBALS_18)+ assert not outside, "%s reads %s from the page" % (name, outside)+ prints = [c for c in ast.walk(node)+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == "print"]+ assert not prints, "%s prints instead of returning" % name+ assert not HARDCODED_BUCKET_18.search(ast.dump(_fn18("scale_for"))), (+ "scale_for hardcodes a bucket name instead of calling temp_bucket")+++ def test_page18_questions_reach_core_bucket_names():+ """Replay the page's own QUESTIONS through core's temp_bucket and through the example's helper."""+ ns = _helpers18()+ questions = _top_literal18("QUESTIONS")+ per_type = [7.0, 8.0, 9.0]+ by_options = {"choice:3-5": 1.7601518630981445, "noul:2": 1.983399510383606}+ fake = _FakeAgent18(by_options, per_type)+ shapes = {}+ for qid, q in questions.items():+ k = ns["option_count"](q)+ name, applied, in_map = ns["scale_for"](fake, q["type"], k)+ assert name == _temp_bucket18(_QTYPES18[q["type"]], k), (qid, name)+ assert abs(applied - by_options[name]) < 1e-9, (qid, applied)+ assert in_map is True, "%s: %s missing from map" % (qid, name)+ shapes[qid] = (q["type"], k, name, applied)+ assert shapes == {+ "department": ("choice", 4, "choice:3-5", 1.7601518630981445),+ "refund_requested": ("noul", 2, "noul:2", 1.983399510383606),+ }, shapes+++ def test_page18_drops_the_unconditional_calibration_claim():+ """The two phrases main uses fire on the ban and are gone here; main has no temp_bucket call."""+ src = _src18()+ assert UNQUALIFIED_CALIBRATION_18.search(OLD_PAGE_18), "the ban must fire on main's wording"+ assert not UNQUALIFIED_CALIBRATION_18.search(src), (+ "the page still calls `answer_confidence` calibrated without condition")+ assert not DERIVES_SCALING_18.search(OLD_PAGE_18), "main's page has no scaling lookup"+ assert not NAMES_CONDITION_18.search(OLD_PAGE_18), "main's page states no calibration condition"+ assert DERIVES_SCALING_18.search(src), "the page must resolve the bucket via core's temp_bucket"+ assert NAMES_CONDITION_18.search(src), (+ "the page must carry the README's Calibration condition (mean ECE 0.466 -> 0.081, "+ "both shipped checkpoints over-confident)")+ # The README's exact language lives at line 1586; the page must not misquote it.+ readme = open(os.path.join(ROOT, "README.md"), encoding="utf-8").read()+ assert "Both checkpoints are over-confident as shipped" in readme+ assert "0.466 -> 0.081" in readme+++ for _fn in (test_page18_helpers_are_live_and_defer_to_core,+ test_page18_questions_reach_core_bucket_names,+ test_page18_drops_the_unconditional_calibration_claim):+ try:+ _fn()+ except AssertionError as e:+ FAIL.append("page-18/%s: %s" % (_fn.__name__, e))+ except Exception as e:+ FAIL.append("page-18/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))+ else:+ PASS.append("page-18/%s" % _fn.__name__)+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL", f)