Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(example): the demo's certainty chip must name the question type's formula
Aashishea70779a52e9
diff --git a/examples/server.py b/examples/server.pyindex cf5f095d..d46f30a9 100644--- a/examples/server.py+++ b/examples/server.py@@ -1215,10 +1215,20 @@ def _answer_dist(kind: str, ans: Dict[str, Any], question: Dict[str, Any]) -> tu_CERTAINTY_TIP = (- "The `confidence` field is 1 minus the normalized entropy of the whole distribution: how peaked it is. "- "It is not calibrated, so do not gate on it. `answer_confidence` is the probability of the "- "reported answer, calibrated so that answers returned at 0.9 are right about 90% of the time."+ "`confidence` is not one formula, and it is not calibrated, so do not gate on it. On a `choice` or a "+ "`score` it is 1 minus the normalized entropy of the whole distribution: how peaked it is, on a scale "+ "that moves with the number of options. On a `noul` it is the probability of the side being reported, "+ "max(p_true, 1 - p_true). `answer_confidence` is the probability of the reported answer, the quantity "+ "temperature scaling fits and this repository's calibration figures are computed on; a figure measured "+ "on a benchmark is not a promise about your traffic, so gate on a threshold you have measured.")+# The browser builds the same chip, so the boot script serializes this dict to it instead of the two+# languages each typing a formula for a field whose formula depends on the question type.+_CERTAINTY_LABEL = {+ "choice": "entropy, not calibrated",+ "score": "entropy, not calibrated",+ "noul": "the reported side, not calibrated",+}_UNSURE = 0.6@@ -1286,7 +1296,8 @@ def _answer_row(name: str, ans: Any, question: Dict[str, Any], n: int) -> str:if isinstance(certainty, (int, float)) and abs(float(certainty) - calibrated) > 5e-5:meta += (f"<span class='tipw'><button type='button' class='tipt' aria-describedby='tip-{n}'>"- f"<code>confidence</code> <b>{float(certainty):.4f}</b> entropy, not calibrated {_icon('info')}</button>"+ f"<code>confidence</code> <b>{float(certainty):.4f}</b> "+ f"{_CERTAINTY_LABEL[kind]} {_icon('info')}</button>"f"<span class='tipb' role='tooltip' id='tip-{n}'>{_with_code(_CERTAINTY_TIP)}</span></span>")return (@@ -3020,8 +3031,8 @@ function answerRow(key, ans, q, n) {if (typeof ans.confidence === "number" && Math.abs(ans.confidence - calibrated) > 5e-5)meta.append(h("span", {class: "tipw"},h("button", {type: "button", class: "tipt", "aria-describedby": "tip-" + n},- h("code", {text: "confidence"}), h("b", {text: ans.confidence.toFixed(4)}), "entropy, not calibrated",- icon("info")),+ h("code", {text: "confidence"}), h("b", {text: ans.confidence.toFixed(4)}),+ CERTAINTY_LABEL[kind], icon("info")),h("span", {class: "tipb", role: "tooltip", id: "tip-" + n}, withCode(CERTAINTY_TIP))));return h("details", {class: "ans", open: true}, h("summary", null, head, verdict),h("div", {class: "a-body"}, score != null && maxLevel > 0 ? scale(score, maxLevel) : null, dist, meta));@@ -3506,6 +3517,7 @@ def index(request: Request) -> HTMLResponse:f"<script>const EXAMPLE = {_script_json(example)}; const PRESETS = {presets_js};"f" const QTYPES = {_script_json(sorted(getattr(laya, 'QTYPES', {}) or {}) or ['choice', 'score', 'noul'])};"f" const LIMITS = {_script_json({'questions': MAX_QUESTIONS, 'stateChars': MAX_STATE_CHARS})};"+ f" const CERTAINTY_LABEL = {_script_json(_CERTAINTY_LABEL)};""</script>")return _html(diff --git a/tests/test_example_server_limits.py b/tests/test_example_server_limits.pyindex 776c13d4..b17549b9 100644--- a/tests/test_example_server_limits.py+++ b/tests/test_example_server_limits.py@@ -65,9 +65,11 @@ this file.Run: python tests/test_example_server_limits.py"""+import astimport importlibimport jsonimport os+import re as _re # `main()` already binds a local `re` further down; F811 off bothimport sysimport types@@ -841,6 +843,139 @@ def main():_router_mod._ALIASES.pop("es", None)demo = importlib.reload(demo)+ # --- the certainty chip: it names a formula, so it has to name that type's formula ---+ # `confidence` is per-type in the decoders (normalized entropy on `choice`/`score`, the reported+ # side's probability on `noul`), but the demo's chip said "entropy" for all three and its tooltip+ # defined the field as one formula. The three have to agree: what the agents write, what the chip+ # says per type, and what the tooltip attributes to which type. Weight-free -- no Router is built.+ with open(os.path.join(ROOT, "examples", "server.py"), encoding="utf-8") as handle:+ srv = handle.read()++ def _decoded_kinds():+ """The question types this page draws distribution rows for."""+ tree = ast.parse(srv)+ fn = next(n for n in ast.walk(tree)+ if isinstance(n, ast.FunctionDef) and n.name == "_answer_dist")+ return sorted({c.value for n in ast.walk(fn) if isinstance(n, ast.Compare)+ and isinstance(n.left, ast.Name) and n.left.id == "kind"+ for c in n.comparators+ if isinstance(c, ast.Constant) and isinstance(c.value, str)})++ def _decoder_formulas(relpath):+ """{question type: "entropy" | "maxp"}, read off the `confidence` each builder writes."""+ with open(os.path.join(ROOT, relpath), encoding="utf-8") as handle:+ tree = ast.parse(handle.read())+ fn = next(n for n in ast.walk(tree)+ if isinstance(n, ast.FunctionDef) and n.name == "_decode_answers")+ bound = {}+ for node in ast.walk(fn):+ if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):+ bound[node.targets[0].id] = node.value+ got = {}+ for node in ast.walk(fn):+ if not isinstance(node, ast.Dict):+ continue+ fields = {k.value: v for k, v in zip(node.keys, node.values)+ if isinstance(k, ast.Constant) and isinstance(k.value, str)}+ if "type" not in fields or "confidence" not in fields:+ continue+ kind = fields["type"].value+ expr = fields["confidence"]+ while isinstance(expr, ast.Name) and expr.id in bound: # `conf_score = round(...)`+ expr = bound[expr.id]+ called = {c.func.id for c in ast.walk(expr)+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)}+ formula = "entropy" if "confidence_from_probs" in called else "maxp" if "max" in called else None+ if formula is None:+ raise SystemExit("%s: %s's `confidence` is neither confidence_from_probs nor a max(): %s"+ % (relpath, kind, ast.dump(expr)))+ if got.get(kind) not in (None, formula):+ raise SystemExit("%s: %s writes `confidence` two ways" % (relpath, kind))+ got[kind] = formula+ return got++ def _sentence_types(prose, kinds):+ """{formula: the question types named, in backticks, in the same sentence as it}."""+ got = {}+ for sentence in _re.split(r"(?<=[.!?])\s+", prose):+ named = {t for t in _re.findall(r"`([a-z]+)`", sentence) if t in kinds}+ if not named:+ continue+ for formula, words in (("entropy", "entropy"), ("maxp", "max(p")):+ if words in sentence:+ got.setdefault(formula, set()).update(named)+ return {key: sorted(value) for key, value in sorted(got.items())}++ def _chip(row):+ hit = _re.search(r"<code>confidence</code> <b>([0-9.]+)</b> (.*?) <svg", row)+ return (hit.group(1), hit.group(2)) if hit else (None, None)++ kinds = _decoded_kinds()+ label = demo._CERTAINTY_LABEL+ tip = demo._CERTAINTY_TIP+ ok("the chip has a label for every type the page draws rows for, and no other",+ sorted(label) == kinds, "%r vs the page's %r" % (sorted(label), kinds))++ page_formula = {kind: ("entropy" if "entropy" in text else "maxp") for kind, text in sorted(label.items())}+ for relpath in ("laya/agent.py", "laya/onnx_agent.py"):+ decoders = _decoder_formulas(relpath)+ ok("%s writes `confidence` the way this page labels it" % relpath,+ decoders == page_formula, "%r vs the chip's %r" % (decoders, page_formula))++ said = _sentence_types(tip, set(kinds))+ ok("the tooltip attributes entropy to exactly the types whose chip says entropy",+ said.get("entropy") == sorted(k for k, f in page_formula.items() if f == "entropy"),+ "tooltip %r vs chip %r" % (said, page_formula))+ ok("and the reported side's max(p_true, 1 - p_true) to exactly the types whose chip does not",+ said.get("maxp") == sorted(k for k, f in page_formula.items() if f == "maxp"),+ "tooltip %r vs chip %r" % (said, page_formula))+ ok("so the tooltip names both formulas rather than defining the field as one",+ len(said) == 2 and set(said) == {"entropy", "maxp"}, repr(said))++ OLD_BLANKET = "The `confidence` field is 1 minus the normalized entropy of the whole distribution:"+ ok("the blanket sentence this replaced is gone", OLD_BLANKET not in srv, OLD_BLANKET)+ ok("and the page no longer promises an accuracy rate for a confidence value",+ "are right about 90% of the time" not in srv+ and "gate on a threshold you have measured" in tip, tip)++ OLD_JS_CHIP = 'toFixed(4)}), "entropy, not calibrated",\n icon("info")'+ ok("the browser's chip does not type a formula beside itself either",+ OLD_JS_CHIP not in srv and "CERTAINTY_LABEL[kind]" in srv)+ page = client.get("/").text+ ok("the browser gets the label table from this dict, not a copy of it",+ "const CERTAINTY_LABEL = " + json.dumps(label) in page, json.dumps(label))+ ok("and the tooltip it shows is this same text",+ "is not one formula, and it is not calibrated, so do not gate on it" in page)++ import numpy as np+ from laya.common import confidence_from_probs++ spread = [0.62, 0.25, 0.13]+ ent = round(confidence_from_probs(np.asarray(spread), 3), 4)+ row = demo._answer_row("q", {"type": "choice", "choice": "billing", "answer_confidence": 0.62,+ "confidence": ent, "probabilities": {"billing": 0.62, "support": 0.25,+ "sales": 0.13}},+ {"type": "choice", "instructions": "?"}, 1)+ ok("a choice row reads as the entropy it is", _chip(row) == ("%.4f" % ent, label["choice"]),+ "%r" % (_chip(row),))+ row = demo._answer_row("q", {"type": "score", "score": 1.2, "answer_confidence": 0.5,+ "confidence": round(confidence_from_probs(np.asarray([0.3, 0.5, 0.2]), 3), 4),+ "legend": {"0": "low", "1": "mid", "2": "high"},+ "probabilities": {"0": 0.3, "1": 0.5, "2": 0.2}},+ {"type": "score", "instructions": "?"}, 1)+ ok("so does a score row", _chip(row) == ("%.4f" % round(+ confidence_from_probs(np.asarray([0.3, 0.5, 0.2]), 3), 4), label["score"]), "%r" % (_chip(row),))++ p_true = 0.8727+ side = round(max(p_true, 1 - p_true), 4)+ two_option_entropy = round(confidence_from_probs(np.asarray([p_true, 1 - p_true]), 2), 4)+ row = demo._answer_row("q", {"type": "noul", "noul": p_true, "answer_confidence": 0.79,+ "confidence": side}, {"type": "noul", "instructions": "?"}, 1)+ ok("a noul row's confidence is the side being reported, not an entropy",+ side == p_true and abs(side - two_option_entropy) > 0.4+ and _chip(row) == ("%.4f" % side, label["noul"])+ and "entropy" not in _chip(row)[1], "%r %r" % (_chip(row), two_option_entropy))+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL:print(" FAIL " + f)