Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(example): the demo's certainty chip must name the question type's formula

Aashishea70779a52e9
diff --git a/examples/server.py b/examples/server.py
index cf5f095d..d46f30a9 100644
--- a/examples/server.py
+++ b/examples/server.py
@@ -1215,10 +1215,20 @@ def _answer_dist(kind: str, ans: Dict[str, Any], question: Dict[str, Any]) -> tu
_CERTAINTY_TIP = (
- "The `confidence` field is 1 minus the normalized entropy of the whole distribution: how peaked it is. "
- "It is not calibrated, so do not gate on it. `answer_confidence` is the probability of the "
- "reported answer, calibrated so that answers returned at 0.9 are right about 90% of the time."
+ "`confidence` is not one formula, and it is not calibrated, so do not gate on it. On a `choice` or a "
+ "`score` it is 1 minus the normalized entropy of the whole distribution: how peaked it is, on a scale "
+ "that moves with the number of options. On a `noul` it is the probability of the side being reported, "
+ "max(p_true, 1 - p_true). `answer_confidence` is the probability of the reported answer, the quantity "
+ "temperature scaling fits and this repository's calibration figures are computed on; a figure measured "
+ "on a benchmark is not a promise about your traffic, so gate on a threshold you have measured."
)
+# The browser builds the same chip, so the boot script serializes this dict to it instead of the two
+# languages each typing a formula for a field whose formula depends on the question type.
+_CERTAINTY_LABEL = {
+ "choice": "entropy, not calibrated",
+ "score": "entropy, not calibrated",
+ "noul": "the reported side, not calibrated",
+}
_UNSURE = 0.6
@@ -1286,7 +1296,8 @@ def _answer_row(name: str, ans: Any, question: Dict[str, Any], n: int) -> str:
if isinstance(certainty, (int, float)) and abs(float(certainty) - calibrated) > 5e-5:
meta += (
f"<span class='tipw'><button type='button' class='tipt' aria-describedby='tip-{n}'>"
- f"<code>confidence</code> <b>{float(certainty):.4f}</b> entropy, not calibrated {_icon('info')}</button>"
+ f"<code>confidence</code> <b>{float(certainty):.4f}</b> "
+ f"{_CERTAINTY_LABEL[kind]} {_icon('info')}</button>"
f"<span class='tipb' role='tooltip' id='tip-{n}'>{_with_code(_CERTAINTY_TIP)}</span></span>"
)
return (
@@ -3020,8 +3031,8 @@ function answerRow(key, ans, q, n) {
if (typeof ans.confidence === "number" && Math.abs(ans.confidence - calibrated) > 5e-5)
meta.append(h("span", {class: "tipw"},
h("button", {type: "button", class: "tipt", "aria-describedby": "tip-" + n},
- h("code", {text: "confidence"}), h("b", {text: ans.confidence.toFixed(4)}), "entropy, not calibrated",
- icon("info")),
+ h("code", {text: "confidence"}), h("b", {text: ans.confidence.toFixed(4)}),
+ CERTAINTY_LABEL[kind], icon("info")),
h("span", {class: "tipb", role: "tooltip", id: "tip-" + n}, withCode(CERTAINTY_TIP))));
return h("details", {class: "ans", open: true}, h("summary", null, head, verdict),
h("div", {class: "a-body"}, score != null && maxLevel > 0 ? scale(score, maxLevel) : null, dist, meta));
@@ -3506,6 +3517,7 @@ def index(request: Request) -> HTMLResponse:
f"<script>const EXAMPLE = {_script_json(example)}; const PRESETS = {presets_js};"
f" const QTYPES = {_script_json(sorted(getattr(laya, 'QTYPES', {}) or {}) or ['choice', 'score', 'noul'])};"
f" const LIMITS = {_script_json({'questions': MAX_QUESTIONS, 'stateChars': MAX_STATE_CHARS})};"
+ f" const CERTAINTY_LABEL = {_script_json(_CERTAINTY_LABEL)};"
"</script>"
)
return _html(
diff --git a/tests/test_example_server_limits.py b/tests/test_example_server_limits.py
index 776c13d4..b17549b9 100644
--- a/tests/test_example_server_limits.py
+++ b/tests/test_example_server_limits.py
@@ -65,9 +65,11 @@ this file.
Run: python tests/test_example_server_limits.py
"""
+import ast
import importlib
import json
import os
+import re as _re # `main()` already binds a local `re` further down; F811 off both
import sys
import types
@@ -841,6 +843,139 @@ def main():
_router_mod._ALIASES.pop("es", None)
demo = importlib.reload(demo)
+ # --- the certainty chip: it names a formula, so it has to name that type's formula ---
+ # `confidence` is per-type in the decoders (normalized entropy on `choice`/`score`, the reported
+ # side's probability on `noul`), but the demo's chip said "entropy" for all three and its tooltip
+ # defined the field as one formula. The three have to agree: what the agents write, what the chip
+ # says per type, and what the tooltip attributes to which type. Weight-free -- no Router is built.
+ with open(os.path.join(ROOT, "examples", "server.py"), encoding="utf-8") as handle:
+ srv = handle.read()
+
+ def _decoded_kinds():
+ """The question types this page draws distribution rows for."""
+ tree = ast.parse(srv)
+ fn = next(n for n in ast.walk(tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "_answer_dist")
+ return sorted({c.value for n in ast.walk(fn) if isinstance(n, ast.Compare)
+ and isinstance(n.left, ast.Name) and n.left.id == "kind"
+ for c in n.comparators
+ if isinstance(c, ast.Constant) and isinstance(c.value, str)})
+
+ def _decoder_formulas(relpath):
+ """{question type: "entropy" | "maxp"}, read off the `confidence` each builder writes."""
+ with open(os.path.join(ROOT, relpath), encoding="utf-8") as handle:
+ tree = ast.parse(handle.read())
+ fn = next(n for n in ast.walk(tree)
+ if isinstance(n, ast.FunctionDef) and n.name == "_decode_answers")
+ bound = {}
+ for node in ast.walk(fn):
+ if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+ bound[node.targets[0].id] = node.value
+ got = {}
+ for node in ast.walk(fn):
+ if not isinstance(node, ast.Dict):
+ continue
+ fields = {k.value: v for k, v in zip(node.keys, node.values)
+ if isinstance(k, ast.Constant) and isinstance(k.value, str)}
+ if "type" not in fields or "confidence" not in fields:
+ continue
+ kind = fields["type"].value
+ expr = fields["confidence"]
+ while isinstance(expr, ast.Name) and expr.id in bound: # `conf_score = round(...)`
+ expr = bound[expr.id]
+ called = {c.func.id for c in ast.walk(expr)
+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)}
+ formula = "entropy" if "confidence_from_probs" in called else "maxp" if "max" in called else None
+ if formula is None:
+ raise SystemExit("%s: %s's `confidence` is neither confidence_from_probs nor a max(): %s"
+ % (relpath, kind, ast.dump(expr)))
+ if got.get(kind) not in (None, formula):
+ raise SystemExit("%s: %s writes `confidence` two ways" % (relpath, kind))
+ got[kind] = formula
+ return got
+
+ def _sentence_types(prose, kinds):
+ """{formula: the question types named, in backticks, in the same sentence as it}."""
+ got = {}
+ for sentence in _re.split(r"(?<=[.!?])\s+", prose):
+ named = {t for t in _re.findall(r"`([a-z]+)`", sentence) if t in kinds}
+ if not named:
+ continue
+ for formula, words in (("entropy", "entropy"), ("maxp", "max(p")):
+ if words in sentence:
+ got.setdefault(formula, set()).update(named)
+ return {key: sorted(value) for key, value in sorted(got.items())}
+
+ def _chip(row):
+ hit = _re.search(r"<code>confidence</code> <b>([0-9.]+)</b> (.*?) <svg", row)
+ return (hit.group(1), hit.group(2)) if hit else (None, None)
+
+ kinds = _decoded_kinds()
+ label = demo._CERTAINTY_LABEL
+ tip = demo._CERTAINTY_TIP
+ ok("the chip has a label for every type the page draws rows for, and no other",
+ sorted(label) == kinds, "%r vs the page's %r" % (sorted(label), kinds))
+
+ page_formula = {kind: ("entropy" if "entropy" in text else "maxp") for kind, text in sorted(label.items())}
+ for relpath in ("laya/agent.py", "laya/onnx_agent.py"):
+ decoders = _decoder_formulas(relpath)
+ ok("%s writes `confidence` the way this page labels it" % relpath,
+ decoders == page_formula, "%r vs the chip's %r" % (decoders, page_formula))
+
+ said = _sentence_types(tip, set(kinds))
+ ok("the tooltip attributes entropy to exactly the types whose chip says entropy",
+ said.get("entropy") == sorted(k for k, f in page_formula.items() if f == "entropy"),
+ "tooltip %r vs chip %r" % (said, page_formula))
+ ok("and the reported side's max(p_true, 1 - p_true) to exactly the types whose chip does not",
+ said.get("maxp") == sorted(k for k, f in page_formula.items() if f == "maxp"),
+ "tooltip %r vs chip %r" % (said, page_formula))
+ ok("so the tooltip names both formulas rather than defining the field as one",
+ len(said) == 2 and set(said) == {"entropy", "maxp"}, repr(said))
+
+ OLD_BLANKET = "The `confidence` field is 1 minus the normalized entropy of the whole distribution:"
+ ok("the blanket sentence this replaced is gone", OLD_BLANKET not in srv, OLD_BLANKET)
+ ok("and the page no longer promises an accuracy rate for a confidence value",
+ "are right about 90% of the time" not in srv
+ and "gate on a threshold you have measured" in tip, tip)
+
+ OLD_JS_CHIP = 'toFixed(4)}), "entropy, not calibrated",\n icon("info")'
+ ok("the browser's chip does not type a formula beside itself either",
+ OLD_JS_CHIP not in srv and "CERTAINTY_LABEL[kind]" in srv)
+ page = client.get("/").text
+ ok("the browser gets the label table from this dict, not a copy of it",
+ "const CERTAINTY_LABEL = " + json.dumps(label) in page, json.dumps(label))
+ ok("and the tooltip it shows is this same text",
+ "is not one formula, and it is not calibrated, so do not gate on it" in page)
+
+ import numpy as np
+ from laya.common import confidence_from_probs
+
+ spread = [0.62, 0.25, 0.13]
+ ent = round(confidence_from_probs(np.asarray(spread), 3), 4)
+ row = demo._answer_row("q", {"type": "choice", "choice": "billing", "answer_confidence": 0.62,
+ "confidence": ent, "probabilities": {"billing": 0.62, "support": 0.25,
+ "sales": 0.13}},
+ {"type": "choice", "instructions": "?"}, 1)
+ ok("a choice row reads as the entropy it is", _chip(row) == ("%.4f" % ent, label["choice"]),
+ "%r" % (_chip(row),))
+ row = demo._answer_row("q", {"type": "score", "score": 1.2, "answer_confidence": 0.5,
+ "confidence": round(confidence_from_probs(np.asarray([0.3, 0.5, 0.2]), 3), 4),
+ "legend": {"0": "low", "1": "mid", "2": "high"},
+ "probabilities": {"0": 0.3, "1": 0.5, "2": 0.2}},
+ {"type": "score", "instructions": "?"}, 1)
+ ok("so does a score row", _chip(row) == ("%.4f" % round(
+ confidence_from_probs(np.asarray([0.3, 0.5, 0.2]), 3), 4), label["score"]), "%r" % (_chip(row),))
+
+ p_true = 0.8727
+ side = round(max(p_true, 1 - p_true), 4)
+ two_option_entropy = round(confidence_from_probs(np.asarray([p_true, 1 - p_true]), 2), 4)
+ row = demo._answer_row("q", {"type": "noul", "noul": p_true, "answer_confidence": 0.79,
+ "confidence": side}, {"type": "noul", "instructions": "?"}, 1)
+ ok("a noul row's confidence is the side being reported, not an entropy",
+ side == p_true and abs(side - two_option_entropy) > 0.4
+ and _chip(row) == ("%.4f" % side, label["noul"])
+ and "entropy" not in _chip(row)[1], "%r %r" % (_chip(row), two_option_entropy))
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)