Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(examples): example 28 must compute the conclusions it prints

Aashishe01b06c6b5b3
diff --git a/examples/28_presets_moderation.py b/examples/28_presets_moderation.py
index 7bb584ec..31d59ddc 100644
--- a/examples/28_presets_moderation.py
+++ b/examples/28_presets_moderation.py
@@ -1,7 +1,9 @@
"""Example 28 -- moderation preset across four kinds of post.
Runs `laya.moderation_questions()` over a targeted insult, a generic insult, spam and a
-benign message, so each category's signal pattern is visible.
+benign message, so each category's signal pattern is visible. The summary at the bottom is
+computed from the probabilities this run measured, because a conclusion typed into a print
+statement describes one model's output and silently goes stale on the next.
"""
from _common import laya, banner, describe, heading, load
@@ -9,9 +11,16 @@ from _common import laya, banner, describe, heading, load
banner("28", "Preset: content moderation", """
`moderation_questions()` asks the four safety questions a moderation queue cares about
-- toxic, harassment, threat, spam -- plus an ordinal severity rubric, all in one
- forward pass. The point of the example is the shape of each pattern: a targeted insult
- lights up harassment, a generic one only toxic, spam only spam, and a normal post
- nothing at all.
+ forward pass. The point of the example is the shape of each pattern: which questions a
+ post carries past 0.5. A targeted insult lights up harassment as well as toxic, a
+ generic one lights up toxic alone, spam lights up spam alone, and an ordinary post
+ lights up nothing.
+
+ Nothing here means nothing past 0.5, not zero: an ordinary post still reads a little
+ on `spam`, and which question crosses the line is a property of this checkpoint on this
+ device. So every name, gap and ranking below is read off the probabilities printed above
+ instead of being hardcoded -- `tests/test_confidence.py` runs the functions on fabricated
+ answers and fails if a conclusion stops following its data.
""")
POSTS = [
@@ -25,42 +34,86 @@ POSTS = [
"Happy to share the config."),
]
+
+# --- the page's conclusions, as pure functions over answers so a test can drive them ---------
+def noul_flags(questions):
+ """The preset's flag questions, in declaration order -- derived, so a fifth flag shows up."""
+ return [k for k, q in questions.items() if q["type"] == "noul"]
+
+
+def score_ceiling(questions):
+ """The top of the ordinal rubric: the last criterion's index, not a typed-in 3."""
+ return max(len(q["criteria"]) - 1 for q in questions.values() if q["type"] == "score")
+
+
+def flag_line(answers, flags):
+ """Every flag with its probability. A line that speaks about all of them prints all of them."""
+ return " ".join("`%s` %.3f" % (k, answers[k]["noul"]) for k in flags)
+
+
+def past_half(answers, flags):
+ """The flags this answer reads as true -- the same 0.5 cut `_common.describe` labels."""
+ return [k for k in flags if answers[k]["noul"] > 0.5]
+
+
+def gaps(a, b, flags):
+ """Per-flag difference between two answers, widest first; `+` means `a` reads higher."""
+ g = [(k, a[k]["noul"] - b[k]["noul"]) for k in flags]
+ g.sort(key=lambda kv: -abs(kv[1]))
+ return g
+
+
+def rank_by_severity(results, labels):
+ """Rank the posts on `severity`, widest score first.
+
+ Returns (chain, in_listed_order). The chain's `>` signs are only ever printed between
+ scores that really descend, and the verdict compares the ranking to the order the posts
+ are listed in -- so neither can be asserted by the format string that prints it.
+ """
+ ranked = sorted(labels, key=lambda l: -results[l]["severity"]["score"])
+ chain = " > ".join("%s %.2f" % (l, results[l]["severity"]["score"]) for l in ranked)
+ return chain, ranked == list(labels)
+
+
agent = load("english")
+QUESTIONS = laya.moderation_questions()
+FLAGS = noul_flags(QUESTIONS)
+CEILING = score_ceiling(QUESTIONS)
+
results = {}
for label, post in POSTS:
heading(label)
- answers = agent.predict({"post": post}, laya.moderation_questions())["answers"]
+ answers = agent.predict({"post": post}, QUESTIONS)["answers"]
results[label] = answers
describe(answers)
heading("side by side")
-print(" %-16s %-7s %-11s %-7s %-7s %s" %
- ("post", "toxic", "harassment", "threat", "spam", "severity"))
+print(" %-16s %s severity" % ("post", " ".join("%-12s" % k for k in FLAGS)))
for label, _ in POSTS:
a = results[label]
- print(" %-16s %-7.3f %-11.3f %-7.3f %-7.3f %.2f / 3" % (
- label, a["toxic"]["noul"], a["harassment"]["noul"], a["threat"]["noul"],
- a["spam"]["noul"], a["severity"]["score"]))
+ print(" %-16s %s %.2f / %d"
+ % (label, " ".join("%-12.3f" % a[k]["noul"] for k in FLAGS),
+ a["severity"]["score"], CEILING))
heading("what the pattern means")
-tgt = results["targeted insult"]
-gen = results["generic insult"]
-spam = results["spam"]
-ben = results["benign"]
-print(" targeted insult -> toxic %.3f, harassment %.3f, spam %.3f (a person is the target)"
- % (tgt["toxic"]["noul"], tgt["harassment"]["noul"], tgt["spam"]["noul"]))
-print(" generic insult -> toxic %.3f, harassment %.3f (rude, not aimed at anyone)"
- % (gen["toxic"]["noul"], gen["harassment"]["noul"]))
-print(" spam -> spam %.3f, toxic %.3f (advertising, not abuse)"
- % (spam["spam"]["noul"], spam["toxic"]["noul"]))
-print(" benign -> every flag %.3f, severity %.2f / 3"
- % (ben["toxic"]["noul"], ben["severity"]["score"]))
-print()
-print(" `severity` orders the set correctly (%.2f > %.2f > %.2f > %.2f) but is a coarse 0-3"
- % (tgt["severity"]["score"], gen["severity"]["score"],
- spam["severity"]["score"], ben["severity"]["score"]))
-print(" rubric, so use it to sort a queue rather than as a hard threshold. The two hostile")
-print(" posts differ by only %.3f on `toxic`: the questions that separate them are"
- % abs(tgt["toxic"]["noul"] - gen["toxic"]["noul"]))
-print(" `harassment` and `threat`, not `toxic` on its own.")
+for label, _ in POSTS:
+ a = results[label]
+ hits = past_half(a, FLAGS)
+ print(" %-16s -> %s" % (label, flag_line(a, FLAGS)))
+ print(" %-16s past 0.5: %s"
+ % ("", ", ".join("`%s`" % k for k in hits) or "nothing"))
+
+tgt, gen = results["targeted insult"], results["generic insult"]
+g = gaps(tgt, gen, FLAGS)
+print("\n the two hostile posts, per-flag gap (widest first, `+` = the targeted one reads higher):")
+print(" %s" % ", ".join("`%s` %+.3f" % (k, v) for k, v in g))
+print(" the question that separates them most is `%s` (%.3f) and the least is `%s` (%.3f),"
+ % (g[0][0], abs(g[0][1]), g[-1][0], abs(g[-1][1])))
+print(" so the least of them cannot tell a targeted insult from a generic one on its own.")
+
+chain, in_order = rank_by_severity(results, [label for label, _ in POSTS])
+print("\n `severity` ranks the set: %s" % chain)
+print(" that is%s the order the posts are listed in. Whatever the order, `severity` is a coarse"
+ % ("" if in_order else " not"))
+print(" 0-%d rubric, so use it to sort a queue rather than as a hard threshold." % CEILING)
diff --git a/tests/test_confidence.py b/tests/test_confidence.py
index d4c210bf..2ef047d1 100644
--- a/tests/test_confidence.py
+++ b/tests/test_confidence.py
@@ -350,6 +350,211 @@ check("export/GATE_STATES is the vocabulary a caller iterates", list(GATE_STATES
[_confidence.GATE_PASSED, _confidence.GATE_ABSTAINED, _confidence.GATE_UNEVALUATED])
check("export/laya re-exports the same tuple", laya.GATE_STATES, _confidence.GATE_STATES)
+# ------------------------------------------------- a preset page's conclusions follow their numbers
+# `examples/28_presets_moderation.py` prints a summary over `laya.moderation_questions()` answers,
+# and the version this section replaces hardcoded three of its conclusions:
+# * `benign -> every flag 0.000, severity 0.24 / 3` printed `toxic` alone, while the same page's
+# table showed the benign post reading 0.066 on `spam`;
+# * ``severity orders the set correctly (1.34 > 1.05 > 0.99 > 0.24)`` put the `>` signs in the
+# format string, so the sentence stayed "correct" whatever the scores came back as;
+# * `the questions that separate them are harassment and threat` named two of four flags, while
+# the gaps measured on that same run were harassment 0.518, spam 0.193, threat 0.054 -- threat
+# was third, and spam was never mentioned.
+# The page now builds those sentences from pure functions over its answers. No weights are loaded:
+# the functions are extracted from the example's own AST and exec'd here on fabricated answers, so
+# the gate drives the code the page actually runs rather than re-reading its prose.
+import ast # noqa: E402
+import builtins # noqa: E402
+import re # noqa: E402
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+EXAMPLE_28 = os.path.join(ROOT, "examples", "28_presets_moderation.py")
+HELPERS_28 = ("noul_flags", "score_ceiling", "flag_line", "past_half", "gaps", "rank_by_severity")
+
+# The page as it ships on main: the banner's universality claim and the three conclusions, as
+# literal source lines. Every ban below is witnessed against this text, and every positive rule
+# below is witnessed by showing this text does not satisfy it.
+OLD_PAGE = '''
+ lights up harassment, a generic one only toxic, spam only spam, and a normal post
+ nothing at all.
+ label, a["toxic"]["noul"], a["harassment"]["noul"], a["threat"]["noul"],
+print(" benign -> every flag %.3f, severity %.2f / 3"
+ % (ben["toxic"]["noul"], ben["severity"]["score"]))
+print(" `severity` orders the set correctly (%.2f > %.2f > %.2f > %.2f) but is a coarse 0-3"
+print(" posts differ by only %.3f on `toxic`: the questions that separate them are")
+print(" `harassment` and `threat`, not `toxic` on its own.")
+'''
+
+UNIVERSAL_ONE_FLOAT = re.compile(r"every (?:flag|score|level)[^.]{0,12}%\.\d+f", re.I)
+ASSERTED_CHAIN = re.compile(r"%\.2f\s*>\s*%\.2f")
+NAMED_SEPARATORS = re.compile(r"separat\w+[^.]{0,60}`(?:toxic|harassment|threat|spam)`"
+ r"[^.]{0,30}`(?:toxic|harassment|threat|spam)`", re.I)
+HARDCODED_FLAG_READ = re.compile(r"\[[\"'](?:toxic|harassment|threat|spam)[\"']\]\[[\"']noul[\"']\]")
+NOTHING_QUALIFIED = re.compile(r"nothing past 0\.5[^,]{0,10}, not zero", re.I)
+DERIVED_FLAGS = re.compile(r"q\[.type.\]\s*==\s*[\"']noul[\"']")
+DERIVED_CEILING = re.compile(r"len\(q\[.criteria.\]\)\s*-\s*1")
+
+
+def _src28():
+ with open(EXAMPLE_28, encoding="utf-8") as fh:
+ return fh.read()
+
+
+def _helpers28():
+ """Exec only the example's top-level helper functions -- its `load()` call needs weights."""
+ tree = ast.parse(_src28())
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_28]
+ ns = {}
+ exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_28, "exec"), ns) # noqa: S102
+ return ns
+
+
+def _called28():
+ """Names the example's top-level *statements* use, so a helper cannot pass by being dead."""
+ tree = ast.parse(_src28())
+ top = [n for n in tree.body if not isinstance(
+ n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))]
+ return {node.id for stmt in top for node in ast.walk(stmt)
+ if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}
+
+
+def _a(toxic, harassment, threat, spam, severity):
+ return {"toxic": {"noul": toxic}, "harassment": {"noul": harassment},
+ "threat": {"noul": threat}, "spam": {"noul": spam},
+ "severity": {"score": severity}}
+
+
+def test_preset_is_four_flags_and_one_rubric():
+ """The docstring and the example's banner both claim this shape; nothing asserted it."""
+ mq = laya.moderation_questions()
+ assert list(mq) == ["toxic", "harassment", "threat", "spam", "severity"], list(mq)
+ assert {k: q["type"] for k, q in mq.items()} == {
+ "toxic": "noul", "harassment": "noul", "threat": "noul", "spam": "noul",
+ "severity": "score"}
+ assert len(mq["severity"]["criteria"]) == 4, len(mq["severity"]["criteria"])
+
+
+def _free_names(node):
+ """Names a function reads that it neither binds locally nor takes as an argument."""
+ bound = {a.arg for fn in [node] + [k for k in ast.walk(node) if isinstance(k, ast.Lambda)]
+ for a in fn.args.args}
+ bound |= {n.id for n in ast.walk(node)
+ if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store)}
+ return {n.id for n in ast.walk(node)
+ if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Load) and n.id not in bound}
+
+
+def test_helpers_are_live_and_pure():
+ """Each helper is called by the page, takes its whole world as arguments, and returns."""
+ ns = _helpers28()
+ got = sorted(k for k in ns if not k.startswith("__"))
+ assert got == sorted(HELPERS_28), "example 28 lost a helper: %s" % got
+ unused = set(HELPERS_28) - _called28()
+ assert not unused, "defined but never called at module level: %s" % sorted(unused)
+ for name in HELPERS_28:
+ node = _fn_node(name)
+ outside = sorted(n for n in _free_names(node) if not hasattr(builtins, n))
+ assert not outside, "%s reads %s from the page, so this gate cannot drive it" % (name, outside)
+ prints = [c for c in ast.walk(node)
+ if isinstance(c, ast.Call) and isinstance(c.func, ast.Name) and c.func.id == "print"]
+ assert not prints, "%s prints instead of returning, so the summary is not checkable" % name
+
+
+def _fn_node(name):
+ for n in ast.parse(_src28()).body:
+ if isinstance(n, ast.FunctionDef) and n.name == name:
+ return n
+ raise KeyError(name)
+
+
+def test_flag_line_prints_every_flag():
+ ns = _helpers28()
+ flags = ns["noul_flags"](laya.moderation_questions())
+ assert flags == ["toxic", "harassment", "threat", "spam"], flags
+ line = ns["flag_line"](_a(0.0, 0.0, 0.0, 0.066, 0.24), flags)
+ assert "0.066" in line, "the non-zero flag vanished: %r" % line
+ assert re.findall(r"`([a-z]+)`", line) == flags, line
+ assert len(re.findall(r"\d\.\d{3}", line)) == 4, "one number per flag: %r" % line
+
+
+def test_past_half_uses_describes_cut():
+ """`_common.describe` labels a noul true at `> 0.5`; the page must agree with it exactly."""
+ ns = _helpers28()
+ flags = ns["noul_flags"](laya.moderation_questions())
+ assert ns["past_half"](_a(0.0, 0.0, 0.0, 0.066, 0.24), flags) == []
+ assert ns["past_half"](_a(0.587, 0.662, 0.142, 0.049, 1.34), flags) == ["toxic", "harassment"]
+ assert ns["past_half"](_a(0.5, 0.5, 0.5, 0.5, 1.0), flags) == [], "0.5 reads false, not true"
+
+
+def test_gaps_names_the_widest_separators():
+ """The old page said harassment and threat; on these numbers it is harassment and spam."""
+ ns = _helpers28()
+ flags = ns["noul_flags"](laya.moderation_questions())
+ tgt, gen = _a(0.587, 0.662, 0.142, 0.049, 1.34), _a(0.584, 0.144, 0.088, 0.242, 1.05)
+ g = ns["gaps"](tgt, gen, flags)
+ assert [k for k, _ in g] == ["harassment", "spam", "threat", "toxic"], g
+ assert abs(dict(g)["harassment"] - 0.518) < 1e-9 and dict(g)["spam"] < 0, g
+ # and the name follows the data: move `threat` to the top and it is the separator.
+ assert ns["gaps"](_a(0.1, 0.1, 0.9, 0.1, 1), _a(0.1, 0.1, 0.1, 0.1, 1), flags)[0][0] == "threat"
+
+
+def test_rank_cannot_assert_its_own_order():
+ ns = _helpers28()
+ labels = ["a", "b", "c", "d"]
+ scores = {"a": 1.34, "b": 1.05, "c": 0.99, "d": 0.24}
+ results = {k: _a(0, 0, 0, 0, v) for k, v in scores.items()}
+ chain, in_order = ns["rank_by_severity"](results, labels)
+ assert in_order is True, chain
+ assert [float(x) for x in re.findall(r"\d+\.\d{2}", chain)] == [1.34, 1.05, 0.99, 0.24], chain
+
+ shuffled = {k: _a(0, 0, 0, 0, scores[j]) for k, j in
+ zip(labels, ["d", "a", "c", "b"])}
+ chain2, in_order2 = ns["rank_by_severity"](shuffled, labels)
+ assert in_order2 is False, "the verdict must flip when the ranking does: %s" % chain2
+ assert [float(x) for x in re.findall(r"\d+\.\d{2}", chain2)] == [1.34, 1.05, 0.99, 0.24], (
+ "the printed chain must descend, whatever the order it lists: %s" % chain2)
+ assert chain2.index("b") < chain2.index("a"), chain2
+
+
+def test_page_drops_the_hardcoded_conclusions():
+ """Each ban fires on main's page and not on this one -- a ban with no witness is a guess."""
+ src = _src28()
+ for name, rule in (("every flag %.3f", UNIVERSAL_ONE_FLOAT),
+ ("%.2f > %.2f in a format string", ASSERTED_CHAIN),
+ ("separate them are `harassment` and `threat`", NAMED_SEPARATORS),
+ ('a["toxic"]["noul"] in the summary', HARDCODED_FLAG_READ)):
+ assert rule.search(OLD_PAGE), "%s does not fire on the wording it bans" % name
+ assert not rule.search(src), "%s is still in example 28" % name
+
+
+def test_page_qualifies_the_banner_and_derives_its_numbers():
+ """The positive half: the page must read its flag list and rubric size from the preset."""
+ src = _src28()
+ assert NOTHING_QUALIFIED.search(src), "the banner must say that `nothing` is not zero"
+ assert not re.search(r"nothing at all", src, re.I), "the unqualified claim is back"
+ assert DERIVED_FLAGS.search(src), "flags must be read off the preset, not typed"
+ assert DERIVED_CEILING.search(src), "the rubric ceiling must be read off the criteria"
+ assert not re.search(r"/ 3\b", src), "a hardcoded severity divisor is back"
+ # and main's page satisfies none of that, so these rules could not have passed before.
+ assert not NOTHING_QUALIFIED.search(OLD_PAGE)
+ assert re.search(r"nothing at all", OLD_PAGE, re.I)
+ assert not DERIVED_FLAGS.search(OLD_PAGE) and not DERIVED_CEILING.search(OLD_PAGE)
+
+
+for _fn in (test_preset_is_four_flags_and_one_rubric, test_helpers_are_live_and_pure,
+ test_flag_line_prints_every_flag, test_past_half_uses_describes_cut,
+ test_gaps_names_the_widest_separators, test_rank_cannot_assert_its_own_order,
+ test_page_drops_the_hardcoded_conclusions,
+ test_page_qualifies_the_banner_and_derives_its_numbers):
+ try:
+ _fn()
+ except AssertionError as e:
+ FAIL.append("page-28/%s: %s" % (_fn.__name__, e))
+ except Exception as e: # a crash is a failure, never a silent pass
+ FAIL.append("page-28/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
+ else:
+ PASS.append("page-28/%s" % _fn.__name__)
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)