Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
docs(shortlist): the ordering contract must be the signed cosine the code ranks by
Aashishc291eed4cb02
diff --git a/laya/shortlist.py b/laya/shortlist.pyindex 777400cf..17a0b3b2 100644--- a/laya/shortlist.py+++ b/laya/shortlist.py@@ -44,11 +44,13 @@ def shortlist_choice(When ``k`` is at least the number of labels, every label is returned in itsoriginal order and ``embed_fn`` is not called.- Ties keep the earlier label. A zero vector scores 0 and does not outrank a- label that came before it.+ Ties keep the earlier label. Ranking is a signed cosine, not a similarity floor:+ a label that scores 0 -- no signal at all, or a non-finite vector treated as one+ -- does outrank an earlier label that scored negative, and ``k`` drops the+ negative labels first.With ``return_scores=True`` the return is the ``(labels, scores)`` pair, where- ``scores`` holds the cosine similarity per kept label in rank order -- the same+ ``scores`` holds the signed cosine per kept label in rank order -- the samevalues ``predict_shortlist`` reports in its ``shortlist`` metadata. ``scores``is ``None`` when nothing was dropped, exactly as in that metadata."""@@ -74,8 +76,11 @@ def predict_shortlist(The returned dict is the model result plus a ``shortlist`` entry. Probabilitieson a shortlisted choice are over the kept labels only. ``shortlist[qid]`` holds- ``labels`` (rank order), ``scores`` (cosine, or ``None`` when nothing was- dropped), ``k``, ``n``, and ``passthrough``.+ ``labels``, ``scores``, ``k``, ``n``, and ``passthrough``. ``labels`` is the rank+ order a shortlist produced, or the criteria order itself when ``passthrough`` is+ set and no ranking ran; ``scores`` is the signed cosine of each kept label in that+ order -- negative included, never clamped to 0 -- or ``None`` when nothing was+ dropped.Extra keyword arguments are forwarded to ``predict`` / ``system_one`` (forexample ``model=`` on a ``Router``).diff --git a/tests/test_shortlist_cosine.py b/tests/test_shortlist_cosine.pyindex 34c3fb93..022c1603 100644--- a/tests/test_shortlist_cosine.py+++ b/tests/test_shortlist_cosine.py@@ -1,6 +1,8 @@+import inspect+import reimport unittestimport numpy as np-from laya.shortlist import _cosine+from laya.shortlist import _cosine, predict_shortlist, shortlist_choiceclass ShortlistCosineTests(unittest.TestCase):def test_cosine_identical_vectors(self):@@ -22,5 +24,149 @@ class ShortlistCosineTests(unittest.TestCase):sims = _cosine(v, docs)self.assertEqual(len(sims), 0)++# A query of [1, 0] against six option vectors, so the whole signed range is in play:+# pos is cosine 1, orth scores 0 by being perpendicular, zero by having no signal, nan+# by being non-finite, n2 is -0.707 and n1 is -1.+VECTORS = {+ "q": [1.0, 0.0],+ "pos": [1.0, 0.0],+ "orth": [0.0, 1.0],+ "zero": [0.0, 0.0],+ "nan": [float("nan"), float("nan")],+ "n2": [-1.0, 1.0],+ "n1": [-1.0, 0.0],+}+++class Embed:+ """A stand-in embedder: records every call and answers from a vector table."""++ def __init__(self, vectors=VECTORS):+ self.vectors = vectors+ self.calls = []++ def __call__(self, texts):+ self.calls.append(list(texts))+ return [self.vectors[text] for text in texts]+++class FakeAgent:+ def __init__(self):+ self.questions = None++ def predict(self, state, questions, **kwargs):+ self.questions = questions+ return {"model": "fake", "answers": {}}+++def choice(labels):+ return {"i": {"type": "choice", "criteria": dict.fromkeys(labels)}}+++def ranked(labels, k, vectors=VECTORS):+ # criteria values of None make render_options send the bare label as the option text+ return shortlist_choice("q", dict.fromkeys(labels), Embed(vectors), k)+++class ShortlistOrderingTests(unittest.TestCase):+ """What a signed cosine does to top-``k``. Fabricated vectors only: no checkpoint, no Hub.++ ``_cosine`` returns negative similarities, so the ordering the shortlist docstrings+ promise has to be read across the whole ``[-1, 1]`` range, not off 0 alone.+ """++ def test_a_zero_score_outranks_an_earlier_negative(self):+ # n1 is declared first and scores -1; zero and orth both score 0 and pass it.+ self.assertEqual(ranked(["n1", "zero", "orth"], 2), ["zero", "orth"])+ self.assertEqual(ranked(["n1", "pos", "zero", "orth"], 3), ["pos", "zero", "orth"])++ def test_equal_scores_keep_criteria_order(self):+ self.assertEqual(ranked(["zero", "orth", "n1"], 2), ["zero", "orth"])+ self.assertEqual(ranked(["orth", "zero", "n1"], 2), ["orth", "zero"])++ def test_a_non_finite_vector_scores_zero_not_last(self):+ self.assertEqual(ranked(["nan", "orth", "n1"], 2), ["nan", "orth"])++ def test_a_zero_query_keeps_criteria_order(self):+ vectors = dict(VECTORS)+ vectors["q"] = [0.0, 0.0]+ self.assertEqual(ranked(["n1", "pos", "zero", "orth"], 2, vectors), ["n1", "pos"])++ def test_scores_are_signed_not_clamped(self):+ agent = FakeAgent()+ result = predict_shortlist(agent, "q", choice(["pos", "n1", "n2"]), Embed(), k=2)+ meta = result["shortlist"]["i"]+ self.assertEqual(meta["labels"], ["pos", "n2"])+ self.assertEqual(meta["scores"], [1.0, -0.7071067811865475])+ self.assertEqual(list(agent.questions["i"]["criteria"]), ["pos", "n2"])++ def test_return_scores_reports_the_same_pair(self):+ self.assertEqual(shortlist_choice("q", dict.fromkeys(["n1", "pos", "zero"]), Embed(), 2,+ return_scores=True),+ (["pos", "zero"], [1.0, 0.0]))+ self.assertEqual(shortlist_choice("q", dict.fromkeys(["n1", "pos", "zero"]), Embed(), 3,+ return_scores=True),+ (["n1", "pos", "zero"], None))++ def test_passthrough_labels_are_criteria_order(self):+ agent, embed = FakeAgent(), Embed()+ meta = predict_shortlist(agent, "q", choice(["n1", "pos", "zero"]), embed, k=3)["shortlist"]["i"]+ # pos would win any ranking and n1 would lose one, so this is only the declared+ # order: nothing was ranked, and embed_fn never ran.+ self.assertEqual(meta["labels"], ["n1", "pos", "zero"])+ self.assertIsNone(meta["scores"])+ self.assertTrue(meta["passthrough"])+ self.assertEqual(embed.calls, [])+ self.assertEqual(list(agent.questions["i"]["criteria"]), ["n1", "pos", "zero"])+++class ShortlistOrderingDocTests(unittest.TestCase):+ """Both docstrings render onto docs/reference/helpers.md, so they are the published+ ordering contract. Each rule is witnessed against main's own wording, kept verbatim+ below, so a rule the old text already satisfied could not be called a fix.+ """++ OLD_TIE = ("Ties keep the earlier label. A zero vector scores 0 and does not outrank a\n"+ "label that came before it.")+ OLD_META = ("``shortlist[qid]`` holds\n"+ "``labels`` (rank order), ``scores`` (cosine, or ``None`` when nothing was\n"+ "dropped), ``k``, ``n``, and ``passthrough``.")++ DOES_NOT_OUTRANK = re.compile(r"does not outrank", re.I)++ def paragraphs(self, fn):+ doc = inspect.getdoc(fn) or ""+ return [" ".join(part.split()) for part in re.split(r"\n\s*\n", doc) if part.strip()]++ def test_the_tie_paragraph_names_the_negative_side(self):+ doc = inspect.getdoc(shortlist_choice) or ""+ self.assertNotRegex(doc, self.DOES_NOT_OUTRANK)+ about = [p for p in self.paragraphs(shortlist_choice) if "outrank" in p]+ self.assertTrue(about, "nothing says what a score of 0 does against a negative one")+ self.assertIn("ties keep the earlier label", about[0].lower())+ self.assertIn("negative", about[0].lower())+ # main's sentence asserts the opposite of the first rule and none of the second.+ self.assertRegex(self.OLD_TIE, self.DOES_NOT_OUTRANK)+ self.assertNotIn("negative", self.OLD_TIE.lower())++ def test_the_metadata_paragraph_names_both_label_orders(self):+ about = [p for p in self.paragraphs(predict_shortlist) if "rank order" in p]+ self.assertTrue(about, "the labels/scores contract has left the docstring")+ self.assertIn("criteria order", about[0].lower())+ self.assertIn("passthrough", about[0])+ self.assertIn("signed cosine", about[0].lower())+ self.assertNotIn("criteria order", self.OLD_META.lower())+ self.assertNotIn("signed", self.OLD_META.lower())++ def test_a_negative_score_really_survives_the_ranking(self):+ # the metadata paragraph promises scores can be negative; prove the pair it+ # describes is what a signed cosine actually returns for a label pointing away.+ self.assertIn("never clamped to 0", " ".join((inspect.getdoc(predict_shortlist) or "").split()))+ scores = predict_shortlist(FakeAgent(), "q", choice(["pos", "n1", "n2"]), Embed(),+ k=2)["shortlist"]["i"]["scores"]+ self.assertTrue(any(score < 0.0 for score in scores), scores)++if __name__ == "__main__":unittest.main()