Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

9.6 KB
Raw Blame
import inspect
import re
import unittest
import numpy as np
from laya.shortlist import _cosine, predict_shortlist, shortlist_choice

class ShortlistCosineTests(unittest.TestCase):
    def test_cosine_identical_vectors(self):
        v = np.array([1.0, 2.0, 3.0], dtype=np.float64)
        docs = np.array([[1.0, 2.0, 3.0], [-1.0, -2.0, -3.0]], dtype=np.float64)
        sims = _cosine(v, docs)
        self.assertAlmostEqual(sims[0], 1.0, places=5)
        self.assertAlmostEqual(sims[1], -1.0, places=5)

    def test_cosine_zero_vectors(self):
        v = np.zeros(4, dtype=np.float64)
        docs = np.ones((3, 4), dtype=np.float64)
        sims = _cosine(v, docs)
        self.assertTrue(np.all(sims == 0.0))

    def test_cosine_empty_docs(self):
        v = np.ones(4, dtype=np.float64)
        docs = np.zeros((0, 4), dtype=np.float64)
        sims = _cosine(v, docs)
        self.assertEqual(len(sims), 0)


# A query of [1, 0] against seven option vectors, so the whole signed range is in play:
# pos is cosine 1, orth scores 0 by being perpendicular, zero by having no signal, nan
# and inf by being non-finite, n2 is -0.707 and n1 is -1.
VECTORS = {
    "q": [1.0, 0.0],
    "pos": [1.0, 0.0],
    "orth": [0.0, 1.0],
    "zero": [0.0, 0.0],
    "nan": [float("nan"), float("nan")],
    "inf": [float("inf"), 0.0],
    "n2": [-1.0, 1.0],
    "n1": [-1.0, 0.0],
}


class Embed:
    """A stand-in embedder: records every call and answers from a vector table."""

    def __init__(self, vectors=VECTORS):
        self.vectors = vectors
        self.calls = []

    def __call__(self, texts):
        self.calls.append(list(texts))
        return [self.vectors[text] for text in texts]


class FakeAgent:
    def __init__(self):
        self.questions = None

    def predict(self, state, questions, **kwargs):
        self.questions = questions
        return {"model": "fake", "answers": {}}


def choice(labels):
    return {"i": {"type": "choice", "criteria": dict.fromkeys(labels)}}


# Twenty labels is the size the module exists for, and long enough that a tie of ten
# gets visibly shuffled by a non-stable argsort.
MANY = ["L%02d" % i for i in range(20)]


def tied_table(query):
    """``VECTORS`` plus twenty labels: cosine 1.0 on the even indexes, 0.0 on the odd."""
    vectors = dict(VECTORS)
    vectors["q"] = list(query)
    for i, label in enumerate(MANY):
        vectors[label] = [1.0, 0.0] if i % 2 == 0 else [0.0, 1.0]
    return vectors


def ranked(labels, k, vectors=VECTORS):
    # criteria values of None make render_options send the bare label as the option text
    return shortlist_choice("q", dict.fromkeys(labels), Embed(vectors), k)


class ShortlistOrderingTests(unittest.TestCase):
    """What a signed cosine does to top-``k``. Fabricated vectors only: no checkpoint, no Hub.

    ``_cosine`` returns negative similarities, so the ordering the shortlist docstrings
    promise has to be read across the whole ``[-1, 1]`` range, not off 0 alone.
    """

    def test_a_zero_score_outranks_an_earlier_negative(self):
        # n1 is declared first and scores -1; zero and orth both score 0 and pass it.
        self.assertEqual(ranked(["n1", "zero", "orth"], 2), ["zero", "orth"])
        self.assertEqual(ranked(["n1", "pos", "zero", "orth"], 3), ["pos", "zero", "orth"])

    def test_equal_scores_keep_criteria_order(self):
        self.assertEqual(ranked(["zero", "orth", "n1"], 2), ["zero", "orth"])
        self.assertEqual(ranked(["orth", "zero", "n1"], 2), ["orth", "zero"])

    def test_a_non_finite_vector_scores_zero_not_last(self):
        self.assertEqual(ranked(["nan", "orth", "n1"], 2), ["nan", "orth"])

    def test_an_infinite_vector_is_zeroed_before_ranking(self):
        # A NaN row already reaches 0 through _cosine's own guard on a non-positive
        # denominator, so only an infinite row can arrive at the ranker as a NaN
        # similarity -- and a NaN sorts last, which would drop the label silently.
        self.assertEqual(ranked(["inf", "orth", "n1"], 2), ["inf", "orth"])

    def test_ties_come_back_in_criteria_order_not_in_argsort_order(self):
        # The docstrings' whole tie-break promise is that equal scores fall back to the
        # declared order. Ten labels tie at 1.0 and ten at 0.0, so the kept twelve are the
        # evens then L01 and L03 -- a non-stable sort returns them shuffled.
        self.assertEqual(ranked(MANY, 12, tied_table([1.0, 0.0])),
                         MANY[0::2] + ["L01", "L03"])

    def test_a_zero_query_keeps_criteria_order(self):
        # every cosine is 0, so all twenty labels tie and the top-k is the declared prefix
        self.assertEqual(ranked(MANY, 12, tied_table([0.0, 0.0])), MANY[:12])

    def test_scores_are_signed_not_clamped(self):
        agent = FakeAgent()
        result = predict_shortlist(agent, "q", choice(["pos", "n1", "n2"]), Embed(), k=2)
        meta = result["shortlist"]["i"]
        self.assertEqual(meta["labels"], ["pos", "n2"])
        self.assertEqual(meta["scores"], [1.0, -0.7071067811865475])
        self.assertEqual(list(agent.questions["i"]["criteria"]), ["pos", "n2"])

    def test_return_scores_reports_the_same_pair(self):
        self.assertEqual(shortlist_choice("q", dict.fromkeys(["n1", "pos", "zero"]), Embed(), 2,
                                          return_scores=True),
                         (["pos", "zero"], [1.0, 0.0]))
        self.assertEqual(shortlist_choice("q", dict.fromkeys(["n1", "pos", "zero"]), Embed(), 3,
                                          return_scores=True),
                         (["n1", "pos", "zero"], None))

    def test_passthrough_labels_are_criteria_order(self):
        agent, embed = FakeAgent(), Embed()
        result = predict_shortlist(agent, "q", choice(["n1", "pos", "zero"]), embed, k=3)
        meta = result["shortlist"]["i"]
        # pos would win any ranking and n1 would lose one, so this is only the declared
        # order: nothing was ranked, and embed_fn never ran.
        self.assertEqual(meta["labels"], ["n1", "pos", "zero"])
        self.assertIsNone(meta["scores"])
        self.assertTrue(meta["passthrough"])
        self.assertEqual(embed.calls, [])
        self.assertEqual(list(agent.questions["i"]["criteria"]), ["n1", "pos", "zero"])


class ShortlistOrderingDocTests(unittest.TestCase):
    """Both docstrings render onto docs/reference/helpers.md, so they are the published
    ordering contract. Each rule is witnessed against main's own wording, kept verbatim
    below, so a rule the old text already satisfied could not be called a fix. The rules
    take synonyms -- "declared order" for "criteria order", "points away" for "negative"
    -- so a rewording that keeps the meaning cannot fail them.
    """

    OLD_TIE = ("Ties keep the earlier label. A zero vector scores 0 and does not outrank a\n"
               "label that came before it.")
    OLD_META = ("``shortlist[qid]`` holds\n"
                "``labels`` (rank order), ``scores`` (cosine, or ``None`` when nothing was\n"
                "dropped), ``k``, ``n``, and ``passthrough``.")

    DOES_NOT_OUTRANK = re.compile(r"does not outrank", re.I)
    NAMES_NEGATIVE_SIDE = re.compile(r"negative|below 0|points away", re.I)
    # "earlier label" alone would not do: the counter-case clause says it too, so the
    # rule asks for the tie itself to be named.
    NAMES_THE_TIE_RULE = re.compile(r"\bties\b|equal scores|same score", re.I)
    RANK_ORDER = re.compile(r"rank(?:ing)? order", re.I)
    UNRANKED_PATH = re.compile(r"criteria order|declared order|input order", re.I)
    SIGNED_SCORES = re.compile(r"signed (?:cosine|similarity|score)", re.I)
    SIGN_KEPT = re.compile(r"(?:not|never) clamped|negative included", re.I)

    def paragraphs(self, fn):
        doc = inspect.getdoc(fn) or ""
        return [" ".join(part.split()) for part in re.split(r"\n\s*\n", doc) if part.strip()]

    def test_the_tie_paragraph_names_the_negative_side(self):
        doc = inspect.getdoc(shortlist_choice) or ""
        self.assertNotRegex(doc, self.DOES_NOT_OUTRANK)
        about = [p for p in self.paragraphs(shortlist_choice) if "outrank" in p]
        self.assertTrue(about, "nothing says what a score of 0 does against a negative one")
        self.assertRegex(about[0], self.NAMES_THE_TIE_RULE)
        self.assertRegex(about[0], self.NAMES_NEGATIVE_SIDE)
        # main's sentence asserts the opposite of the ban and satisfies neither rule.
        self.assertRegex(self.OLD_TIE, self.DOES_NOT_OUTRANK)
        self.assertNotRegex(self.OLD_TIE, self.NAMES_NEGATIVE_SIDE)

    def test_the_metadata_paragraph_names_both_label_orders(self):
        about = [p for p in self.paragraphs(predict_shortlist) if self.RANK_ORDER.search(p)]
        self.assertTrue(about, "the labels/scores contract has left the docstring")
        self.assertRegex(about[0], self.UNRANKED_PATH)
        self.assertIn("passthrough", about[0])
        self.assertRegex(about[0], self.SIGNED_SCORES)
        self.assertNotRegex(self.OLD_META, self.UNRANKED_PATH)
        self.assertNotRegex(self.OLD_META, self.SIGNED_SCORES)

    def test_a_negative_score_really_survives_the_ranking(self):
        # the metadata paragraph promises scores can be negative; prove the pair it
        # describes is what a signed cosine actually returns for a label pointing away.
        about = [p for p in self.paragraphs(predict_shortlist) if self.RANK_ORDER.search(p)]
        self.assertTrue(about and self.SIGN_KEPT.search(about[0]),
                        "the sign of scores is promised nowhere")
        scores = predict_shortlist(FakeAgent(), "q", choice(["pos", "n1", "n2"]), Embed(),
                                   k=2)["shortlist"]["i"]["scores"]
        self.assertTrue(any(score < 0.0 for score in scores), scores)


if __name__ == "__main__":
    unittest.main()