Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(shortlist): return cosine scores from shortlist_choice on request

Aashishd096f3617fb5
diff --git a/README.md b/README.md
index 328a6194..9e259a6d 100644
--- a/README.md
+++ b/README.md
@@ -1312,7 +1312,7 @@ result = laya.predict_shortlist(
result["shortlist"]["intent"]["labels"] # the top 20 labels sent to the model
```
-`embed_fn(texts)` returns one vector per string. `embed_fn_from_agent` mean-pools the encoder already loaded on the agent; the decision head runs in the following `predict` / `system_one` call. Probabilities on a shortlisted choice are over those `k` labels. When `k` is at least the number of labels, the original question is passed through and `embed_fn` is not called.
+`embed_fn(texts)` returns one vector per string. `embed_fn_from_agent` mean-pools the encoder already loaded on the agent; the decision head runs in the following `predict` / `system_one` call. Probabilities on a shortlisted choice are over those `k` labels. When `k` is at least the number of labels, the original question is passed through and `embed_fn` is not called. `laya.shortlist_choice` returns bare labels; pass `return_scores=True` for the `(labels, scores)` pair in rank order (`None` when nothing was dropped), the same cosines `predict_shortlist` reports in its `shortlist` metadata.
Shortlisting the same option set on every request re-embeds option texts that do not change. Wrap the embedder once with `laya.cached_embed_fn(embed_fn)` and repeat calls embed only the new query text: lookups are exact string matches into an LRU of at most 4,096 entries (about `maxsize * dim * 4` bytes, so ~12 MB at the default with a 768-dim encoder), and texts missing from the cache are still embedded in one batched call. The wrapper's `cache_info()` reports hits and misses; call `cache_clear()` if the model behind `embed_fn` changes.
diff --git a/laya/shortlist.py b/laya/shortlist.py
index 105cf805..6650e9bd 100644
--- a/laya/shortlist.py
+++ b/laya/shortlist.py
@@ -33,7 +33,8 @@ def shortlist_choice(
k: int = DEFAULT_SHORTLIST_K,
*,
instructions: Optional[str] = None,
-) -> List[Any]:
+ return_scores: bool = False,
+) -> Any:
"""Return the top-``k`` choice labels for ``state``.
``embed_fn`` maps a list of strings to an array of shape ``(len(texts), dim)``.
@@ -45,8 +46,15 @@ def shortlist_choice(
Ties keep the earlier label. A zero vector scores 0 and does not outrank a
label that came before it.
+
+ With ``return_scores=True`` the return is the ``(labels, scores)`` pair, where
+ ``scores`` holds the cosine similarity per kept label in rank order -- the same
+ values ``predict_shortlist`` reports in its ``shortlist`` metadata. ``scores``
+ is ``None`` when nothing was dropped, exactly as in that metadata.
"""
- labels, _scores, _passthrough, _n = _rank(state, criteria, embed_fn, k, instructions)
+ labels, scores, _passthrough, _n = _rank(state, criteria, embed_fn, k, instructions)
+ if return_scores:
+ return labels, scores
return labels
diff --git a/tests/test_hooks_api.py b/tests/test_hooks_api.py
index ccd59572..bea92428 100644
--- a/tests/test_hooks_api.py
+++ b/tests/test_hooks_api.py
@@ -131,6 +131,11 @@ for param in ("max_len", "head_max_len"):
check("Router.predict_batch/%s is per-request, not a call argument" % param,
param in sig(Router.predict_batch), False)
+# shortlist_choice returns bare labels by default; return_scores=True also hands back the
+# rank-order cosines, so the flag is keyword-only and defaults to off
+check_param("shortlist_choice", laya.shortlist_choice, "return_scores", False,
+ inspect.Parameter.KEYWORD_ONLY)
+
# route() takes per-call hooks so a hook can pin a checkpoint for one call
check_param("Router.route", Router.route, "hooks", None)
check_param("Router.route", Router.route, "hooks_raise", None)
diff --git a/tests/test_shortlist.py b/tests/test_shortlist.py
index e52dc528..805dacc3 100644
--- a/tests/test_shortlist.py
+++ b/tests/test_shortlist.py
@@ -223,6 +223,28 @@ check("pass/k == n returns every label in order", shortlist_choice("pay me", CRI
check("pass/k > n returns every label in order", shortlist_choice("pay me", CRITERIA, boom, k=20), list(CRITERIA))
+# ---------------------------------------------------------------- return_scores
+score_embed = _embed_for("pay me", OPTION_TEXTS)
+pair = shortlist_choice("pay me", CRITERIA, score_embed, k=2, return_scores=True)
+check("scores/labels match the bare call", pair[0], ["alpha", "delta"])
+check("scores/cosines are in rank order", pair[1], [1.0, 1.0])
+trio = shortlist_choice("pay me", CRITERIA, _embed_for("pay me", OPTION_TEXTS), k=3, return_scores=True)
+check("scores/third label is gamma", trio[0], ["alpha", "delta", "gamma"])
+check("scores/third cosine rounds to gamma", [round(s, 9) for s in trio[1]], [1.0, 1.0, 0.6])
+bare = shortlist_choice("pay me", CRITERIA, _embed_for("pay me", OPTION_TEXTS), k=2)
+check("scores/default still returns bare labels", bare, ["alpha", "delta"])
+pass_pair = shortlist_choice("pay me", CRITERIA, BoomEmbed(), k=20, return_scores=True)
+check("scores/passthrough keeps every label", pass_pair[0], list(CRITERIA))
+check("scores/passthrough reports no scores", pass_pair[1], None)
+meta_agent = Recorder()
+meta_out = predict_shortlist(
+ meta_agent, "pay me", {"intent": {"type": "choice", "criteria": CRITERIA}},
+ _embed_for("pay me", OPTION_TEXTS), k=2,
+)
+check("scores/agree with predict_shortlist metadata",
+ pair[1], meta_out["shortlist"]["intent"]["scores"])
+
+
# ---------------------------------------------------------------- mock predict sees only k criteria
sentinel = {"desc": "payments"}
full = {"billing": sentinel, "tech": "bugs", "sales": None, "other": "misc"}