Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf: cache shortlist option embeddings across calls (cached_embed_fn)

Aashish79632ceaa5eb
diff --git a/README.md b/README.md
index 507e230e..10b51cfb 100644
--- a/README.md
+++ b/README.md
@@ -925,6 +925,8 @@ result["shortlist"]["intent"]["labels"] # the top 20 labels sent to the model
`embed_fn(texts)` returns one vector per string. `embed_fn_from_agent` mean-pools the encoder already loaded on the agent; the decision head runs in the following `predict` / `system_one` call. Probabilities on a shortlisted choice are over those `k` labels. When `k` is at least the number of labels, the original question is passed through and `embed_fn` is not called.
+Shortlisting the same option set on every request re-embeds option texts that do not change. Wrap the embedder once with `laya.cached_embed_fn(embed_fn)` and repeat calls embed only the new query text: lookups are exact string matches into an LRU of at most 4,096 entries (about `maxsize * dim * 4` bytes, so ~12 MB at the default with a 768-dim encoder), and texts missing from the cache are still embedded in one batched call. The wrapper's `cache_info()` reports hits and misses; call `cache_clear()` if the model behind `embed_fn` changes.
+
[Issue #102](https://github.com/NandhaKishorM/laya/issues/102) reports that a top-20 zero-shot shortlist moved a BANKING77 run from 54.3% to 60.8% on the reporter's setup. Those figures are the reporter's; this repository has not remeasured them.
* Ordinal `score` questions are the weakest primitive (SST-5 0.372).
diff --git a/laya/__init__.py b/laya/__init__.py
index 792d9089..fea74ab5 100644
--- a/laya/__init__.py
+++ b/laya/__init__.py
@@ -34,6 +34,7 @@ _LAZY_ATTRS = {
"shortlist_choice": (".shortlist", "shortlist_choice"),
"predict_shortlist": (".shortlist", "predict_shortlist"),
"embed_fn_from_agent": (".shortlist", "embed_fn_from_agent"),
+ "cached_embed_fn": (".shortlist", "cached_embed_fn"),
"LayaRouter": (".integrations", "LayaRouter"),
"LayaGuardrail": (".integrations", "LayaGuardrail"),
"LayaGuardrailError": (".integrations", "LayaGuardrailError"),
@@ -68,6 +69,7 @@ __all__ = [
"shortlist_choice",
"predict_shortlist",
"embed_fn_from_agent",
+ "cached_embed_fn",
"detect_language",
"detect_script",
"is_english",
diff --git a/laya/shortlist.py b/laya/shortlist.py
index d4606be5..602c60bc 100644
--- a/laya/shortlist.py
+++ b/laya/shortlist.py
@@ -15,6 +15,8 @@ whatever vectors ``embed_fn`` returns. Issue #102's BANKING77 figures belong to
report; this module does not measure them.
"""
import json
+import threading
+from collections import OrderedDict
from typing import Any, Callable, Dict, List, Optional, Sequence
import numpy as np
@@ -164,6 +166,99 @@ def embed_fn_from_agent(
return embed_fn
+def cached_embed_fn(
+ embed_fn: Callable[[Sequence[str]], Any],
+ maxsize: int = 4096,
+) -> Callable[[Sequence[str]], np.ndarray]:
+ """Cache ``embed_fn`` output per input string, under an LRU bound.
+
+ ``predict_shortlist`` embeds the query plus every option text on each call. When the
+ same option set is shortlisted on every request -- a fixed intent or label list, as
+ in the README's BANKING77 example -- the option rows do not change between calls,
+ yet they are re-embedded every time. Wrapping the embedder once::
+
+ embed_fn = cached_embed_fn(embed_fn_from_agent(agent))
+
+ leaves the first call unchanged and reduces each repeat call to embedding the new
+ query alone.
+
+ Lookups are exact string matches. Texts missing from the cache are deduplicated and
+ embedded in a single ``embed_fn`` call, so a cold cache costs the same number of
+ batched calls as the unwrapped function. Rows are stored as float32; the cache holds
+ at most ``maxsize`` strings and then evicts the least recently used entry, bounding
+ memory at about ``maxsize * dim * 4`` bytes. Nothing is cached when ``embed_fn``
+ raises or returns a bad shape.
+
+ The wrapper is safe to share between threads: the lock covers only cache reads and
+ writes, never the embedding call. The returned callable carries ``cache_info()`` --
+ a dict with ``size``, ``maxsize``, ``hits`` and ``misses`` -- and ``cache_clear()``.
+ Clear the cache if the model or weights behind ``embed_fn`` change.
+ """
+ if not callable(embed_fn):
+ raise TypeError("embed_fn must be callable")
+ if isinstance(maxsize, bool) or not isinstance(maxsize, int) or maxsize < 1:
+ raise ValueError("maxsize must be a positive integer, got %r" % (maxsize,))
+
+ rows_by_text: OrderedDict[str, np.ndarray] = OrderedDict()
+ lock = threading.Lock()
+ counts = {"hits": 0, "misses": 0}
+
+ def cached(texts: Sequence[str]) -> np.ndarray:
+ keys = ["" if text is None else str(text) for text in texts]
+ if not keys:
+ return np.zeros((0, 0), dtype=np.float32)
+ with lock:
+ found: Dict[str, np.ndarray] = {}
+ hits = 0
+ for key in keys:
+ row = rows_by_text.get(key)
+ if row is not None:
+ rows_by_text.move_to_end(key)
+ found[key] = row
+ hits += 1
+ counts["hits"] += hits
+ counts["misses"] += len(keys) - hits
+ missing = [key for key in dict.fromkeys(keys) if key not in found]
+ if missing:
+ raw = embed_fn(list(missing))
+ if hasattr(raw, "detach"):
+ raw = raw.detach().float().cpu().numpy()
+ fresh = np.asarray(raw, dtype=np.float32)
+ if fresh.ndim != 2 or fresh.shape[0] != len(missing) or fresh.shape[1] < 1:
+ raise ValueError(
+ "embed_fn must return an array of shape (%d, dim), got %s"
+ % (len(missing), tuple(fresh.shape))
+ )
+ fresh = np.nan_to_num(fresh, copy=True, nan=0.0, posinf=0.0, neginf=0.0)
+ with lock:
+ for key, row in zip(missing, fresh):
+ rows_by_text[key] = row
+ rows_by_text.move_to_end(key)
+ while len(rows_by_text) > maxsize:
+ rows_by_text.popitem(last=False)
+ found[key] = row
+ return np.stack([found[key] for key in keys])
+
+ def cache_info() -> Dict[str, int]:
+ with lock:
+ return {
+ "size": len(rows_by_text),
+ "maxsize": maxsize,
+ "hits": counts["hits"],
+ "misses": counts["misses"],
+ }
+
+ def cache_clear() -> None:
+ with lock:
+ rows_by_text.clear()
+ counts["hits"] = 0
+ counts["misses"] = 0
+
+ cached.cache_info = cache_info
+ cached.cache_clear = cache_clear
+ return cached
+
+
def _rank(state, criteria, embed_fn, k, instructions):
checked = _check_k(k)
items = _criteria_items(criteria)
diff --git a/tests/test_shortlist.py b/tests/test_shortlist.py
index 5cb17758..e52dc528 100644
--- a/tests/test_shortlist.py
+++ b/tests/test_shortlist.py
@@ -6,6 +6,7 @@ decision model is never constructed.
import inspect
import os
import sys
+from concurrent.futures import ThreadPoolExecutor
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -15,6 +16,7 @@ import laya # noqa: E402
from laya.agent import Agent # noqa: E402
from laya.common import DecisionModel, render_options # noqa: E402
from laya.shortlist import ( # noqa: E402
+ cached_embed_fn,
embed_fn_from_agent,
predict_shortlist,
shortlist_choice,
@@ -123,7 +125,8 @@ def _embed_for(query_text, option_vectors):
check_true("export/shortlist_choice", laya.shortlist_choice is shortlist_choice)
check_true("export/predict_shortlist", laya.predict_shortlist is predict_shortlist)
check_true("export/embed_fn_from_agent", laya.embed_fn_from_agent is embed_fn_from_agent)
-for _name in ("shortlist_choice", "predict_shortlist", "embed_fn_from_agent"):
+check_true("export/cached_embed_fn", laya.cached_embed_fn is cached_embed_fn)
+for _name in ("shortlist_choice", "predict_shortlist", "embed_fn_from_agent", "cached_embed_fn"):
check_true("all/%s" % _name, _name in laya.__all__)
_predict_src = inspect.getsource(Agent.system_one)
@@ -527,6 +530,198 @@ check("pipe/marker count equals k", len(scored["crit"]), 2)
check("pipe/answer choice is inside the shortlist", piped["answers"]["intent"]["choice"] in piped["shortlist"]["intent"]["labels"], True)
+# ---------------------------------------------------------------- cached_embed_fn
+CACHE_VEC = {"pay me": [1.0, 0.0], "refund please": [0.0, 1.0]}
+CACHE_VEC.update(OPTION_TEXTS)
+CACHE_TEXTS = ["pay me", "alpha", "beta", "gamma: mid", "delta: same"]
+
+check_raises("cache/embed_fn must be callable", lambda: cached_embed_fn(None), TypeError)
+check_raises("cache/maxsize bool rejected", lambda: cached_embed_fn(TableEmbed({}), True))
+check_raises("cache/maxsize zero rejected", lambda: cached_embed_fn(TableEmbed({}), 0))
+check_raises("cache/maxsize negative rejected", lambda: cached_embed_fn(TableEmbed({}), -3))
+check_raises("cache/maxsize must be an int", lambda: cached_embed_fn(TableEmbed({}), "8"))
+
+cache_table = TableEmbed(dict(CACHE_VEC))
+wrapped = cached_embed_fn(cache_table)
+first = wrapped(CACHE_TEXTS)
+check("cache/cold call embeds every text once", cache_table.calls, [CACHE_TEXTS])
+check("cache/cold output shape", first.shape, (5, 2))
+check("cache/cold output dtype is float32", first.dtype == np.float32, True)
+check_true("cache/cold values match unwrapped", np.allclose(first, [CACHE_VEC[t] for t in CACHE_TEXTS]))
+second = wrapped(CACHE_TEXTS)
+check("cache/warm call makes no embed call", len(cache_table.calls), 1)
+check_true("cache/warm returns identical values", np.array_equal(first, second))
+third = wrapped(["refund please", "alpha", "beta"])
+check("cache/repeat embeds only the new text", cache_table.calls[-1], ["refund please"])
+check("cache/underlying calls total", len(cache_table.calls), 2)
+check_true(
+ "cache/partial rows keep request order",
+ np.allclose(third, [CACHE_VEC["refund please"], CACHE_VEC["alpha"], CACHE_VEC["beta"]]),
+)
+
+dup_table = TableEmbed(dict(CACHE_VEC))
+dup_cached = cached_embed_fn(dup_table)
+dup = dup_cached(["alpha", "beta", "alpha"])
+check("cache/duplicate text embedded once per call", dup_table.calls, [["alpha", "beta"]])
+check("cache/duplicate output keeps request length", len(dup), 3)
+check_true("cache/duplicate rows repeat the vector", np.array_equal(dup[0], dup[2]))
+
+lru_table = TableEmbed(dict(CACHE_VEC))
+lru_cached = cached_embed_fn(lru_table, maxsize=2)
+lru_cached(["alpha", "beta"]) # cache: alpha, beta
+lru_cached(["alpha"]) # hit; alpha now newest, beta is LRU
+check("cache/lru touch needs no embed", len(lru_table.calls), 1)
+lru_cached(["gamma: mid"]) # inserts gamma, evicts beta
+check("cache/lru insert embeds the new text", lru_table.calls[-1], ["gamma: mid"])
+lru_cached(["alpha", "gamma: mid"]) # both hits
+check("cache/lru survivors are both cached", len(lru_table.calls), 2)
+lru_cached(["beta"]) # beta was evicted
+check("cache/lru evicted entry is re-embedded", lru_table.calls[-1], ["beta"])
+check("cache/lru size stays at the bound", lru_cached.cache_info()["size"], 2)
+lru_cached(["gamma: mid", "beta"]) # both hits
+check("cache/lru most recent pair survives", len(lru_table.calls), 3)
+lru_cached(["alpha"]) # alpha was the oldest of the three
+check("cache/lru oldest of three was evicted", lru_table.calls[-1], ["alpha"])
+
+info_table = TableEmbed(dict(CACHE_VEC))
+info_cached = cached_embed_fn(info_table, maxsize=8)
+info_cached(["alpha", "beta"]) # 2 misses
+info_cached(["alpha", "gamma: mid"]) # 1 hit + 1 miss
+info = info_cached.cache_info()
+check("cache/info size", info["size"], 3)
+check("cache/info maxsize", info["maxsize"], 8)
+check("cache/info hits", info["hits"], 1)
+check("cache/info misses", info["misses"], 3)
+info_cached.cache_clear()
+check("cache/clear resets info", info_cached.cache_info(), {"size": 0, "maxsize": 8, "hits": 0, "misses": 0})
+info_cached(["alpha"])
+check("cache/clear forces re-embed", info_table.calls[-1], ["alpha"])
+
+
+class FlakyEmbed:
+ """Fails on the first call, succeeds after."""
+
+ def __init__(self):
+ self.calls = []
+
+ def __call__(self, texts):
+ self.calls.append(list(texts))
+ if len(self.calls) == 1:
+ raise RuntimeError("boom")
+ return [[1.0, 0.0] for _text in texts]
+
+
+flaky = FlakyEmbed()
+flaky_cached = cached_embed_fn(flaky)
+check_raises("cache/embed error propagates", lambda: flaky_cached(["alpha"]), RuntimeError)
+check("cache/failed call caches nothing", flaky_cached.cache_info()["size"], 0)
+retry = flaky_cached(["alpha"])
+check("cache/retry calls embed again", len(flaky.calls), 2)
+check_true(
+ "cache/retried row is cached",
+ flaky_cached.cache_info()["size"] == 1 and np.allclose(retry[0], [1.0, 0.0]),
+)
+
+
+class BadShapeEmbed:
+ def __call__(self, texts):
+ return [[1.0, 0.0]] # one row no matter how many texts arrive
+
+
+bad_cached = cached_embed_fn(BadShapeEmbed())
+check_raises("cache/bad shape raises", lambda: bad_cached(["alpha", "beta"]))
+check("cache/bad shape caches nothing", bad_cached.cache_info()["size"], 0)
+
+
+class FakeTensor:
+ """Mimics the detach().float().cpu().numpy() chain of a torch tensor."""
+
+ def __init__(self, rows):
+ self._rows = rows
+
+ def detach(self):
+ return self
+
+ def float(self):
+ return self
+
+ def cpu(self):
+ return self
+
+ def numpy(self):
+ return np.array(self._rows, dtype=np.float64)
+
+
+class TensorEmbed:
+ def __call__(self, texts):
+ return FakeTensor([[1.0, 0.0] for _text in texts])
+
+
+tensor_cached = cached_embed_fn(TensorEmbed())
+tensor_out = tensor_cached(["alpha"])
+check("cache/torch-style return accepted", tensor_out.shape, (1, 2))
+check("cache/torch-style rows stored float32", tensor_out.dtype == np.float32, True)
+
+none_table = TableEmbed({"": [1.0, 0.0]})
+none_cached = cached_embed_fn(none_table)
+none_cached([None])
+check("cache/None text normalizes to empty string", none_table.calls, [[""]])
+none_cached([""])
+check("cache/empty string hits the same entry", len(none_table.calls), 1)
+check("cache/empty input embeds nothing", none_cached([]).shape, (0, 0))
+
+nan_table = TableEmbed({"alpha": [float("nan"), 0.0], "beta": [0.0, 1.0]})
+nan_cached = cached_embed_fn(nan_table)
+nan_out = nan_cached(["alpha", "beta"])
+check_true("cache/nan row stored cleaned", bool(np.isfinite(nan_out).all()))
+nan_cached(["alpha", "beta"])
+check("cache/cleaned row served from cache", len(nan_table.calls), 1)
+
+e2e_table = TableEmbed(dict(CACHE_VEC))
+e2e_cached = cached_embed_fn(e2e_table)
+run1 = shortlist_choice("pay me", CRITERIA, e2e_cached, k=2)
+run2 = shortlist_choice("refund please", CRITERIA, e2e_cached, k=2)
+check("cache/e2e first call embeds query and options", e2e_table.calls[0], CACHE_TEXTS)
+check("cache/e2e repeat embeds only the new query", e2e_table.calls[1], ["refund please"])
+check("cache/e2e two calls total", len(e2e_table.calls), 2)
+check("cache/e2e labels match the uncached path", run1, ["alpha", "delta"])
+check("cache/e2e second query labels", run2, ["beta", "gamma"])
+
+pipe_agent2 = Recorder()
+pipe_vectors2 = dict(CACHE_VEC)
+pipe_vectors2["category\npay me"] = [1.0, 0.0]
+pipe_vectors2["category\nrefund please"] = [0.0, 1.0]
+pipe_table2 = TableEmbed(pipe_vectors2)
+pipe_cached2 = cached_embed_fn(pipe_table2)
+pipe_q2 = {"intent": {"type": "choice", "instructions": "category", "criteria": CRITERIA}}
+predict_shortlist(pipe_agent2, "pay me", pipe_q2, pipe_cached2, k=2)
+pipe_b = predict_shortlist(pipe_agent2, "refund please", pipe_q2, pipe_cached2, k=2)
+check("cache/predict repeat embeds query only", pipe_table2.calls[1], ["category\nrefund please"])
+check("cache/predict keeps shortlist metadata", pipe_b["shortlist"]["intent"]["labels"], ["beta", "gamma"])
+check("cache/predict answer comes from the shortlist", pipe_b["answers"]["intent"]["choice"], "beta")
+
+mt_table = TableEmbed(dict(CACHE_VEC))
+mt_cached = cached_embed_fn(mt_table)
+
+
+def _mt_work(i):
+ texts = ["alpha", "beta"] if i % 2 else ["beta", "gamma: mid"]
+ return texts, mt_cached(texts)
+
+
+with ThreadPoolExecutor(max_workers=4) as pool:
+ mt_results = list(pool.map(_mt_work, range(16)))
+check_true(
+ "cache/concurrent calls return correct rows",
+ all(np.allclose(rows, [CACHE_VEC[t] for t in texts]) for texts, rows in mt_results),
+)
+check_true("cache/concurrent size stays bounded", mt_cached.cache_info()["size"] <= 4096)
+check_true(
+ "cache/concurrent counters consistent",
+ mt_cached.cache_info()["hits"] + mt_cached.cache_info()["misses"] == 32,
+)
+
+
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)