Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(shortlist): take the cache lock in the dimensionality guard
Aashish586366e1ef4f
diff --git a/laya/shortlist.py b/laya/shortlist.pyindex 123d4edb..16fdca70 100644--- a/laya/shortlist.py+++ b/laya/shortlist.py@@ -343,21 +343,23 @@ def cached_embed_fn(% (len(missing), tuple(fresh.shape)))fresh = np.nan_to_num(fresh, copy=True, nan=0.0, posinf=0.0, neginf=0.0)- if rows_by_text:- # Rows stored under one dimensionality cannot be stacked against rows of- # another: a changed embedder (or one whose dimension drifts between calls)- # would otherwise surface rows of mixed width, which `_rank` reads as one- # matrix. Deciding by identity is what the docstring already asks of the- # caller ("clear the cache if the model changes") -- this refuses the call- # instead of returning a matrix that silently mixes both.- want = next(iter(rows_by_text.values())).shape[0]- if fresh.shape[1] != want:- raise ValueError(- "embed_fn returned dim %d, but the cache holds dim %d; "- "call cache_clear() if the model behind embed_fn changed"- % (fresh.shape[1], want)- )with lock:+ if rows_by_text:+ # Rows stored under one dimensionality cannot be stacked against rows of+ # another: a changed embedder (or one whose dimension drifts between calls)+ # would otherwise surface rows of mixed width, which `_rank` reads as one+ # matrix. Deciding by identity is what the docstring already asks of the+ # caller ("clear the cache if the model changes") -- this refuses the call+ # instead of returning a matrix that silently mixes both. It is inside the+ # lock because a concurrent insert, eviction or `cache_clear()` mutates the+ # OrderedDict this reads.+ want = next(iter(rows_by_text.values())).shape[0]+ if fresh.shape[1] != want:+ raise ValueError(+ "embed_fn returned dim %d, but the cache holds dim %d; "+ "call cache_clear() if the model behind embed_fn changed"+ % (fresh.shape[1], want)+ )for key, row in zip(missing, fresh):rows_by_text[key] = rowrows_by_text.move_to_end(key)diff --git a/tests/test_shortlist.py b/tests/test_shortlist.pyindex c45b66a4..e111b4e9 100644--- a/tests/test_shortlist.py+++ b/tests/test_shortlist.py@@ -6,6 +6,7 @@ decision model is never constructed.import inspectimport osimport sys+import threadingimport warningsfrom concurrent.futures import ThreadPoolExecutor@@ -779,6 +780,58 @@ swap_cached(["beta"])check("cache/dim clear then re-embed works", swap_cached.cache_info()["size"], 1)+# ------------------------------------------------------- the dimensionality guard is a read+# That guard reads `rows_by_text`, so it needs the lock every other read holds. Measured on+# the unlocked version: 3 callers plus a `cache_clear()` churner over 6,000 calls let 41 to 61+# exceptions escape on each of five runs, every one of them a RuntimeError ("OrderedDict mutated+# during iteration") or a StopIteration out of `next(iter(...))`, raised at the guard line and+# handed to the caller instead of a matrix. The switch interval is compressed because at CPython's+# default 5 ms the same 6,000 calls escaped 0 -- the window between the truthiness test and the+# iterator is two bytecodes wide, and a thread only lands in it when the interpreter chooses to+# switch there.+race_errors = []+race_shapes = []+race_stop = threading.Event()+race_fn = cached_embed_fn(lambda texts: np.zeros((len(texts), 8), dtype=np.float32), maxsize=4)+++def _race_caller(tid):+ for n in range(2000):+ try:+ rows = race_fn(["t%d-%d" % (tid, n)])+ if rows.shape != (1, 8):+ race_shapes.append(rows.shape)+ except BaseException as exc:+ race_errors.append(type(exc).__name__)+++def _race_churner():+ while not race_stop.is_set():+ race_fn.cache_clear()+++_switch_interval = sys.getswitchinterval()+sys.setswitchinterval(1e-6)+try:+ _churner = threading.Thread(target=_race_churner)+ _churner.start()+ _callers = [threading.Thread(target=_race_caller, args=(t,)) for t in range(3)]+ for _caller in _callers:+ _caller.start()+ for _caller in _callers:+ _caller.join()+ race_stop.set()+ _churner.join()+finally:+ sys.setswitchinterval(_switch_interval)++check_true("cache/concurrent dimensionality guard does not iterate a mutating cache",+ not race_errors,+ "%d escaped: %s" % (len(race_errors), sorted(set(race_errors))))+check_true("cache/concurrent guarded calls still return full-width rows",+ not race_shapes, "%d calls returned %s" % (len(race_shapes), race_shapes[:3]))++# --------------------------------------------------------------- tournamentclass Ranker:"""Stand-in for Agent that always answers the label it ranks highest, in any group."""