Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(router): make model lifecycle thread-safe (fixes #95) (#100)
b1tzera7ec8421e7c5
diff --git a/laya/router.py b/laya/router.pyindex 545063ab..1f123520 100644--- a/laya/router.py+++ b/laya/router.py@@ -28,6 +28,7 @@ primary routing signal.synthetic workflows and should not be a silent default."""import os+import threadingfrom typing import Any, Dict, List, Optional, Unionfrom .lang import analyse@@ -162,37 +163,48 @@ class Router:self.auto_task_detection = bool(auto_task_detection)self._agents: Dict[str, Any] = {}self._order: List[str] = [] # least-recently-used first+ # Re-entrant lock guarding model lifecycle (load/unload/attach/preload) and the+ # LRU bookkeeping. RLock so the public methods can call the private `_touch`/`_evict`+ # helpers without deadlocking. Inference (`Agent.system_one`) is deliberately left+ # outside the lock so concurrent predictions share a checkpoint without serialising.+ self._lock = threading.RLock()if preload:self.preload()# ------------------------------------------------------------------ loadingdef load(self, name: str):- """Return the Agent for `name`, downloading and building it on first use."""+ """Return the Agent for `name`, downloading and building it on first use.++ Concurrent callers share a single Agent instead of building duplicates.+ """key = normalise_name(name)- if key in self._agents:- self._touch(key)- return self._agents[key]- from .agent import Agent- repo, sub = _split(self.models[key])- agent = Agent(repo, device=self.device, token=self.token, subfolder=sub)- self._agents[key] = agent- self._order.append(key)- self._evict()- return agent+ with self._lock:+ if key in self._agents:+ self._touch(key)+ return self._agents[key]+ from .agent import Agent+ repo, sub = _split(self.models[key])+ agent = Agent(repo, device=self.device, token=self.token, subfolder=sub)+ self._agents[key] = agent+ self._order.append(key)+ self._evict()+ return agentdef _touch(self, key: str):- if key in self._order:- self._order.remove(key)- self._order.append(key)+ with self._lock:+ if key in self._order:+ self._order.remove(key)+ self._order.append(key)def _evict(self):- while len(self._order) > self.max_loaded:- victim = self._order.pop(0)- self._agents.pop(victim, None)- if len(self._order) < len(self._agents): # keep the two views consistent- for k in list(self._agents):- if k not in self._order:- self._agents.pop(k, None)+ with self._lock:+ while len(self._order) > self.max_loaded:+ victim = self._order.pop(0)+ self._agents.pop(victim, None)+ if len(self._order) < len(self._agents): # keep the two views consistent+ for k in list(self._agents):+ if k not in self._order:+ self._agents.pop(k, None)def attach(self, name: str, agent: Any):"""Register an already-built Agent under `name` instead of loading a second copy.@@ -202,9 +214,10 @@ class Router:in memory -- a duplicate 421M parameters."""key = normalise_name(name)- self._agents[key] = agent- self._touch(key)- self.max_loaded = max(self.max_loaded, len(self._agents))+ with self._lock:+ self._agents[key] = agent+ self._touch(key)+ self.max_loaded = max(self.max_loaded, len(self._agents))return agentdef preload(self, names: Optional[List[str]] = None):@@ -216,26 +229,29 @@ class Router:the LRU would immediately evict what this just built."""names = [normalise_name(n) for n in (names or list(self.models))]- self.max_loaded = max(self.max_loaded, len(names), len(self._agents))- for n in names:- if n not in self._agents: # an attached agent is already built- self.load(n)+ with self._lock:+ self.max_loaded = max(self.max_loaded, len(names), len(self._agents))+ for n in names:+ if n not in self._agents: # an attached agent is already built+ self.load(n)return selfdef unload(self, name: Optional[str] = None):"""Free one model, or all of them."""- if name is None:- self._agents.clear()- self._order.clear()- else:- key = normalise_name(name)- self._agents.pop(key, None)- if key in self._order:- self._order.remove(key)+ with self._lock:+ if name is None:+ self._agents.clear()+ self._order.clear()+ else:+ key = normalise_name(name)+ self._agents.pop(key, None)+ if key in self._order:+ self._order.remove(key)@propertydef loaded(self) -> List[str]:- return list(self._order)+ with self._lock:+ return list(self._order)# ------------------------------------------------------------------ routingdef route(diff --git a/tests/test_router.py b/tests/test_router.pyindex f01b082c..c84b3149 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -1,6 +1,8 @@"""Routing and language-detection tests. No model weights are loaded: `Router.route` is pure."""import sysimport os+import threading+import time as _timesys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))@@ -279,6 +281,81 @@ check("attach/still the same object", ra._agents["english"] is sentinel, True)check("attach/accepts aliases", stubbed_router(1).attach("en", _Stub("x")) is not None, True)+# --------------------------------------------------------------------- thread safety (issue #95)++def _concurrent_load_dedup():+ """Concurrent load() of the same checkpoint must build one Agent, shared by all callers."""+ import laya.agent as _agent_mod+ constructions = []+ cl = threading.Lock()++ class _SlowAgent:+ def __init__(self, *args, **kwargs):+ _time.sleep(0.05) # widen the check-then-build window+ with cl:+ constructions.append(1)++ def system_one(self, state, questions):+ return {"model": "fake", "answers": {}, "usage": {}}++ old = _agent_mod.Agent+ _agent_mod.Agent = _SlowAgent+ try:+ r = Router()+ got = []++ def _worker():+ got.append(r.load("english"))++ threads = [threading.Thread(target=_worker) for _ in range(8)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()+ return len({id(x) for x in got}), len(constructions), len(r._order), sorted(r._agents)+ finally:+ _agent_mod.Agent = old++unique, built, order_len, agents = _concurrent_load_dedup()+check("threads/8 concurrent loads share one Agent", unique, 1)+check("threads/Agent constructed exactly once", built, 1)+check("threads/LRU views stay consistent", (order_len == 1 and agents == ["english"]), True)+++def _concurrent_hotpath():+ """Concurrent hot-path loads of an already-cached model must keep _order/_agents consistent."""+ import laya.agent as _agent_mod++ class _Agent:+ def __init__(self, *args, **kwargs):+ pass++ def system_one(self, state, questions):+ return {"model": "fake", "answers": {}, "usage": {}}++ old = _agent_mod.Agent+ _agent_mod.Agent = _Agent+ try:+ r = Router(max_loaded=3)+ r.load("english") # warm the cache++ def _worker():+ r.load("english")++ threads = [threading.Thread(target=_worker) for _ in range(20)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()+ return len(r._order), len(r._agents), r._order+ finally:+ _agent_mod.Agent = old++order_len, agents_len, order = _concurrent_hotpath()+check("threads/hot-path loads keep one entry", order_len, 1)+check("threads/hot-path loads keep agents consistent", agents_len, 1)+check("threads/hot-path order intact", order, ["english"])+# --------------------------------------------------------------------- reportprint("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL: