Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(router): make model lifecycle thread-safe (fixes #95) (#100)

b1tzera7ec8421e7c5
diff --git a/laya/router.py b/laya/router.py
index 545063ab..1f123520 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -28,6 +28,7 @@ primary routing signal.
synthetic workflows and should not be a silent default.
"""
import os
+import threading
from typing import Any, Dict, List, Optional, Union
from .lang import analyse
@@ -162,37 +163,48 @@ class Router:
self.auto_task_detection = bool(auto_task_detection)
self._agents: Dict[str, Any] = {}
self._order: List[str] = [] # least-recently-used first
+ # Re-entrant lock guarding model lifecycle (load/unload/attach/preload) and the
+ # LRU bookkeeping. RLock so the public methods can call the private `_touch`/`_evict`
+ # helpers without deadlocking. Inference (`Agent.system_one`) is deliberately left
+ # outside the lock so concurrent predictions share a checkpoint without serialising.
+ self._lock = threading.RLock()
if preload:
self.preload()
# ------------------------------------------------------------------ loading
def load(self, name: str):
- """Return the Agent for `name`, downloading and building it on first use."""
+ """Return the Agent for `name`, downloading and building it on first use.
+
+ Concurrent callers share a single Agent instead of building duplicates.
+ """
key = normalise_name(name)
- if key in self._agents:
- self._touch(key)
- return self._agents[key]
- from .agent import Agent
- repo, sub = _split(self.models[key])
- agent = Agent(repo, device=self.device, token=self.token, subfolder=sub)
- self._agents[key] = agent
- self._order.append(key)
- self._evict()
- return agent
+ with self._lock:
+ if key in self._agents:
+ self._touch(key)
+ return self._agents[key]
+ from .agent import Agent
+ repo, sub = _split(self.models[key])
+ agent = Agent(repo, device=self.device, token=self.token, subfolder=sub)
+ self._agents[key] = agent
+ self._order.append(key)
+ self._evict()
+ return agent
def _touch(self, key: str):
- if key in self._order:
- self._order.remove(key)
- self._order.append(key)
+ with self._lock:
+ if key in self._order:
+ self._order.remove(key)
+ self._order.append(key)
def _evict(self):
- while len(self._order) > self.max_loaded:
- victim = self._order.pop(0)
- self._agents.pop(victim, None)
- if len(self._order) < len(self._agents): # keep the two views consistent
- for k in list(self._agents):
- if k not in self._order:
- self._agents.pop(k, None)
+ with self._lock:
+ while len(self._order) > self.max_loaded:
+ victim = self._order.pop(0)
+ self._agents.pop(victim, None)
+ if len(self._order) < len(self._agents): # keep the two views consistent
+ for k in list(self._agents):
+ if k not in self._order:
+ self._agents.pop(k, None)
def attach(self, name: str, agent: Any):
"""Register an already-built Agent under `name` instead of loading a second copy.
@@ -202,9 +214,10 @@ class Router:
in memory -- a duplicate 421M parameters.
"""
key = normalise_name(name)
- self._agents[key] = agent
- self._touch(key)
- self.max_loaded = max(self.max_loaded, len(self._agents))
+ with self._lock:
+ self._agents[key] = agent
+ self._touch(key)
+ self.max_loaded = max(self.max_loaded, len(self._agents))
return agent
def preload(self, names: Optional[List[str]] = None):
@@ -216,26 +229,29 @@ class Router:
the LRU would immediately evict what this just built.
"""
names = [normalise_name(n) for n in (names or list(self.models))]
- self.max_loaded = max(self.max_loaded, len(names), len(self._agents))
- for n in names:
- if n not in self._agents: # an attached agent is already built
- self.load(n)
+ with self._lock:
+ self.max_loaded = max(self.max_loaded, len(names), len(self._agents))
+ for n in names:
+ if n not in self._agents: # an attached agent is already built
+ self.load(n)
return self
def unload(self, name: Optional[str] = None):
"""Free one model, or all of them."""
- if name is None:
- self._agents.clear()
- self._order.clear()
- else:
- key = normalise_name(name)
- self._agents.pop(key, None)
- if key in self._order:
- self._order.remove(key)
+ with self._lock:
+ if name is None:
+ self._agents.clear()
+ self._order.clear()
+ else:
+ key = normalise_name(name)
+ self._agents.pop(key, None)
+ if key in self._order:
+ self._order.remove(key)
@property
def loaded(self) -> List[str]:
- return list(self._order)
+ with self._lock:
+ return list(self._order)
# ------------------------------------------------------------------ routing
def route(
diff --git a/tests/test_router.py b/tests/test_router.py
index f01b082c..c84b3149 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -1,6 +1,8 @@
"""Routing and language-detection tests. No model weights are loaded: `Router.route` is pure."""
import sys
import os
+import threading
+import time as _time
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -279,6 +281,81 @@ check("attach/still the same object", ra._agents["english"] is sentinel, True)
check("attach/accepts aliases", stubbed_router(1).attach("en", _Stub("x")) is not None, True)
+# --------------------------------------------------------------------- thread safety (issue #95)
+
+def _concurrent_load_dedup():
+ """Concurrent load() of the same checkpoint must build one Agent, shared by all callers."""
+ import laya.agent as _agent_mod
+ constructions = []
+ cl = threading.Lock()
+
+ class _SlowAgent:
+ def __init__(self, *args, **kwargs):
+ _time.sleep(0.05) # widen the check-then-build window
+ with cl:
+ constructions.append(1)
+
+ def system_one(self, state, questions):
+ return {"model": "fake", "answers": {}, "usage": {}}
+
+ old = _agent_mod.Agent
+ _agent_mod.Agent = _SlowAgent
+ try:
+ r = Router()
+ got = []
+
+ def _worker():
+ got.append(r.load("english"))
+
+ threads = [threading.Thread(target=_worker) for _ in range(8)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ return len({id(x) for x in got}), len(constructions), len(r._order), sorted(r._agents)
+ finally:
+ _agent_mod.Agent = old
+
+unique, built, order_len, agents = _concurrent_load_dedup()
+check("threads/8 concurrent loads share one Agent", unique, 1)
+check("threads/Agent constructed exactly once", built, 1)
+check("threads/LRU views stay consistent", (order_len == 1 and agents == ["english"]), True)
+
+
+def _concurrent_hotpath():
+ """Concurrent hot-path loads of an already-cached model must keep _order/_agents consistent."""
+ import laya.agent as _agent_mod
+
+ class _Agent:
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def system_one(self, state, questions):
+ return {"model": "fake", "answers": {}, "usage": {}}
+
+ old = _agent_mod.Agent
+ _agent_mod.Agent = _Agent
+ try:
+ r = Router(max_loaded=3)
+ r.load("english") # warm the cache
+
+ def _worker():
+ r.load("english")
+
+ threads = [threading.Thread(target=_worker) for _ in range(20)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ return len(r._order), len(r._agents), r._order
+ finally:
+ _agent_mod.Agent = old
+
+order_len, agents_len, order = _concurrent_hotpath()
+check("threads/hot-path loads keep one entry", order_len, 1)
+check("threads/hot-path loads keep agents consistent", agents_len, 1)
+check("threads/hot-path order intact", order, ["english"])
+
# --------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL: