Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(router): build checkpoints outside the lifecycle lock
Tiago Vilas Boasbba951f2bad1
diff --git a/laya/router.py b/laya/router.pyindex 33bb0aee..348e6f35 100644--- a/laya/router.py+++ b/laya/router.py@@ -575,6 +575,11 @@ class Router(HookRegistry):# helpers without deadlocking. Inference (`Agent.system_one`) is deliberately left# outside the lock so concurrent predictions share a checkpoint without serialising.self._lock = threading.RLock()+ # Serialises checkpoint builds -- one at a time, as before, so peak memory is unchanged --+ # without holding `_lock` across one. A cold build takes seconds to minutes (download plus+ # construction), and under `_lock` it stalled `loaded`, `/health` and every request for a+ # checkpoint that was already resident.+ self._build_lock = threading.Lock()if preload:self.preload()@@ -589,7 +594,31 @@ class Router(HookRegistry):if key in self._agents:self._touch(key)return self._agents[key]- from .agent import Agent+ with self._build_lock:+ with self._lock:+ # Built (or attached) while this caller waited for the build lock.+ if key in self._agents:+ self._touch(key)+ return self._agents[key]+ agent = self._build(key)+ with self._lock:+ if key in self._agents: # attached while it was building: keep that one+ self._touch(key)+ return self._agents[key]+ self._agents[key] = agent+ self._order.append(key)+ evicted = self._evict_locked()+ # Lifecycle hooks fire after the lock is released, so a hook can safely call the Router.+ self._dispatch_lifecycle("on_evict", evicted)+ dispatch(compose_hooks(self.hooks), "on_load",+ PredictContext(states=[], questions={}, model=key, agent=agent, router=self),+ raise_errors=self.hooks_raise, lock=self._hooks_lock, timeout=self.hooks_timeout)+ return agent++ def _build(self, key: str):+ """Construct the Agent for `key`: config is read under `_lock`, the build runs outside it."""+ from .agent import Agent+ with self._lock:repo, sub = _split(self.models[key])kwargs = {"device": self.device, "token": self.token, "subfolder": sub}# A None or blank entry in `revisions` is "no pin of its own", so the checkpoint@@ -607,16 +636,7 @@ class Router(HookRegistry):_digest_entry(self.sha256_digests, key))if expected is not None:kwargs["expected_sha256"] = expected- agent = Agent(repo, **kwargs)- self._agents[key] = agent- self._order.append(key)- evicted = self._evict_locked()- # Lifecycle hooks fire after the lock is released, so a hook can safely call the Router.- self._dispatch_lifecycle("on_evict", evicted)- dispatch(compose_hooks(self.hooks), "on_load",- PredictContext(states=[], questions={}, model=key, agent=agent, router=self),- raise_errors=self.hooks_raise, lock=self._hooks_lock, timeout=self.hooks_timeout)- return agent+ return Agent(repo, **kwargs)def _touch(self, key: str):with self._lock:@@ -695,7 +715,9 @@ class Router(HookRegistry):def unload(self, name: Optional[str] = None):"""Free one model, or all of them."""- with self._lock:+ # Wait for an in-flight build first, as before, so a checkpoint being built cannot land+ # after the unload that was meant to free it.+ with self._build_lock, self._lock:if name is None:freed = list(self._order)self._agents.clear()diff --git a/tests/test_router.py b/tests/test_router.pyindex aebba0f7..9202d055 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -1094,6 +1094,67 @@ check("threads/hot-path loads keep agents consistent", agents_len, 1)check("threads/hot-path order intact", order, ["english"])+def _cold_build_does_not_hold_lifecycle_lock():+ """A cold build must not stall `loaded` or a resident checkpoint for its whole duration.++ The build runs a download plus construction, seconds to minutes. Held under `_lock`, it made+ `GET /health` (which reads `loaded`) and every request for an already-resident checkpoint wait+ for it, so a liveness probe timed out during a lazy load.+ """+ import laya.agent as _agent_mod+ building, release = threading.Event(), threading.Event()+ constructions = []++ class _BlockingAgent:+ def __init__(self, *args, **kwargs):+ constructions.append(1)+ building.set()+ release.wait(5)++ old = _agent_mod.Agent+ _agent_mod.Agent = _BlockingAgent+ try:+ r = Router(max_loaded=3)+ resident = _Stub("english")+ r.attach("english", resident)+ loaders = [threading.Thread(target=r.load, args=("multilingual",)) for _ in range(2)]+ for t in loaders:+ t.start()+ building.wait(5)+ timings = {}+ for label, call in (("loaded", lambda: r.loaded),+ ("resident load", lambda: r.load("english"))):+ done = threading.Event()+ threading.Thread(target=lambda c=call, d=done: (c(), d.set()), daemon=True).start()+ timings[label] = done.wait(1.0)+ release.set()+ for t in loaders:+ t.join(5)+ loaded_after, built = sorted(r.loaded), len(constructions)+ # An unload issued while a build is in flight waits for it, so the build cannot land after it.+ building.clear()+ release.clear()+ threading.Thread(target=r.load, args=("typed-decisions",), daemon=True).start()+ building.wait(5)+ unloader = threading.Thread(target=r.unload, args=("typed-decisions",))+ unloader.start()+ _time.sleep(0.1)+ release.set()+ unloader.join(5)+ timings["unload waits for the build"] = "typed-decisions" not in r.loaded+ return timings, built, loaded_after+ finally:+ release.set()+ _agent_mod.Agent = old++timings, built, resident_after = _cold_build_does_not_hold_lifecycle_lock()+check("threads/loaded answers during a cold build", timings["loaded"], True)+check("threads/resident checkpoint answers during a cold build", timings["resident load"], True)+check("threads/concurrent cold loads still build once", built, 1)+check("threads/cold build lands in the LRU", resident_after, ["english", "multilingual"])+check("threads/unload during a cold build frees it", timings["unload waits for the build"], True)++# --------------------------------------------------------------------- unlisted scripts# `detect_script` counts an alphabetic character only when one of `_SCRIPT_RANGES` claims