Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(router): build checkpoints outside the lifecycle lock

Tiago Vilas Boasbba951f2bad1
diff --git a/laya/router.py b/laya/router.py
index 33bb0aee..348e6f35 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -575,6 +575,11 @@ class Router(HookRegistry):
# helpers without deadlocking. Inference (`Agent.system_one`) is deliberately left
# outside the lock so concurrent predictions share a checkpoint without serialising.
self._lock = threading.RLock()
+ # Serialises checkpoint builds -- one at a time, as before, so peak memory is unchanged --
+ # without holding `_lock` across one. A cold build takes seconds to minutes (download plus
+ # construction), and under `_lock` it stalled `loaded`, `/health` and every request for a
+ # checkpoint that was already resident.
+ self._build_lock = threading.Lock()
if preload:
self.preload()
@@ -589,7 +594,31 @@ class Router(HookRegistry):
if key in self._agents:
self._touch(key)
return self._agents[key]
- from .agent import Agent
+ with self._build_lock:
+ with self._lock:
+ # Built (or attached) while this caller waited for the build lock.
+ if key in self._agents:
+ self._touch(key)
+ return self._agents[key]
+ agent = self._build(key)
+ with self._lock:
+ if key in self._agents: # attached while it was building: keep that one
+ self._touch(key)
+ return self._agents[key]
+ self._agents[key] = agent
+ self._order.append(key)
+ evicted = self._evict_locked()
+ # Lifecycle hooks fire after the lock is released, so a hook can safely call the Router.
+ self._dispatch_lifecycle("on_evict", evicted)
+ dispatch(compose_hooks(self.hooks), "on_load",
+ PredictContext(states=[], questions={}, model=key, agent=agent, router=self),
+ raise_errors=self.hooks_raise, lock=self._hooks_lock, timeout=self.hooks_timeout)
+ return agent
+
+ def _build(self, key: str):
+ """Construct the Agent for `key`: config is read under `_lock`, the build runs outside it."""
+ from .agent import Agent
+ with self._lock:
repo, sub = _split(self.models[key])
kwargs = {"device": self.device, "token": self.token, "subfolder": sub}
# A None or blank entry in `revisions` is "no pin of its own", so the checkpoint
@@ -607,16 +636,7 @@ class Router(HookRegistry):
_digest_entry(self.sha256_digests, key))
if expected is not None:
kwargs["expected_sha256"] = expected
- agent = Agent(repo, **kwargs)
- self._agents[key] = agent
- self._order.append(key)
- evicted = self._evict_locked()
- # Lifecycle hooks fire after the lock is released, so a hook can safely call the Router.
- self._dispatch_lifecycle("on_evict", evicted)
- dispatch(compose_hooks(self.hooks), "on_load",
- PredictContext(states=[], questions={}, model=key, agent=agent, router=self),
- raise_errors=self.hooks_raise, lock=self._hooks_lock, timeout=self.hooks_timeout)
- return agent
+ return Agent(repo, **kwargs)
def _touch(self, key: str):
with self._lock:
@@ -695,7 +715,9 @@ class Router(HookRegistry):
def unload(self, name: Optional[str] = None):
"""Free one model, or all of them."""
- with self._lock:
+ # Wait for an in-flight build first, as before, so a checkpoint being built cannot land
+ # after the unload that was meant to free it.
+ with self._build_lock, self._lock:
if name is None:
freed = list(self._order)
self._agents.clear()
diff --git a/tests/test_router.py b/tests/test_router.py
index aebba0f7..9202d055 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -1094,6 +1094,67 @@ check("threads/hot-path loads keep agents consistent", agents_len, 1)
check("threads/hot-path order intact", order, ["english"])
+def _cold_build_does_not_hold_lifecycle_lock():
+ """A cold build must not stall `loaded` or a resident checkpoint for its whole duration.
+
+ The build runs a download plus construction, seconds to minutes. Held under `_lock`, it made
+ `GET /health` (which reads `loaded`) and every request for an already-resident checkpoint wait
+ for it, so a liveness probe timed out during a lazy load.
+ """
+ import laya.agent as _agent_mod
+ building, release = threading.Event(), threading.Event()
+ constructions = []
+
+ class _BlockingAgent:
+ def __init__(self, *args, **kwargs):
+ constructions.append(1)
+ building.set()
+ release.wait(5)
+
+ old = _agent_mod.Agent
+ _agent_mod.Agent = _BlockingAgent
+ try:
+ r = Router(max_loaded=3)
+ resident = _Stub("english")
+ r.attach("english", resident)
+ loaders = [threading.Thread(target=r.load, args=("multilingual",)) for _ in range(2)]
+ for t in loaders:
+ t.start()
+ building.wait(5)
+ timings = {}
+ for label, call in (("loaded", lambda: r.loaded),
+ ("resident load", lambda: r.load("english"))):
+ done = threading.Event()
+ threading.Thread(target=lambda c=call, d=done: (c(), d.set()), daemon=True).start()
+ timings[label] = done.wait(1.0)
+ release.set()
+ for t in loaders:
+ t.join(5)
+ loaded_after, built = sorted(r.loaded), len(constructions)
+ # An unload issued while a build is in flight waits for it, so the build cannot land after it.
+ building.clear()
+ release.clear()
+ threading.Thread(target=r.load, args=("typed-decisions",), daemon=True).start()
+ building.wait(5)
+ unloader = threading.Thread(target=r.unload, args=("typed-decisions",))
+ unloader.start()
+ _time.sleep(0.1)
+ release.set()
+ unloader.join(5)
+ timings["unload waits for the build"] = "typed-decisions" not in r.loaded
+ return timings, built, loaded_after
+ finally:
+ release.set()
+ _agent_mod.Agent = old
+
+timings, built, resident_after = _cold_build_does_not_hold_lifecycle_lock()
+check("threads/loaded answers during a cold build", timings["loaded"], True)
+check("threads/resident checkpoint answers during a cold build", timings["resident load"], True)
+check("threads/concurrent cold loads still build once", built, 1)
+check("threads/cold build lands in the LRU", resident_after, ["english", "multilingual"])
+check("threads/unload during a cold build frees it", timings["unload waits for the build"], True)
+
+
# --------------------------------------------------------------------- unlisted scripts
# `detect_script` counts an alphabetic character only when one of `_SCRIPT_RANGES` claims