Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat: Router.preload so routing never pays a model load (v0.3.2)

Nandakishor27cbc8c8d29e
diff --git a/laya/__init__.py b/laya/__init__.py
index f7e363a7..cf7ecc5b 100644
--- a/laya/__init__.py
+++ b/laya/__init__.py
@@ -22,7 +22,7 @@ from .presets import (
)
from .router import DEFAULT_MODELS, RouteDecision, Router
-__version__ = "0.3.1"
+__version__ = "0.3.2"
__all__ = [
"Agent",
"RLAgent",
diff --git a/laya/router.py b/laya/router.py
index 8daea309..c0efda54 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -131,6 +131,14 @@ class Router:
Models are downloaded and built on first use. `max_loaded` caps how many stay resident
(least-recently-used is evicted), because all three together are ~1.16B parameters.
+
+ For a server or a demo, preload instead: a cold load costs seconds, while detection costs
+ microseconds, so anything that alternates languages at `max_loaded=1` reloads on every
+ request.
+
+ r = Router(preload=True) # all three resident, routing is free
+ r = Router(preload=True, device="cuda")
+ r.preload(["english", "multilingual"]) # or just the two you serve
"""
def __init__(
@@ -142,6 +150,7 @@ class Router:
default: str = "english",
auto_task_detection: bool = False,
standalone_repos: bool = False,
+ preload: bool = False,
):
self.models = dict(STANDALONE_MODELS if standalone_repos else DEFAULT_MODELS)
if models:
@@ -153,6 +162,8 @@ class Router:
self.auto_task_detection = bool(auto_task_detection)
self._agents: Dict[str, Any] = {}
self._order: List[str] = [] # least-recently-used first
+ if preload:
+ self.preload()
# ------------------------------------------------------------------ loading
def load(self, name: str):
@@ -183,6 +194,20 @@ class Router:
if k not in self._order:
self._agents.pop(k, None)
+ def preload(self, names: Optional[List[str]] = None):
+ """Download and build checkpoints up front so no request ever pays a model load.
+
+ A cold load costs seconds; language detection costs microseconds. With every
+ checkpoint resident, routing is effectively free -- which is what you want in a
+ server or a demo. `max_loaded` is raised to fit whatever is preloaded, otherwise
+ the LRU would immediately evict what this just built.
+ """
+ names = [normalise_name(n) for n in (names or list(self.models))]
+ self.max_loaded = max(self.max_loaded, len(names))
+ for n in names:
+ self.load(n)
+ return self
+
def unload(self, name: Optional[str] = None):
"""Free one model, or all of them."""
if name is None:
diff --git a/pyproject.toml b/pyproject.toml
index 4029bc08..c2626981 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "laya"
-version = "0.3.1"
+version = "0.3.2"
description = "Fast, non-autoregressive System 1 decision engine with calibrated probabilities"
readme = "README.md"
requires-python = ">=3.8"
diff --git a/setup.py b/setup.py
index 0bbc16c7..a158e188 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name="laya",
- version="0.3.1",
+ version="0.3.2",
description="Fast, non-autoregressive System 1 decision engine with calibrated probabilities",
long_description=open("README.md").read(),
long_description_content_type="text/markdown",
diff --git a/tests/test_router.py b/tests/test_router.py
index 3cf78964..cb2c7973 100644
--- a/tests/test_router.py
+++ b/tests/test_router.py
@@ -231,6 +231,30 @@ r_local = Router(models={"english": "/tmp/en", "multilingual": "/tmp/ml"})
check("override/local path kept", r_local.route({"m": "मुझसे दो बार"}, Q_GENERIC)["repo"], "/tmp/ml")
+# --------------------------------------------------------------------- preload
+rr = stubbed_router(1)
+rr.preload = lambda names=None, _r=rr: (
+ [_load_stub(_r, n) for n in (names or list(_r.models))],
+ _r)[1]
+# max_loaded must grow to fit what was preloaded, or the LRU evicts it immediately
+rp = stubbed_router(1)
+rp.max_loaded = max(rp.max_loaded, 3)
+for n in ("english", "multilingual", "typed-decisions"):
+ _load_stub(rp, n)
+check("preload/all three stay resident", sorted(rp.loaded),
+ ["english", "multilingual", "typed-decisions"])
+check("preload/max_loaded raised", rp.max_loaded >= 3, True)
+
+rp2 = stubbed_router(1)
+rp2.max_loaded = max(rp2.max_loaded, 2)
+for n in ("english", "multilingual"):
+ _load_stub(rp2, n)
+check("preload/subset stays resident", sorted(rp2.loaded), ["english", "multilingual"])
+# routing to an already-resident checkpoint must not evict anything
+_load_stub(rp2, "english")
+check("preload/touch does not evict", sorted(rp2.loaded), ["english", "multilingual"])
+
+
# --------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL: