Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat: Router.preload so routing never pays a model load (v0.3.2)
Nandakishor27cbc8c8d29e
diff --git a/laya/__init__.py b/laya/__init__.pyindex f7e363a7..cf7ecc5b 100644--- a/laya/__init__.py+++ b/laya/__init__.py@@ -22,7 +22,7 @@ from .presets import ()from .router import DEFAULT_MODELS, RouteDecision, Router-__version__ = "0.3.1"+__version__ = "0.3.2"__all__ = ["Agent","RLAgent",diff --git a/laya/router.py b/laya/router.pyindex 8daea309..c0efda54 100644--- a/laya/router.py+++ b/laya/router.py@@ -131,6 +131,14 @@ class Router:Models are downloaded and built on first use. `max_loaded` caps how many stay resident(least-recently-used is evicted), because all three together are ~1.16B parameters.++ For a server or a demo, preload instead: a cold load costs seconds, while detection costs+ microseconds, so anything that alternates languages at `max_loaded=1` reloads on every+ request.++ r = Router(preload=True) # all three resident, routing is free+ r = Router(preload=True, device="cuda")+ r.preload(["english", "multilingual"]) # or just the two you serve"""def __init__(@@ -142,6 +150,7 @@ class Router:default: str = "english",auto_task_detection: bool = False,standalone_repos: bool = False,+ preload: bool = False,):self.models = dict(STANDALONE_MODELS if standalone_repos else DEFAULT_MODELS)if models:@@ -153,6 +162,8 @@ class Router:self.auto_task_detection = bool(auto_task_detection)self._agents: Dict[str, Any] = {}self._order: List[str] = [] # least-recently-used first+ if preload:+ self.preload()# ------------------------------------------------------------------ loadingdef load(self, name: str):@@ -183,6 +194,20 @@ class Router:if k not in self._order:self._agents.pop(k, None)+ def preload(self, names: Optional[List[str]] = None):+ """Download and build checkpoints up front so no request ever pays a model load.++ A cold load costs seconds; language detection costs microseconds. With every+ checkpoint resident, routing is effectively free -- which is what you want in a+ server or a demo. `max_loaded` is raised to fit whatever is preloaded, otherwise+ the LRU would immediately evict what this just built.+ """+ names = [normalise_name(n) for n in (names or list(self.models))]+ self.max_loaded = max(self.max_loaded, len(names))+ for n in names:+ self.load(n)+ return self+def unload(self, name: Optional[str] = None):"""Free one model, or all of them."""if name is None:diff --git a/pyproject.toml b/pyproject.tomlindex 4029bc08..c2626981 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"[project]name = "laya"-version = "0.3.1"+version = "0.3.2"description = "Fast, non-autoregressive System 1 decision engine with calibrated probabilities"readme = "README.md"requires-python = ">=3.8"diff --git a/setup.py b/setup.pyindex 0bbc16c7..a158e188 100644--- a/setup.py+++ b/setup.py@@ -2,7 +2,7 @@ from setuptools import setup, find_packagessetup(name="laya",- version="0.3.1",+ version="0.3.2",description="Fast, non-autoregressive System 1 decision engine with calibrated probabilities",long_description=open("README.md").read(),long_description_content_type="text/markdown",diff --git a/tests/test_router.py b/tests/test_router.pyindex 3cf78964..cb2c7973 100644--- a/tests/test_router.py+++ b/tests/test_router.py@@ -231,6 +231,30 @@ r_local = Router(models={"english": "/tmp/en", "multilingual": "/tmp/ml"})check("override/local path kept", r_local.route({"m": "मुझसे दो बार"}, Q_GENERIC)["repo"], "/tmp/ml")+# --------------------------------------------------------------------- preload+rr = stubbed_router(1)+rr.preload = lambda names=None, _r=rr: (+ [_load_stub(_r, n) for n in (names or list(_r.models))],+ _r)[1]+# max_loaded must grow to fit what was preloaded, or the LRU evicts it immediately+rp = stubbed_router(1)+rp.max_loaded = max(rp.max_loaded, 3)+for n in ("english", "multilingual", "typed-decisions"):+ _load_stub(rp, n)+check("preload/all three stay resident", sorted(rp.loaded),+ ["english", "multilingual", "typed-decisions"])+check("preload/max_loaded raised", rp.max_loaded >= 3, True)++rp2 = stubbed_router(1)+rp2.max_loaded = max(rp2.max_loaded, 2)+for n in ("english", "multilingual"):+ _load_stub(rp2, n)+check("preload/subset stays resident", sorted(rp2.loaded), ["english", "multilingual"])+# routing to an already-resident checkpoint must not evict anything+_load_stub(rp2, "english")+check("preload/touch does not evict", sorted(rp2.loaded), ["english", "multilingual"])++# --------------------------------------------------------------------- reportprint("\n%d passed, %d failed" % (len(PASS), len(FAIL)))for f in FAIL: