Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
perf(router): reclaim VRAM and trigger garbage collection on eviction and unload (#147)
Kartik Chilkotibe94a166f402
diff --git a/laya/router.py b/laya/router.pyindex 7d9323b6..c6fd1317 100644--- a/laya/router.py+++ b/laya/router.py@@ -27,6 +27,7 @@ primary routing signal.`auto_task_detection=True` or pass `task="typed_decisions"`: it is fine-tuned on four specificsynthetic workflows and should not be a silent default."""+import gcimport osimport threadingfrom typing import Any, Dict, List, Optional, Union@@ -236,13 +237,28 @@ class Router:def _evict(self):with self._lock:+ evicted = Falsewhile len(self._order) > self.max_loaded:victim = self._order.pop(0)- self._agents.pop(victim, None)+ agent = self._agents.pop(victim, None)+ if agent is not None:+ evicted = True+ del agentif len(self._order) < len(self._agents): # keep the two views consistentfor k in list(self._agents):if k not in self._order:- self._agents.pop(k, None)+ agent = self._agents.pop(k, None)+ if agent is not None:+ evicted = True+ del agent+ if evicted:+ gc.collect()+ try:+ import torch+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+ except Exception:+ passdef attach(self, name: str, agent: Any):"""Register an already-built Agent under `name` instead of loading a second copy.@@ -282,9 +298,17 @@ class Router:self._order.clear()else:key = normalise_name(name)- self._agents.pop(key, None)+ agent = self._agents.pop(key, None)if key in self._order:self._order.remove(key)+ del agent+ gc.collect()+ try:+ import torch+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+ except Exception:+ pass@propertydef loaded(self) -> List[str]:diff --git a/tests/test_router_memory.py b/tests/test_router_memory.pynew file mode 100644index 00000000..486eba5a--- /dev/null+++ b/tests/test_router_memory.py@@ -0,0 +1,25 @@+import unittest+from laya.router import Router++class MockAgent:+ def __init__(self, name):+ self.name = name++class RouterMemoryTests(unittest.TestCase):+ def test_evict_and_unload_memory_release(self):+ r = Router(max_loaded=1)+ r.attach("english", MockAgent("en"))+ r.attach("multilingual", MockAgent("multi"))+ self.assertEqual(len(r.loaded), 2)++ # Trigger unload of specific model+ r.unload("english")+ self.assertNotIn("english", r.loaded)+ self.assertIn("multilingual", r.loaded)++ # Trigger unload all+ r.unload()+ self.assertEqual(len(r.loaded), 0)++if __name__ == "__main__":+ unittest.main()