Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

perf(router): reclaim VRAM and trigger garbage collection on eviction and unload (#147)

Kartik Chilkotibe94a166f402
diff --git a/laya/router.py b/laya/router.py
index 7d9323b6..c6fd1317 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -27,6 +27,7 @@ primary routing signal.
`auto_task_detection=True` or pass `task="typed_decisions"`: it is fine-tuned on four specific
synthetic workflows and should not be a silent default.
"""
+import gc
import os
import threading
from typing import Any, Dict, List, Optional, Union
@@ -236,13 +237,28 @@ class Router:
def _evict(self):
with self._lock:
+ evicted = False
while len(self._order) > self.max_loaded:
victim = self._order.pop(0)
- self._agents.pop(victim, None)
+ agent = self._agents.pop(victim, None)
+ if agent is not None:
+ evicted = True
+ del agent
if len(self._order) < len(self._agents): # keep the two views consistent
for k in list(self._agents):
if k not in self._order:
- self._agents.pop(k, None)
+ agent = self._agents.pop(k, None)
+ if agent is not None:
+ evicted = True
+ del agent
+ if evicted:
+ gc.collect()
+ try:
+ import torch
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ except Exception:
+ pass
def attach(self, name: str, agent: Any):
"""Register an already-built Agent under `name` instead of loading a second copy.
@@ -282,9 +298,17 @@ class Router:
self._order.clear()
else:
key = normalise_name(name)
- self._agents.pop(key, None)
+ agent = self._agents.pop(key, None)
if key in self._order:
self._order.remove(key)
+ del agent
+ gc.collect()
+ try:
+ import torch
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ except Exception:
+ pass
@property
def loaded(self) -> List[str]:
diff --git a/tests/test_router_memory.py b/tests/test_router_memory.py
new file mode 100644
index 00000000..486eba5a
--- /dev/null
+++ b/tests/test_router_memory.py
@@ -0,0 +1,25 @@
+import unittest
+from laya.router import Router
+
+class MockAgent:
+ def __init__(self, name):
+ self.name = name
+
+class RouterMemoryTests(unittest.TestCase):
+ def test_evict_and_unload_memory_release(self):
+ r = Router(max_loaded=1)
+ r.attach("english", MockAgent("en"))
+ r.attach("multilingual", MockAgent("multi"))
+ self.assertEqual(len(r.loaded), 2)
+
+ # Trigger unload of specific model
+ r.unload("english")
+ self.assertNotIn("english", r.loaded)
+ self.assertIn("multilingual", r.loaded)
+
+ # Trigger unload all
+ r.unload()
+ self.assertEqual(len(r.loaded), 0)
+
+if __name__ == "__main__":
+ unittest.main()