Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(runtime): add context manager support to Agent and Router for deterministic resource cleanup (#150)

Kartik Chilkoti1fd60b937a5e
diff --git a/laya/agent.py b/laya/agent.py
index dedb69c2..70163bf9 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -429,6 +429,23 @@ class Agent:
"usage": {"input_tokens": n_tokens, "output_tokens": 0},
}
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ if hasattr(self, "model") and self.model is not None:
+ del self.model
+ self.model = None
+ try:
+ import gc
+ gc.collect()
+ import torch
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ except Exception:
+ pass
+ return False
+
predict = system_one
diff --git a/laya/router.py b/laya/router.py
index c6fd1317..5892b7a0 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -434,6 +434,13 @@ class Router:
result["routing"] = dict(decision)
return result
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.unload()
+ return False
+
system_one = predict
def __repr__(self):
diff --git a/tests/test_context_manager.py b/tests/test_context_manager.py
new file mode 100644
index 00000000..4ab7ad62
--- /dev/null
+++ b/tests/test_context_manager.py
@@ -0,0 +1,19 @@
+import unittest
+from laya.router import Router
+
+class MockAgent:
+ def __init__(self, name):
+ self.name = name
+
+class ContextManagerTests(unittest.TestCase):
+ def test_router_context_manager_cleanup(self):
+ with Router(max_loaded=2) as r:
+ r.attach("english", MockAgent("en"))
+ r.attach("multilingual", MockAgent("multi"))
+ self.assertEqual(len(r.loaded), 2)
+
+ # After exiting context block, all models must be unloaded
+ self.assertEqual(len(r.loaded), 0)
+
+if __name__ == "__main__":
+ unittest.main()