Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(router): release Metal cache after checkpoint unload and eviction
Rishit Sharma6b87cb787b08
diff --git a/laya/router.py b/laya/router.pyindex 9e2bafc4..0d907adb 100644--- a/laya/router.py+++ b/laya/router.py@@ -715,6 +715,8 @@ class Router(HookRegistry):import torchif torch.cuda.is_available():torch.cuda.empty_cache()+ if hasattr(torch, "mps") and torch.backends.mps.is_available():+ torch.mps.empty_cache()except Exception:passreturn evicted@@ -786,6 +788,8 @@ class Router(HookRegistry):torch.cuda.empty_cache()if hasattr(torch, "xpu") and torch.xpu.is_available():torch.xpu.empty_cache()+ if hasattr(torch, "mps") and torch.backends.mps.is_available():+ torch.mps.empty_cache()except Exception:passbreak@@ -805,6 +809,8 @@ class Router(HookRegistry):torch.cuda.empty_cache()if hasattr(torch, "xpu") and torch.xpu.is_available():torch.xpu.empty_cache()+ if hasattr(torch, "mps") and torch.backends.mps.is_available():+ torch.mps.empty_cache()except Exception:passbreakdiff --git a/tests/test_router_memory.py b/tests/test_router_memory.pyindex b52e5e88..4812f117 100644--- a/tests/test_router_memory.py+++ b/tests/test_router_memory.py@@ -12,6 +12,8 @@ import reimport sysimport unittestfrom pathlib import Path+from types import SimpleNamespace+from unittest.mock import Mock, patch# The page under test lives in this checkout, so the library it is compared against has to come# from here too: an installed `laya` would let the page and a stale default agree.@@ -112,6 +114,31 @@ def derived_default(tree):class RouterMemoryTests(unittest.TestCase):+ def test_mps_cache_is_released_after_unload_and_eviction(self):+ for operation in ("one", "all", "evict"):+ for available in (False, True):+ with self.subTest(operation=operation, available=available):+ empty_cache = Mock()+ torch = SimpleNamespace(+ cuda=SimpleNamespace(is_available=lambda: False),+ backends=SimpleNamespace(mps=SimpleNamespace(is_available=lambda: available)),+ mps=SimpleNamespace(empty_cache=empty_cache),+ )+ r = Router(max_loaded=1)+ r._agents["english"] = MockAgent("en")+ r._order.append("english")+ with patch.dict(sys.modules, {"torch": torch}):+ if operation == "one":+ r.unload("english")+ elif operation == "all":+ r.unload()+ else:+ r._agents["multilingual"] = MockAgent("multi")+ r._order.append("multilingual")+ r._evict()+ self.assertNotIn("english", r.loaded)+ self.assertEqual(empty_cache.call_count, int(available))+def test_evict_and_unload_memory_release(self):r = Router(max_loaded=1)r.attach("english", MockAgent("en"))