Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(router): release Metal cache after checkpoint unload and eviction

Rishit Sharma6b87cb787b08
diff --git a/laya/router.py b/laya/router.py
index 9e2bafc4..0d907adb 100644
--- a/laya/router.py
+++ b/laya/router.py
@@ -715,6 +715,8 @@ class Router(HookRegistry):
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
+ if hasattr(torch, "mps") and torch.backends.mps.is_available():
+ torch.mps.empty_cache()
except Exception:
pass
return evicted
@@ -786,6 +788,8 @@ class Router(HookRegistry):
torch.cuda.empty_cache()
if hasattr(torch, "xpu") and torch.xpu.is_available():
torch.xpu.empty_cache()
+ if hasattr(torch, "mps") and torch.backends.mps.is_available():
+ torch.mps.empty_cache()
except Exception:
pass
break
@@ -805,6 +809,8 @@ class Router(HookRegistry):
torch.cuda.empty_cache()
if hasattr(torch, "xpu") and torch.xpu.is_available():
torch.xpu.empty_cache()
+ if hasattr(torch, "mps") and torch.backends.mps.is_available():
+ torch.mps.empty_cache()
except Exception:
pass
break
diff --git a/tests/test_router_memory.py b/tests/test_router_memory.py
index b52e5e88..4812f117 100644
--- a/tests/test_router_memory.py
+++ b/tests/test_router_memory.py
@@ -12,6 +12,8 @@ import re
import sys
import unittest
from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
# The page under test lives in this checkout, so the library it is compared against has to come
# from here too: an installed `laya` would let the page and a stale default agree.
@@ -112,6 +114,31 @@ def derived_default(tree):
class RouterMemoryTests(unittest.TestCase):
+ def test_mps_cache_is_released_after_unload_and_eviction(self):
+ for operation in ("one", "all", "evict"):
+ for available in (False, True):
+ with self.subTest(operation=operation, available=available):
+ empty_cache = Mock()
+ torch = SimpleNamespace(
+ cuda=SimpleNamespace(is_available=lambda: False),
+ backends=SimpleNamespace(mps=SimpleNamespace(is_available=lambda: available)),
+ mps=SimpleNamespace(empty_cache=empty_cache),
+ )
+ r = Router(max_loaded=1)
+ r._agents["english"] = MockAgent("en")
+ r._order.append("english")
+ with patch.dict(sys.modules, {"torch": torch}):
+ if operation == "one":
+ r.unload("english")
+ elif operation == "all":
+ r.unload()
+ else:
+ r._agents["multilingual"] = MockAgent("multi")
+ r._order.append("multilingual")
+ r._evict()
+ self.assertNotIn("english", r.loaded)
+ self.assertEqual(empty_cache.call_count, int(available))
+
def test_evict_and_unload_memory_release(self):
r = Router(max_loaded=1)
r.attach("english", MockAgent("en"))