Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
test(router-memory): hold the residency page to the cap laya/router.py defaults to
Aashish9e5fb66de55b
diff --git a/tests/test_router_memory.py b/tests/test_router_memory.pyindex 486eba5a..866b30f4 100644--- a/tests/test_router_memory.py+++ b/tests/test_router_memory.py@@ -1,25 +1,210 @@+"""Residency cap: what `Router` holds, and what the residency page tells readers about it.++`examples/24_preload_and_memory.py` is the page a reader copies a `max_loaded` setting from. Its+banner used to open with "At the default `max_loaded=1`", a number `laya/router.py` retired in #180+and README.md documents as 2. So the page taught the retired cap while demonstrating eviction, and+a reader who believed it thought they had to raise the cap to stop the churn the page was showing.+The checks below read the page's own text and syntax tree -- no checkpoint, no device. `Router()`+loads nothing, so none of this needs weights.+"""+import ast+import reimport unittest+from pathlib import Path+from laya.router import Router+class MockAgent:def __init__(self, name):self.name = name++EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "24_preload_and_memory.py"+TREE = ast.parse(EXAMPLE.read_text(encoding="utf-8"))+CODE_DEFAULT = Router().max_loaded++# Every string constant the page carries: the banner blurb, the module docstring and each print's+# template. Sentences are split inside a constant -- two prints glued together are not one+# sentence -- but a folded one, so a sentence wrapped across two source lines stays one sentence.+CONSTANTS = [n.value for n in ast.walk(TREE) if isinstance(n, ast.Constant) and isinstance(n.value, str)]+PROSE = re.sub(r"\s+", " ", " ".join(CONSTANTS))+SENTENCES = [s.strip() for text in CONSTANTS+ for s in re.split(r"(?<=[.!?])\s+", re.sub(r"\s+", " ", text)) if s.strip()]+# A cap the page states, whether typed in or interpolated, and the typed-in subset of those.+CAP_MENTION = re.compile(r"max_loaded\s*=\s*(?:%[-\d.]*[dfi]|\d+)")+CAP = re.compile(r"max_loaded\s*=\s*(\d+)")+MAGNITUDE = re.compile(r"\b(?:thousands?|millions?|billions?|orders of magnitude)\b", re.I)+++def callee(call):+ """`laya.Router` / `r.route` / `timed` -- a dotted name, so a rename cannot hide a call."""+ parts, node = [], call.func+ while isinstance(node, ast.Attribute):+ parts.append(node.attr)+ node = node.value+ if isinstance(node, ast.Name):+ parts.append(node.id)+ return ".".join(reversed(parts))+++def assigned_names(node):+ """The names an assignment binds, including tuple unpacking (`_, ms = timed(...)`)."""+ out = []+ for target in node.targets:+ if isinstance(target, ast.Name):+ out.append(target.id)+ elif isinstance(target, ast.Tuple):+ out.extend(e.id for e in target.elts if isinstance(e, ast.Name))+ return out+++def measured(tree):+ """(wall-clock deltas, `timed()` medians) -- the two ways this page gets a real number."""+ perf, timed_ms = set(), set()+ for node in ast.walk(tree):+ if not isinstance(node, ast.Assign):+ continue+ names = set(assigned_names(node))+ value = node.value+ if isinstance(value, ast.BinOp) and isinstance(value.op, ast.Sub) \+ and "perf_counter" in ast.dump(value):+ perf |= names+ if isinstance(value, ast.Call) and callee(value) == "timed":+ timed_ms |= names+ return perf, timed_ms+++def printed_timed_route(tree):+ """The names a `timed(...)` call hands back when what it wrapped was a `route()` call."""+ for node in ast.walk(tree):+ if isinstance(node, ast.Call) and callee(node) == "timed" and node.args:+ if any(isinstance(c, ast.Call) and callee(c).split(".")[-1] == "route"+ for c in ast.walk(node.args[0])):+ return True+ return False+++def ratio_of_two_measurements(tree, perf, timed_ms):+ """A division whose operands are one wall-clock delta and one `timed()` median."""+ for node in ast.walk(tree):+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div):+ ids = {n.id for n in ast.walk(node) if isinstance(n, ast.Name)}+ if (ids & perf) and (ids & timed_ms):+ return sorted(ids & (perf | timed_ms))+ return []+++def derived_default(tree):+ """Names bound to a `Router(...)` construction's `.max_loaded` -- the default, read not typed."""+ out = set()+ for node in ast.walk(tree):+ value = node.value if isinstance(node, ast.Assign) else None+ if isinstance(value, ast.Attribute) and value.attr == "max_loaded" \+ and isinstance(value.value, ast.Call) and callee(value.value).split(".")[-1] == "Router":+ out |= set(assigned_names(node))+ return out++class RouterMemoryTests(unittest.TestCase):def test_evict_and_unload_memory_release(self):r = Router(max_loaded=1)r.attach("english", MockAgent("en"))r.attach("multilingual", MockAgent("multi"))self.assertEqual(len(r.loaded), 2)-+# Trigger unload of specific modelr.unload("english")self.assertNotIn("english", r.loaded)self.assertIn("multilingual", r.loaded)-+# Trigger unload allr.unload()self.assertEqual(len(r.loaded), 0)+ def test_the_default_cap_holds_more_languages_than_a_cap_of_one(self):+ """Why the retired number matters: the shipped default keeps more than one language hot.++ Agents are poked in through the internals `load()` would use, because `load()` needs a+ checkpoint and this suite runs with none. `_evict()` is the residency rule itself, so the+ expected count is derived from the cap rather than typed: if the library's default moves,+ this test follows it instead of going red for the wrong reason.+ """+ for cap in (CODE_DEFAULT, 1):+ r = Router(max_loaded=cap)+ for i in range(cap + 2): # two more languages than it can hold+ name = "model_%d" % i+ r._agents[name] = MockAgent(name)+ r._order.append(name)+ r._evict()+ self.assertEqual(len(r.loaded), cap, "max_loaded=%d held %r" % (cap, r.loaded))+ self.assertEqual(r.loaded[-1], "model_%d" % (cap + 1),+ "max_loaded=%d evicted the wrong end: %r" % (cap, r.loaded))+++class ExampleResidencyPageTests(unittest.TestCase):+ """What the residency page may claim about `max_loaded`, held to `laya/router.py`."""++ def test_default_is_more_than_one_checkpoint(self):+ # The witness that the page-claims below are about a library that keeps two resident: with+ # a default of 1 every claim would be true of the retired behaviour instead, and the+ # sentences checked here would have nothing to disagree with.+ self.assertGreater(CODE_DEFAULT, 1, repr(CODE_DEFAULT))++ def test_page_states_the_default_cap_that_router_has(self):+ # A sentence may only pair the word "default" with a cap the constructor actually takes.+ # Restoring "At the default `max_loaded=1`" fails here: 1 != 2.+ paired = [s for s in SENTENCES if re.search(r"\bdefault\b", s, re.I) and CAP_MENTION.search(s)]+ self.assertTrue(paired, "the page states no default cap at all, so this rule sees nothing")+ for sentence in paired:+ for stated in CAP.findall(sentence):+ self.assertEqual(int(stated), CODE_DEFAULT,+ "page says %r; laya/router.py says max_loaded=%d"+ % (sentence[:90], CODE_DEFAULT))++ def test_page_reads_its_default_from_the_library(self):+ # The banner must interpolate a value it obtained from a Router, not a literal it typed.+ names = derived_default(TREE)+ self.assertTrue(names, "nothing in the page assigns from `Router(...).max_loaded`")+ banners = [n for n in ast.walk(TREE)+ if isinstance(n, ast.Call) and callee(n) == "banner" and len(n.args) > 2]+ self.assertTrue(banners, "the page calls banner() with no blurb to hold the default")+ for call in banners:+ referenced = {n.id for n in ast.walk(call.args[2]) if isinstance(n, ast.Name)}+ self.assertTrue(referenced & names,+ "the banner states a default it did not read: %r vs %r"+ % (sorted(referenced), sorted(names)))++ def test_page_states_no_cap_it_never_configures(self):+ configured = {kw.value.value for node in ast.walk(TREE) if isinstance(node, ast.Call)+ for kw in node.keywords+ if kw.arg == "max_loaded" and isinstance(kw.value, ast.Constant)}+ stated = {int(n) for n in CAP.findall(PROSE)}+ allowed = set(configured) | {CODE_DEFAULT}+ self.assertTrue(configured, "the page configures no cap, so this rule has no baseline")+ self.assertEqual(sorted(stated - allowed), [],+ "page states %r but configures only %r" % (sorted(stated - allowed),+ sorted(allowed)))++ def test_reload_versus_route_ratio_is_measured(self):+ # The page used to close with "thousands of times more expensive than the routing itself" --+ # a magnitude with nothing measured behind it. Both sides must now be timed, and the number+ # printed must divide one measurement by the other.+ perf, timed_ms = measured(TREE)+ self.assertTrue(perf and timed_ms, "the page takes no wall-clock readings")+ self.assertTrue(printed_timed_route(TREE), "nothing times a route() call")+ self.assertTrue(ratio_of_two_measurements(TREE, perf, timed_ms),+ "no ratio divides a cold-build delta by a timed route median")++ def test_no_magnitude_word_without_a_measurement(self):+ # A caught-in-the-act class of lie: prose that asserts a scale this page never measured.+ self.assertFalse(MAGNITUDE.findall(PROSE),+ "page claims a scale: %r" % MAGNITUDE.findall(PROSE))+ # The rule above only has teeth if the page really does compare the two readings; a bare+ # `%`-placeholder or a digit in front of "times" is what that looks like in a template.+ self.assertRegex(PROSE, r"(?:%[-\d.]*[dfi]|\d+(?:\.\d+)?)\s+times",+ "the page makes no ratio claim, so the scale rule sees nothing")++if __name__ == "__main__":unittest.main()