Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
fix(moe): isolate layer-index errors from other numeric segments in discovery
Demetrios Agourakis85678c30cce8
diff --git a/python/src/edge0/moe/spec.py b/python/src/edge0/moe/spec.pyindex ced4ac4..a39d403 100644--- a/python/src/edge0/moe/spec.py+++ b/python/src/edge0/moe/spec.py@@ -110,6 +110,35 @@ class MoESpec:obj = getattr(obj, part)return obj+ def layer_exists(self, model, layer: int) -> bool:+ """True if ``layer`` resolves at block_path's layer-index segment.++ Used by layer-count discovery (``install_streaming_experts`` with+ ``num_layers=None``) to find where the layer list ends. Only an+ ``AttributeError``/``IndexError`` raised while resolving the+ segment templated by ``{layer}`` itself means "past the last+ layer"; the same errors raised by a *different* segment further+ down ``block_path`` (e.g. a fixed expert-slot index) indicate a+ bug in the spec or model and are re-raised rather than read as+ end-of-list.+ """+ template_parts = self.block_path.split(".")+ layer_pos = next(+ i for i, p in enumerate(template_parts) if "{layer}" in p)+ obj = model+ for i, raw_part in enumerate(template_parts):+ part = raw_part.format(layer=layer)+ try:+ if part.isdigit():+ obj = obj[int(part)]+ else:+ obj = getattr(obj, part)+ except (AttributeError, IndexError):+ if i == layer_pos:+ return False+ raise+ return True+def layer_of(self, model, layer: int):"""Resolve the decoder layer object (the block's owner).diff --git a/python/src/edge0/streaming/install.py b/python/src/edge0/streaming/install.pyindex acc3585..c0c84de 100644--- a/python/src/edge0/streaming/install.py+++ b/python/src/edge0/streaming/install.py@@ -38,11 +38,7 @@ def install_streaming_experts("""if num_layers is None:n = 0- while True:- try:- spec.block_of(model, n)- except (AttributeError, IndexError): # past the last layer- break+ while spec.layer_exists(model, n):n += 1if n == 0:raise ValueError(diff --git a/python/tests/test_streaming_math.py b/python/tests/test_streaming_math.pyindex 660d289..b8aae01 100644--- a/python/tests/test_streaming_math.py+++ b/python/tests/test_streaming_math.py@@ -374,3 +374,41 @@ def test_install_discovers_layer_count(tmp_path):assert twins[1] is None and twins[2] is None # dense layersassert moe.switch_mlp is twins[0]twins[0].close()+++def test_layer_exists_stops_on_attribute_error():+ """The AttributeError half of ``layer_exists``'s except clause is+ reachable independently of IndexError: a family whose layer container+ is attribute-based (no list, no ``__getitem__``) runs off the end via+ a plain missing attribute, not an out-of-range index."""+ from types import SimpleNamespace++ spec = MoESpec(+ num_experts=N_EXPERTS, top_k=4, intermediate_size=INTER,+ key_template="layer_{layer}.mlp.switch_mlp",+ block_path="layer_{layer}",+ )+ model = SimpleNamespace(layer_0=object(), layer_1=object())+ assert spec.layer_exists(model, 0) is True+ assert spec.layer_exists(model, 1) is True+ assert spec.layer_exists(model, 2) is False # no `layer_2` attribute+++def test_layer_exists_reraises_unrelated_index_error():+ """An IndexError from a segment *other* than the ``{layer}`` slot is a+ real bug (a malformed block_path or a broken model), not end-of-list,+ and must propagate instead of being read as "past the last layer" --+ otherwise install_streaming_experts(num_layers=None) would silently+ under-count layers instead of surfacing the break."""+ from types import SimpleNamespace++ spec = MoESpec(+ num_experts=N_EXPERTS, top_k=4, intermediate_size=INTER,+ key_template="layers.{layer}.experts.9.switch_mlp",+ block_path="layers.{layer}.experts.9",+ )+ # `layer=0` is in range for `layers`, but the fixed trailing index `9`+ # is out of range for `experts` -- unrelated to layer-count discovery.+ model = SimpleNamespace(layers=[SimpleNamespace(experts=[object()])])+ with pytest.raises(IndexError):+ spec.layer_exists(model, 0)