Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python

fix(streaming): stop layer discovery on IndexError too

Demetrios Agourakisc646c4085535
diff --git a/python/src/edge0/streaming/install.py b/python/src/edge0/streaming/install.py
index 37b0e7e..acc3585 100644
--- a/python/src/edge0/streaming/install.py
+++ b/python/src/edge0/streaming/install.py
@@ -41,7 +41,7 @@ def install_streaming_experts(
while True:
try:
spec.block_of(model, n)
- except AttributeError:
+ except (AttributeError, IndexError): # past the last layer
break
n += 1
if n == 0:
diff --git a/python/tests/test_streaming_math.py b/python/tests/test_streaming_math.py
index eeb0f19..660d289 100644
--- a/python/tests/test_streaming_math.py
+++ b/python/tests/test_streaming_math.py
@@ -344,3 +344,33 @@ def test_double_buffered_swap(layer):
ref2 = lay(x, mx.array([second], dtype=mx.int32))
assert mx.allclose(out1, ref1).item()
assert mx.allclose(out2, ref2).item()
+
+
+def test_install_discovers_layer_count(tmp_path):
+ """install_streaming_experts(num_layers=None) probes block_path with
+ increasing layer indices until it stops resolving. Layers live in a
+ list, so running off the end raises IndexError, not AttributeError."""
+ from types import SimpleNamespace
+
+ from edge0.streaming.install import install_streaming_experts
+
+ path = tmp_path / "w.safetensors"
+ _write_shard(path, fuse_gu=False)
+ spec = MoESpec(
+ num_experts=N_EXPERTS, top_k=4, intermediate_size=INTER,
+ quant=QuantSpec(bits=4, group_size=64),
+ layout=WeightLayout.SEPARATE,
+ key_template="layers.{layer}.mlp.switch_mlp",
+ block_path="layers.{layer}.mlp",
+ )
+ moe = SimpleNamespace(switch_mlp=object())
+ model = SimpleNamespace(layers=[SimpleNamespace(mlp=moe),
+ SimpleNamespace(mlp=SimpleNamespace()),
+ SimpleNamespace(mlp=SimpleNamespace())])
+ twins = install_streaming_experts(
+ model, [SafetensorsMmap(str(path))], spec, options=_options())
+ assert len(twins) == 3
+ assert isinstance(twins[0], StreamingSwitchGLU)
+ assert twins[1] is None and twins[2] is None # dense layers
+ assert moe.switch_mlp is twins[0]
+ twins[0].close()