Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
fix(streaming): stop layer discovery on IndexError too
Demetrios Agourakisc646c4085535
diff --git a/python/src/edge0/streaming/install.py b/python/src/edge0/streaming/install.pyindex 37b0e7e..acc3585 100644--- a/python/src/edge0/streaming/install.py+++ b/python/src/edge0/streaming/install.py@@ -41,7 +41,7 @@ def install_streaming_experts(while True:try:spec.block_of(model, n)- except AttributeError:+ except (AttributeError, IndexError): # past the last layerbreakn += 1if n == 0:diff --git a/python/tests/test_streaming_math.py b/python/tests/test_streaming_math.pyindex eeb0f19..660d289 100644--- a/python/tests/test_streaming_math.py+++ b/python/tests/test_streaming_math.py@@ -344,3 +344,33 @@ def test_double_buffered_swap(layer):ref2 = lay(x, mx.array([second], dtype=mx.int32))assert mx.allclose(out1, ref1).item()assert mx.allclose(out2, ref2).item()+++def test_install_discovers_layer_count(tmp_path):+ """install_streaming_experts(num_layers=None) probes block_path with+ increasing layer indices until it stops resolving. Layers live in a+ list, so running off the end raises IndexError, not AttributeError."""+ from types import SimpleNamespace++ from edge0.streaming.install import install_streaming_experts++ path = tmp_path / "w.safetensors"+ _write_shard(path, fuse_gu=False)+ spec = MoESpec(+ num_experts=N_EXPERTS, top_k=4, intermediate_size=INTER,+ quant=QuantSpec(bits=4, group_size=64),+ layout=WeightLayout.SEPARATE,+ key_template="layers.{layer}.mlp.switch_mlp",+ block_path="layers.{layer}.mlp",+ )+ moe = SimpleNamespace(switch_mlp=object())+ model = SimpleNamespace(layers=[SimpleNamespace(mlp=moe),+ SimpleNamespace(mlp=SimpleNamespace()),+ SimpleNamespace(mlp=SimpleNamespace())])+ twins = install_streaming_experts(+ model, [SafetensorsMmap(str(path))], spec, options=_options())+ assert len(twins) == 3+ assert isinstance(twins[0], StreamingSwitchGLU)+ assert twins[1] is None and twins[2] is None # dense layers+ assert moe.switch_mlp is twins[0]+ twins[0].close()